1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "LegalizeTypes.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/Analysis/MemoryLocation.h"
25 #include "llvm/Analysis/VectorUtils.h"
26 #include "llvm/IR/DataLayout.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/TypeSize.h"
29 #include "llvm/Support/raw_ostream.h"
30 #include <numeric>
31
32 using namespace llvm;
33
34 #define DEBUG_TYPE "legalize-types"
35
36 //===----------------------------------------------------------------------===//
37 // Result Vector Scalarization: <1 x ty> -> ty.
38 //===----------------------------------------------------------------------===//
39
ScalarizeVectorResult(SDNode * N,unsigned ResNo)40 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
41 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
42 N->dump(&DAG));
43 SDValue R = SDValue();
44
45 switch (N->getOpcode()) {
46 default:
47 #ifndef NDEBUG
48 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
49 N->dump(&DAG);
50 dbgs() << "\n";
51 #endif
52 report_fatal_error("Do not know how to scalarize the result of this "
53 "operator!\n");
54
55 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
56 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
57 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
58 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
59 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
60 case ISD::FPOWI: R = ScalarizeVecRes_ExpOp(N); break;
61 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
62 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
63 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
64 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
65 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
66 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
67 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
68 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
69 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
70 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
71 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
72 case ISD::ANY_EXTEND_VECTOR_INREG:
73 case ISD::SIGN_EXTEND_VECTOR_INREG:
74 case ISD::ZERO_EXTEND_VECTOR_INREG:
75 R = ScalarizeVecRes_VecInregOp(N);
76 break;
77 case ISD::ABS:
78 case ISD::ANY_EXTEND:
79 case ISD::BITREVERSE:
80 case ISD::BSWAP:
81 case ISD::CTLZ:
82 case ISD::CTLZ_ZERO_UNDEF:
83 case ISD::CTPOP:
84 case ISD::CTTZ:
85 case ISD::CTTZ_ZERO_UNDEF:
86 case ISD::FABS:
87 case ISD::FCEIL:
88 case ISD::FCOS:
89 case ISD::FEXP:
90 case ISD::FEXP2:
91 case ISD::FEXP10:
92 case ISD::FFLOOR:
93 case ISD::FLOG:
94 case ISD::FLOG10:
95 case ISD::FLOG2:
96 case ISD::FNEARBYINT:
97 case ISD::FNEG:
98 case ISD::FREEZE:
99 case ISD::ARITH_FENCE:
100 case ISD::FP_EXTEND:
101 case ISD::FP_TO_SINT:
102 case ISD::FP_TO_UINT:
103 case ISD::FRINT:
104 case ISD::LRINT:
105 case ISD::LLRINT:
106 case ISD::FROUND:
107 case ISD::FROUNDEVEN:
108 case ISD::FSIN:
109 case ISD::FSQRT:
110 case ISD::FTRUNC:
111 case ISD::SIGN_EXTEND:
112 case ISD::SINT_TO_FP:
113 case ISD::TRUNCATE:
114 case ISD::UINT_TO_FP:
115 case ISD::ZERO_EXTEND:
116 case ISD::FCANONICALIZE:
117 R = ScalarizeVecRes_UnaryOp(N);
118 break;
119 case ISD::FFREXP:
120 R = ScalarizeVecRes_FFREXP(N, ResNo);
121 break;
122 case ISD::ADD:
123 case ISD::AND:
124 case ISD::FADD:
125 case ISD::FCOPYSIGN:
126 case ISD::FDIV:
127 case ISD::FMUL:
128 case ISD::FMINNUM:
129 case ISD::FMAXNUM:
130 case ISD::FMINNUM_IEEE:
131 case ISD::FMAXNUM_IEEE:
132 case ISD::FMINIMUM:
133 case ISD::FMAXIMUM:
134 case ISD::FLDEXP:
135 case ISD::SMIN:
136 case ISD::SMAX:
137 case ISD::UMIN:
138 case ISD::UMAX:
139
140 case ISD::SADDSAT:
141 case ISD::UADDSAT:
142 case ISD::SSUBSAT:
143 case ISD::USUBSAT:
144 case ISD::SSHLSAT:
145 case ISD::USHLSAT:
146
147 case ISD::FPOW:
148 case ISD::FREM:
149 case ISD::FSUB:
150 case ISD::MUL:
151 case ISD::MULHS:
152 case ISD::MULHU:
153 case ISD::OR:
154 case ISD::SDIV:
155 case ISD::SREM:
156 case ISD::SUB:
157 case ISD::UDIV:
158 case ISD::UREM:
159 case ISD::XOR:
160 case ISD::SHL:
161 case ISD::SRA:
162 case ISD::SRL:
163 case ISD::ROTL:
164 case ISD::ROTR:
165 R = ScalarizeVecRes_BinOp(N);
166 break;
167 case ISD::FMA:
168 case ISD::FSHL:
169 case ISD::FSHR:
170 R = ScalarizeVecRes_TernaryOp(N);
171 break;
172
173 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
174 case ISD::STRICT_##DAGN:
175 #include "llvm/IR/ConstrainedOps.def"
176 R = ScalarizeVecRes_StrictFPOp(N);
177 break;
178
179 case ISD::FP_TO_UINT_SAT:
180 case ISD::FP_TO_SINT_SAT:
181 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
182 break;
183
184 case ISD::UADDO:
185 case ISD::SADDO:
186 case ISD::USUBO:
187 case ISD::SSUBO:
188 case ISD::UMULO:
189 case ISD::SMULO:
190 R = ScalarizeVecRes_OverflowOp(N, ResNo);
191 break;
192 case ISD::SMULFIX:
193 case ISD::SMULFIXSAT:
194 case ISD::UMULFIX:
195 case ISD::UMULFIXSAT:
196 case ISD::SDIVFIX:
197 case ISD::SDIVFIXSAT:
198 case ISD::UDIVFIX:
199 case ISD::UDIVFIXSAT:
200 R = ScalarizeVecRes_FIX(N);
201 break;
202 }
203
204 // If R is null, the sub-method took care of registering the result.
205 if (R.getNode())
206 SetScalarizedVector(SDValue(N, ResNo), R);
207 }
208
ScalarizeVecRes_BinOp(SDNode * N)209 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
210 SDValue LHS = GetScalarizedVector(N->getOperand(0));
211 SDValue RHS = GetScalarizedVector(N->getOperand(1));
212 return DAG.getNode(N->getOpcode(), SDLoc(N),
213 LHS.getValueType(), LHS, RHS, N->getFlags());
214 }
215
ScalarizeVecRes_TernaryOp(SDNode * N)216 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
217 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
218 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
219 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
220 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
221 Op2, N->getFlags());
222 }
223
ScalarizeVecRes_FIX(SDNode * N)224 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
225 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
226 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
227 SDValue Op2 = N->getOperand(2);
228 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
229 Op2, N->getFlags());
230 }
231
ScalarizeVecRes_FFREXP(SDNode * N,unsigned ResNo)232 SDValue DAGTypeLegalizer::ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo) {
233 assert(N->getValueType(0).getVectorNumElements() == 1 &&
234 "Unexpected vector type!");
235 SDValue Elt = GetScalarizedVector(N->getOperand(0));
236
237 EVT VT0 = N->getValueType(0);
238 EVT VT1 = N->getValueType(1);
239 SDLoc dl(N);
240
241 SDNode *ScalarNode =
242 DAG.getNode(N->getOpcode(), dl,
243 {VT0.getScalarType(), VT1.getScalarType()}, Elt)
244 .getNode();
245
246 // Replace the other vector result not being explicitly scalarized here.
247 unsigned OtherNo = 1 - ResNo;
248 EVT OtherVT = N->getValueType(OtherNo);
249 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
250 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
251 } else {
252 SDValue OtherVal = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, OtherVT,
253 SDValue(ScalarNode, OtherNo));
254 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
255 }
256
257 return SDValue(ScalarNode, ResNo);
258 }
259
ScalarizeVecRes_StrictFPOp(SDNode * N)260 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
261 EVT VT = N->getValueType(0).getVectorElementType();
262 unsigned NumOpers = N->getNumOperands();
263 SDValue Chain = N->getOperand(0);
264 EVT ValueVTs[] = {VT, MVT::Other};
265 SDLoc dl(N);
266
267 SmallVector<SDValue, 4> Opers(NumOpers);
268
269 // The Chain is the first operand.
270 Opers[0] = Chain;
271
272 // Now process the remaining operands.
273 for (unsigned i = 1; i < NumOpers; ++i) {
274 SDValue Oper = N->getOperand(i);
275 EVT OperVT = Oper.getValueType();
276
277 if (OperVT.isVector()) {
278 if (getTypeAction(OperVT) == TargetLowering::TypeScalarizeVector)
279 Oper = GetScalarizedVector(Oper);
280 else
281 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
282 OperVT.getVectorElementType(), Oper,
283 DAG.getVectorIdxConstant(0, dl));
284 }
285
286 Opers[i] = Oper;
287 }
288
289 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
290 Opers, N->getFlags());
291
292 // Legalize the chain result - switch anything that used the old chain to
293 // use the new one.
294 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
295 return Result;
296 }
297
ScalarizeVecRes_OverflowOp(SDNode * N,unsigned ResNo)298 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
299 unsigned ResNo) {
300 SDLoc DL(N);
301 EVT ResVT = N->getValueType(0);
302 EVT OvVT = N->getValueType(1);
303
304 SDValue ScalarLHS, ScalarRHS;
305 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
306 ScalarLHS = GetScalarizedVector(N->getOperand(0));
307 ScalarRHS = GetScalarizedVector(N->getOperand(1));
308 } else {
309 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
310 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
311 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
312 ScalarLHS = ElemsLHS[0];
313 ScalarRHS = ElemsRHS[0];
314 }
315
316 SDVTList ScalarVTs = DAG.getVTList(
317 ResVT.getVectorElementType(), OvVT.getVectorElementType());
318 SDNode *ScalarNode = DAG.getNode(
319 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
320 ScalarNode->setFlags(N->getFlags());
321
322 // Replace the other vector result not being explicitly scalarized here.
323 unsigned OtherNo = 1 - ResNo;
324 EVT OtherVT = N->getValueType(OtherNo);
325 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
326 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
327 } else {
328 SDValue OtherVal = DAG.getNode(
329 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
330 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
331 }
332
333 return SDValue(ScalarNode, ResNo);
334 }
335
ScalarizeVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)336 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
337 unsigned ResNo) {
338 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
339 return GetScalarizedVector(Op);
340 }
341
ScalarizeVecRes_BITCAST(SDNode * N)342 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
343 SDValue Op = N->getOperand(0);
344 if (Op.getValueType().isVector()
345 && Op.getValueType().getVectorNumElements() == 1
346 && !isSimpleLegalType(Op.getValueType()))
347 Op = GetScalarizedVector(Op);
348 EVT NewVT = N->getValueType(0).getVectorElementType();
349 return DAG.getNode(ISD::BITCAST, SDLoc(N),
350 NewVT, Op);
351 }
352
ScalarizeVecRes_BUILD_VECTOR(SDNode * N)353 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
354 EVT EltVT = N->getValueType(0).getVectorElementType();
355 SDValue InOp = N->getOperand(0);
356 // The BUILD_VECTOR operands may be of wider element types and
357 // we may need to truncate them back to the requested return type.
358 if (EltVT.isInteger())
359 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
360 return InOp;
361 }
362
ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode * N)363 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
364 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
365 N->getValueType(0).getVectorElementType(),
366 N->getOperand(0), N->getOperand(1));
367 }
368
ScalarizeVecRes_FP_ROUND(SDNode * N)369 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
370 SDLoc DL(N);
371 SDValue Op = N->getOperand(0);
372 EVT OpVT = Op.getValueType();
373 // The result needs scalarizing, but it's not a given that the source does.
374 // See similar logic in ScalarizeVecRes_UnaryOp.
375 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
376 Op = GetScalarizedVector(Op);
377 } else {
378 EVT VT = OpVT.getVectorElementType();
379 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
380 DAG.getVectorIdxConstant(0, DL));
381 }
382 return DAG.getNode(ISD::FP_ROUND, DL,
383 N->getValueType(0).getVectorElementType(), Op,
384 N->getOperand(1));
385 }
386
ScalarizeVecRes_ExpOp(SDNode * N)387 SDValue DAGTypeLegalizer::ScalarizeVecRes_ExpOp(SDNode *N) {
388 SDValue Op = GetScalarizedVector(N->getOperand(0));
389 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op,
390 N->getOperand(1));
391 }
392
ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode * N)393 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
394 // The value to insert may have a wider type than the vector element type,
395 // so be sure to truncate it to the element type if necessary.
396 SDValue Op = N->getOperand(1);
397 EVT EltVT = N->getValueType(0).getVectorElementType();
398 if (Op.getValueType() != EltVT)
399 // FIXME: Can this happen for floating point types?
400 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
401 return Op;
402 }
403
ScalarizeVecRes_LOAD(LoadSDNode * N)404 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
405 assert(N->isUnindexed() && "Indexed vector load?");
406
407 SDValue Result = DAG.getLoad(
408 ISD::UNINDEXED, N->getExtensionType(),
409 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
410 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
411 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
412 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
413
414 // Legalize the chain result - switch anything that used the old chain to
415 // use the new one.
416 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
417 return Result;
418 }
419
ScalarizeVecRes_UnaryOp(SDNode * N)420 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
421 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
422 EVT DestVT = N->getValueType(0).getVectorElementType();
423 SDValue Op = N->getOperand(0);
424 EVT OpVT = Op.getValueType();
425 SDLoc DL(N);
426 // The result needs scalarizing, but it's not a given that the source does.
427 // This is a workaround for targets where it's impossible to scalarize the
428 // result of a conversion, because the source type is legal.
429 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
430 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
431 // legal and was not scalarized.
432 // See the similar logic in ScalarizeVecRes_SETCC
433 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
434 Op = GetScalarizedVector(Op);
435 } else {
436 EVT VT = OpVT.getVectorElementType();
437 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
438 DAG.getVectorIdxConstant(0, DL));
439 }
440 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
441 }
442
ScalarizeVecRes_InregOp(SDNode * N)443 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
444 EVT EltVT = N->getValueType(0).getVectorElementType();
445 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
446 SDValue LHS = GetScalarizedVector(N->getOperand(0));
447 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
448 LHS, DAG.getValueType(ExtVT));
449 }
450
ScalarizeVecRes_VecInregOp(SDNode * N)451 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
452 SDLoc DL(N);
453 SDValue Op = N->getOperand(0);
454
455 EVT OpVT = Op.getValueType();
456 EVT OpEltVT = OpVT.getVectorElementType();
457 EVT EltVT = N->getValueType(0).getVectorElementType();
458
459 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
460 Op = GetScalarizedVector(Op);
461 } else {
462 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
463 DAG.getVectorIdxConstant(0, DL));
464 }
465
466 switch (N->getOpcode()) {
467 case ISD::ANY_EXTEND_VECTOR_INREG:
468 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
469 case ISD::SIGN_EXTEND_VECTOR_INREG:
470 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
471 case ISD::ZERO_EXTEND_VECTOR_INREG:
472 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
473 }
474
475 llvm_unreachable("Illegal extend_vector_inreg opcode");
476 }
477
ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode * N)478 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
479 // If the operand is wider than the vector element type then it is implicitly
480 // truncated. Make that explicit here.
481 EVT EltVT = N->getValueType(0).getVectorElementType();
482 SDValue InOp = N->getOperand(0);
483 if (InOp.getValueType() != EltVT)
484 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
485 return InOp;
486 }
487
ScalarizeVecRes_VSELECT(SDNode * N)488 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
489 SDValue Cond = N->getOperand(0);
490 EVT OpVT = Cond.getValueType();
491 SDLoc DL(N);
492 // The vselect result and true/value operands needs scalarizing, but it's
493 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
494 // See the similar logic in ScalarizeVecRes_SETCC
495 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
496 Cond = GetScalarizedVector(Cond);
497 } else {
498 EVT VT = OpVT.getVectorElementType();
499 Cond = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
500 DAG.getVectorIdxConstant(0, DL));
501 }
502
503 SDValue LHS = GetScalarizedVector(N->getOperand(1));
504 TargetLowering::BooleanContent ScalarBool =
505 TLI.getBooleanContents(false, false);
506 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
507
508 // If integer and float booleans have different contents then we can't
509 // reliably optimize in all cases. There is a full explanation for this in
510 // DAGCombiner::visitSELECT() where the same issue affects folding
511 // (select C, 0, 1) to (xor C, 1).
512 if (TLI.getBooleanContents(false, false) !=
513 TLI.getBooleanContents(false, true)) {
514 // At least try the common case where the boolean is generated by a
515 // comparison.
516 if (Cond->getOpcode() == ISD::SETCC) {
517 EVT OpVT = Cond->getOperand(0).getValueType();
518 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
519 VecBool = TLI.getBooleanContents(OpVT);
520 } else
521 ScalarBool = TargetLowering::UndefinedBooleanContent;
522 }
523
524 EVT CondVT = Cond.getValueType();
525 if (ScalarBool != VecBool) {
526 switch (ScalarBool) {
527 case TargetLowering::UndefinedBooleanContent:
528 break;
529 case TargetLowering::ZeroOrOneBooleanContent:
530 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
531 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
532 // Vector read from all ones, scalar expects a single 1 so mask.
533 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
534 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
535 break;
536 case TargetLowering::ZeroOrNegativeOneBooleanContent:
537 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
538 VecBool == TargetLowering::ZeroOrOneBooleanContent);
539 // Vector reads from a one, scalar from all ones so sign extend.
540 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
541 Cond, DAG.getValueType(MVT::i1));
542 break;
543 }
544 }
545
546 // Truncate the condition if needed
547 auto BoolVT = getSetCCResultType(CondVT);
548 if (BoolVT.bitsLT(CondVT))
549 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
550
551 return DAG.getSelect(SDLoc(N),
552 LHS.getValueType(), Cond, LHS,
553 GetScalarizedVector(N->getOperand(2)));
554 }
555
ScalarizeVecRes_SELECT(SDNode * N)556 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
557 SDValue LHS = GetScalarizedVector(N->getOperand(1));
558 return DAG.getSelect(SDLoc(N),
559 LHS.getValueType(), N->getOperand(0), LHS,
560 GetScalarizedVector(N->getOperand(2)));
561 }
562
ScalarizeVecRes_SELECT_CC(SDNode * N)563 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
564 SDValue LHS = GetScalarizedVector(N->getOperand(2));
565 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
566 N->getOperand(0), N->getOperand(1),
567 LHS, GetScalarizedVector(N->getOperand(3)),
568 N->getOperand(4));
569 }
570
ScalarizeVecRes_UNDEF(SDNode * N)571 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
572 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
573 }
574
ScalarizeVecRes_VECTOR_SHUFFLE(SDNode * N)575 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
576 // Figure out if the scalar is the LHS or RHS and return it.
577 SDValue Arg = N->getOperand(2).getOperand(0);
578 if (Arg.isUndef())
579 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
580 unsigned Op = !cast<ConstantSDNode>(Arg)->isZero();
581 return GetScalarizedVector(N->getOperand(Op));
582 }
583
ScalarizeVecRes_FP_TO_XINT_SAT(SDNode * N)584 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
585 SDValue Src = N->getOperand(0);
586 EVT SrcVT = Src.getValueType();
587 SDLoc dl(N);
588
589 // Handle case where result is scalarized but operand is not
590 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
591 Src = GetScalarizedVector(Src);
592 else
593 Src = DAG.getNode(
594 ISD::EXTRACT_VECTOR_ELT, dl, SrcVT.getVectorElementType(), Src,
595 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
596
597 EVT DstVT = N->getValueType(0).getVectorElementType();
598 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
599 }
600
ScalarizeVecRes_SETCC(SDNode * N)601 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
602 assert(N->getValueType(0).isVector() &&
603 N->getOperand(0).getValueType().isVector() &&
604 "Operand types must be vectors");
605 SDValue LHS = N->getOperand(0);
606 SDValue RHS = N->getOperand(1);
607 EVT OpVT = LHS.getValueType();
608 EVT NVT = N->getValueType(0).getVectorElementType();
609 SDLoc DL(N);
610
611 // The result needs scalarizing, but it's not a given that the source does.
612 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
613 LHS = GetScalarizedVector(LHS);
614 RHS = GetScalarizedVector(RHS);
615 } else {
616 EVT VT = OpVT.getVectorElementType();
617 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
618 DAG.getVectorIdxConstant(0, DL));
619 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
620 DAG.getVectorIdxConstant(0, DL));
621 }
622
623 // Turn it into a scalar SETCC.
624 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
625 N->getOperand(2));
626 // Vectors may have a different boolean contents to scalars. Promote the
627 // value appropriately.
628 ISD::NodeType ExtendCode =
629 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
630 return DAG.getNode(ExtendCode, DL, NVT, Res);
631 }
632
ScalarizeVecRes_IS_FPCLASS(SDNode * N)633 SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
634 SDLoc DL(N);
635 SDValue Arg = N->getOperand(0);
636 SDValue Test = N->getOperand(1);
637 EVT ArgVT = Arg.getValueType();
638 EVT ResultVT = N->getValueType(0).getVectorElementType();
639
640 if (getTypeAction(ArgVT) == TargetLowering::TypeScalarizeVector) {
641 Arg = GetScalarizedVector(Arg);
642 } else {
643 EVT VT = ArgVT.getVectorElementType();
644 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Arg,
645 DAG.getVectorIdxConstant(0, DL));
646 }
647
648 SDValue Res =
649 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, {Arg, Test}, N->getFlags());
650 // Vectors may have a different boolean contents to scalars. Promote the
651 // value appropriately.
652 ISD::NodeType ExtendCode =
653 TargetLowering::getExtendForContent(TLI.getBooleanContents(ArgVT));
654 return DAG.getNode(ExtendCode, DL, ResultVT, Res);
655 }
656
657 //===----------------------------------------------------------------------===//
658 // Operand Vector Scalarization <1 x ty> -> ty.
659 //===----------------------------------------------------------------------===//
660
ScalarizeVectorOperand(SDNode * N,unsigned OpNo)661 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
662 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
663 N->dump(&DAG));
664 SDValue Res = SDValue();
665
666 switch (N->getOpcode()) {
667 default:
668 #ifndef NDEBUG
669 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
670 N->dump(&DAG);
671 dbgs() << "\n";
672 #endif
673 report_fatal_error("Do not know how to scalarize this operator's "
674 "operand!\n");
675 case ISD::BITCAST:
676 Res = ScalarizeVecOp_BITCAST(N);
677 break;
678 case ISD::ANY_EXTEND:
679 case ISD::ZERO_EXTEND:
680 case ISD::SIGN_EXTEND:
681 case ISD::TRUNCATE:
682 case ISD::FP_TO_SINT:
683 case ISD::FP_TO_UINT:
684 case ISD::SINT_TO_FP:
685 case ISD::UINT_TO_FP:
686 case ISD::LRINT:
687 case ISD::LLRINT:
688 Res = ScalarizeVecOp_UnaryOp(N);
689 break;
690 case ISD::STRICT_SINT_TO_FP:
691 case ISD::STRICT_UINT_TO_FP:
692 case ISD::STRICT_FP_TO_SINT:
693 case ISD::STRICT_FP_TO_UINT:
694 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
695 break;
696 case ISD::CONCAT_VECTORS:
697 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
698 break;
699 case ISD::EXTRACT_VECTOR_ELT:
700 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
701 break;
702 case ISD::VSELECT:
703 Res = ScalarizeVecOp_VSELECT(N);
704 break;
705 case ISD::SETCC:
706 Res = ScalarizeVecOp_VSETCC(N);
707 break;
708 case ISD::STORE:
709 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
710 break;
711 case ISD::STRICT_FP_ROUND:
712 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
713 break;
714 case ISD::FP_ROUND:
715 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
716 break;
717 case ISD::STRICT_FP_EXTEND:
718 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
719 break;
720 case ISD::FP_EXTEND:
721 Res = ScalarizeVecOp_FP_EXTEND(N);
722 break;
723 case ISD::VECREDUCE_FADD:
724 case ISD::VECREDUCE_FMUL:
725 case ISD::VECREDUCE_ADD:
726 case ISD::VECREDUCE_MUL:
727 case ISD::VECREDUCE_AND:
728 case ISD::VECREDUCE_OR:
729 case ISD::VECREDUCE_XOR:
730 case ISD::VECREDUCE_SMAX:
731 case ISD::VECREDUCE_SMIN:
732 case ISD::VECREDUCE_UMAX:
733 case ISD::VECREDUCE_UMIN:
734 case ISD::VECREDUCE_FMAX:
735 case ISD::VECREDUCE_FMIN:
736 case ISD::VECREDUCE_FMAXIMUM:
737 case ISD::VECREDUCE_FMINIMUM:
738 Res = ScalarizeVecOp_VECREDUCE(N);
739 break;
740 case ISD::VECREDUCE_SEQ_FADD:
741 case ISD::VECREDUCE_SEQ_FMUL:
742 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
743 break;
744 }
745
746 // If the result is null, the sub-method took care of registering results etc.
747 if (!Res.getNode()) return false;
748
749 // If the result is N, the sub-method updated N in place. Tell the legalizer
750 // core about this.
751 if (Res.getNode() == N)
752 return true;
753
754 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
755 "Invalid operand expansion");
756
757 ReplaceValueWith(SDValue(N, 0), Res);
758 return false;
759 }
760
761 /// If the value to convert is a vector that needs to be scalarized, it must be
762 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_BITCAST(SDNode * N)763 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
764 SDValue Elt = GetScalarizedVector(N->getOperand(0));
765 return DAG.getNode(ISD::BITCAST, SDLoc(N),
766 N->getValueType(0), Elt);
767 }
768
769 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
770 /// Do the operation on the element instead.
ScalarizeVecOp_UnaryOp(SDNode * N)771 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
772 assert(N->getValueType(0).getVectorNumElements() == 1 &&
773 "Unexpected vector type!");
774 SDValue Elt = GetScalarizedVector(N->getOperand(0));
775 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
776 N->getValueType(0).getScalarType(), Elt);
777 // Revectorize the result so the types line up with what the uses of this
778 // expression expect.
779 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
780 }
781
782 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
783 /// Do the strict FP operation on the element instead.
ScalarizeVecOp_UnaryOp_StrictFP(SDNode * N)784 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
785 assert(N->getValueType(0).getVectorNumElements() == 1 &&
786 "Unexpected vector type!");
787 SDValue Elt = GetScalarizedVector(N->getOperand(1));
788 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
789 { N->getValueType(0).getScalarType(), MVT::Other },
790 { N->getOperand(0), Elt });
791 // Legalize the chain result - switch anything that used the old chain to
792 // use the new one.
793 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
794 // Revectorize the result so the types line up with what the uses of this
795 // expression expect.
796 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
797
798 // Do our own replacement and return SDValue() to tell the caller that we
799 // handled all replacements since caller can only handle a single result.
800 ReplaceValueWith(SDValue(N, 0), Res);
801 return SDValue();
802 }
803
804 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
ScalarizeVecOp_CONCAT_VECTORS(SDNode * N)805 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
806 SmallVector<SDValue, 8> Ops(N->getNumOperands());
807 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
808 Ops[i] = GetScalarizedVector(N->getOperand(i));
809 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
810 }
811
812 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
813 /// so just return the element, ignoring the index.
ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode * N)814 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
815 EVT VT = N->getValueType(0);
816 SDValue Res = GetScalarizedVector(N->getOperand(0));
817 if (Res.getValueType() != VT)
818 Res = VT.isFloatingPoint()
819 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
820 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
821 return Res;
822 }
823
824 /// If the input condition is a vector that needs to be scalarized, it must be
825 /// <1 x i1>, so just convert to a normal ISD::SELECT
826 /// (still with vector output type since that was acceptable if we got here).
ScalarizeVecOp_VSELECT(SDNode * N)827 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
828 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
829 EVT VT = N->getValueType(0);
830
831 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
832 N->getOperand(2));
833 }
834
835 /// If the operand is a vector that needs to be scalarized then the
836 /// result must be v1i1, so just convert to a scalar SETCC and wrap
837 /// with a scalar_to_vector since the res type is legal if we got here
ScalarizeVecOp_VSETCC(SDNode * N)838 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
839 assert(N->getValueType(0).isVector() &&
840 N->getOperand(0).getValueType().isVector() &&
841 "Operand types must be vectors");
842 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
843
844 EVT VT = N->getValueType(0);
845 SDValue LHS = GetScalarizedVector(N->getOperand(0));
846 SDValue RHS = GetScalarizedVector(N->getOperand(1));
847
848 EVT OpVT = N->getOperand(0).getValueType();
849 EVT NVT = VT.getVectorElementType();
850 SDLoc DL(N);
851 // Turn it into a scalar SETCC.
852 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
853 N->getOperand(2));
854
855 // Vectors may have a different boolean contents to scalars. Promote the
856 // value appropriately.
857 ISD::NodeType ExtendCode =
858 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
859
860 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
861
862 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
863 }
864
865 /// If the value to store is a vector that needs to be scalarized, it must be
866 /// <1 x ty>. Just store the element.
ScalarizeVecOp_STORE(StoreSDNode * N,unsigned OpNo)867 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
868 assert(N->isUnindexed() && "Indexed store of one-element vector?");
869 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
870 SDLoc dl(N);
871
872 if (N->isTruncatingStore())
873 return DAG.getTruncStore(
874 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
875 N->getBasePtr(), N->getPointerInfo(),
876 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
877 N->getMemOperand()->getFlags(), N->getAAInfo());
878
879 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
880 N->getBasePtr(), N->getPointerInfo(),
881 N->getOriginalAlign(), N->getMemOperand()->getFlags(),
882 N->getAAInfo());
883 }
884
885 /// If the value to round is a vector that needs to be scalarized, it must be
886 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_ROUND(SDNode * N,unsigned OpNo)887 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
888 assert(OpNo == 0 && "Wrong operand for scalarization!");
889 SDValue Elt = GetScalarizedVector(N->getOperand(0));
890 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
891 N->getValueType(0).getVectorElementType(), Elt,
892 N->getOperand(1));
893 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
894 }
895
ScalarizeVecOp_STRICT_FP_ROUND(SDNode * N,unsigned OpNo)896 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
897 unsigned OpNo) {
898 assert(OpNo == 1 && "Wrong operand for scalarization!");
899 SDValue Elt = GetScalarizedVector(N->getOperand(1));
900 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
901 { N->getValueType(0).getVectorElementType(),
902 MVT::Other },
903 { N->getOperand(0), Elt, N->getOperand(2) });
904 // Legalize the chain result - switch anything that used the old chain to
905 // use the new one.
906 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
907
908 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
909
910 // Do our own replacement and return SDValue() to tell the caller that we
911 // handled all replacements since caller can only handle a single result.
912 ReplaceValueWith(SDValue(N, 0), Res);
913 return SDValue();
914 }
915
916 /// If the value to extend is a vector that needs to be scalarized, it must be
917 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_EXTEND(SDNode * N)918 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
919 SDValue Elt = GetScalarizedVector(N->getOperand(0));
920 SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
921 N->getValueType(0).getVectorElementType(), Elt);
922 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
923 }
924
925 /// If the value to extend is a vector that needs to be scalarized, it must be
926 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_STRICT_FP_EXTEND(SDNode * N)927 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
928 SDValue Elt = GetScalarizedVector(N->getOperand(1));
929 SDValue Res =
930 DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
931 {N->getValueType(0).getVectorElementType(), MVT::Other},
932 {N->getOperand(0), Elt});
933 // Legalize the chain result - switch anything that used the old chain to
934 // use the new one.
935 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
936
937 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
938
939 // Do our own replacement and return SDValue() to tell the caller that we
940 // handled all replacements since caller can only handle a single result.
941 ReplaceValueWith(SDValue(N, 0), Res);
942 return SDValue();
943 }
944
ScalarizeVecOp_VECREDUCE(SDNode * N)945 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
946 SDValue Res = GetScalarizedVector(N->getOperand(0));
947 // Result type may be wider than element type.
948 if (Res.getValueType() != N->getValueType(0))
949 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
950 return Res;
951 }
952
ScalarizeVecOp_VECREDUCE_SEQ(SDNode * N)953 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
954 SDValue AccOp = N->getOperand(0);
955 SDValue VecOp = N->getOperand(1);
956
957 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
958
959 SDValue Op = GetScalarizedVector(VecOp);
960 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
961 AccOp, Op, N->getFlags());
962 }
963
964 //===----------------------------------------------------------------------===//
965 // Result Vector Splitting
966 //===----------------------------------------------------------------------===//
967
968 /// This method is called when the specified result of the specified node is
969 /// found to need vector splitting. At this point, the node may also have
970 /// invalid operands or may have other results that need legalization, we just
971 /// know that (at least) one result needs vector splitting.
SplitVectorResult(SDNode * N,unsigned ResNo)972 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
973 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
974 SDValue Lo, Hi;
975
976 // See if the target wants to custom expand this node.
977 if (CustomLowerNode(N, N->getValueType(ResNo), true))
978 return;
979
980 switch (N->getOpcode()) {
981 default:
982 #ifndef NDEBUG
983 dbgs() << "SplitVectorResult #" << ResNo << ": ";
984 N->dump(&DAG);
985 dbgs() << "\n";
986 #endif
987 report_fatal_error("Do not know how to split the result of this "
988 "operator!\n");
989
990 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
991 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
992 case ISD::VSELECT:
993 case ISD::SELECT:
994 case ISD::VP_MERGE:
995 case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
996 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
997 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
998 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
999 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1000 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1001 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1002 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1003 case ISD::FPOWI:
1004 case ISD::FLDEXP:
1005 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1006 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1007 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1008 case ISD::SPLAT_VECTOR:
1009 case ISD::SCALAR_TO_VECTOR:
1010 SplitVecRes_ScalarOp(N, Lo, Hi);
1011 break;
1012 case ISD::STEP_VECTOR:
1013 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1014 break;
1015 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1016 case ISD::LOAD:
1017 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
1018 break;
1019 case ISD::VP_LOAD:
1020 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(N), Lo, Hi);
1021 break;
1022 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1023 SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N), Lo, Hi);
1024 break;
1025 case ISD::MLOAD:
1026 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
1027 break;
1028 case ISD::MGATHER:
1029 case ISD::VP_GATHER:
1030 SplitVecRes_Gather(cast<MemSDNode>(N), Lo, Hi, /*SplitSETCC*/ true);
1031 break;
1032 case ISD::SETCC:
1033 case ISD::VP_SETCC:
1034 SplitVecRes_SETCC(N, Lo, Hi);
1035 break;
1036 case ISD::VECTOR_REVERSE:
1037 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1038 break;
1039 case ISD::VECTOR_SHUFFLE:
1040 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
1041 break;
1042 case ISD::VECTOR_SPLICE:
1043 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1044 break;
1045 case ISD::VECTOR_DEINTERLEAVE:
1046 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1047 return;
1048 case ISD::VECTOR_INTERLEAVE:
1049 SplitVecRes_VECTOR_INTERLEAVE(N);
1050 return;
1051 case ISD::VAARG:
1052 SplitVecRes_VAARG(N, Lo, Hi);
1053 break;
1054
1055 case ISD::ANY_EXTEND_VECTOR_INREG:
1056 case ISD::SIGN_EXTEND_VECTOR_INREG:
1057 case ISD::ZERO_EXTEND_VECTOR_INREG:
1058 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1059 break;
1060
1061 case ISD::ABS:
1062 case ISD::VP_ABS:
1063 case ISD::BITREVERSE:
1064 case ISD::VP_BITREVERSE:
1065 case ISD::BSWAP:
1066 case ISD::VP_BSWAP:
1067 case ISD::CTLZ:
1068 case ISD::VP_CTLZ:
1069 case ISD::CTTZ:
1070 case ISD::VP_CTTZ:
1071 case ISD::CTLZ_ZERO_UNDEF:
1072 case ISD::VP_CTLZ_ZERO_UNDEF:
1073 case ISD::CTTZ_ZERO_UNDEF:
1074 case ISD::VP_CTTZ_ZERO_UNDEF:
1075 case ISD::CTPOP:
1076 case ISD::VP_CTPOP:
1077 case ISD::FABS: case ISD::VP_FABS:
1078 case ISD::FCEIL:
1079 case ISD::VP_FCEIL:
1080 case ISD::FCOS:
1081 case ISD::FEXP:
1082 case ISD::FEXP2:
1083 case ISD::FEXP10:
1084 case ISD::FFLOOR:
1085 case ISD::VP_FFLOOR:
1086 case ISD::FLOG:
1087 case ISD::FLOG10:
1088 case ISD::FLOG2:
1089 case ISD::FNEARBYINT:
1090 case ISD::VP_FNEARBYINT:
1091 case ISD::FNEG: case ISD::VP_FNEG:
1092 case ISD::FREEZE:
1093 case ISD::ARITH_FENCE:
1094 case ISD::FP_EXTEND:
1095 case ISD::VP_FP_EXTEND:
1096 case ISD::FP_ROUND:
1097 case ISD::VP_FP_ROUND:
1098 case ISD::FP_TO_SINT:
1099 case ISD::VP_FP_TO_SINT:
1100 case ISD::FP_TO_UINT:
1101 case ISD::VP_FP_TO_UINT:
1102 case ISD::FRINT:
1103 case ISD::VP_FRINT:
1104 case ISD::LRINT:
1105 case ISD::LLRINT:
1106 case ISD::FROUND:
1107 case ISD::VP_FROUND:
1108 case ISD::FROUNDEVEN:
1109 case ISD::VP_FROUNDEVEN:
1110 case ISD::FSIN:
1111 case ISD::FSQRT: case ISD::VP_SQRT:
1112 case ISD::FTRUNC:
1113 case ISD::VP_FROUNDTOZERO:
1114 case ISD::SINT_TO_FP:
1115 case ISD::VP_SINT_TO_FP:
1116 case ISD::TRUNCATE:
1117 case ISD::VP_TRUNCATE:
1118 case ISD::UINT_TO_FP:
1119 case ISD::VP_UINT_TO_FP:
1120 case ISD::FCANONICALIZE:
1121 SplitVecRes_UnaryOp(N, Lo, Hi);
1122 break;
1123 case ISD::FFREXP:
1124 SplitVecRes_FFREXP(N, ResNo, Lo, Hi);
1125 break;
1126
1127 case ISD::ANY_EXTEND:
1128 case ISD::SIGN_EXTEND:
1129 case ISD::ZERO_EXTEND:
1130 case ISD::VP_SIGN_EXTEND:
1131 case ISD::VP_ZERO_EXTEND:
1132 SplitVecRes_ExtendOp(N, Lo, Hi);
1133 break;
1134
1135 case ISD::ADD: case ISD::VP_ADD:
1136 case ISD::SUB: case ISD::VP_SUB:
1137 case ISD::MUL: case ISD::VP_MUL:
1138 case ISD::MULHS:
1139 case ISD::MULHU:
1140 case ISD::FADD: case ISD::VP_FADD:
1141 case ISD::FSUB: case ISD::VP_FSUB:
1142 case ISD::FMUL: case ISD::VP_FMUL:
1143 case ISD::FMINNUM: case ISD::VP_FMINNUM:
1144 case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
1145 case ISD::FMINIMUM:
1146 case ISD::VP_FMINIMUM:
1147 case ISD::FMAXIMUM:
1148 case ISD::VP_FMAXIMUM:
1149 case ISD::SDIV: case ISD::VP_SDIV:
1150 case ISD::UDIV: case ISD::VP_UDIV:
1151 case ISD::FDIV: case ISD::VP_FDIV:
1152 case ISD::FPOW:
1153 case ISD::AND: case ISD::VP_AND:
1154 case ISD::OR: case ISD::VP_OR:
1155 case ISD::XOR: case ISD::VP_XOR:
1156 case ISD::SHL: case ISD::VP_SHL:
1157 case ISD::SRA: case ISD::VP_ASHR:
1158 case ISD::SRL: case ISD::VP_LSHR:
1159 case ISD::UREM: case ISD::VP_UREM:
1160 case ISD::SREM: case ISD::VP_SREM:
1161 case ISD::FREM: case ISD::VP_FREM:
1162 case ISD::SMIN: case ISD::VP_SMIN:
1163 case ISD::SMAX: case ISD::VP_SMAX:
1164 case ISD::UMIN: case ISD::VP_UMIN:
1165 case ISD::UMAX: case ISD::VP_UMAX:
1166 case ISD::SADDSAT:
1167 case ISD::UADDSAT:
1168 case ISD::SSUBSAT:
1169 case ISD::USUBSAT:
1170 case ISD::SSHLSAT:
1171 case ISD::USHLSAT:
1172 case ISD::ROTL:
1173 case ISD::ROTR:
1174 case ISD::VP_FCOPYSIGN:
1175 SplitVecRes_BinOp(N, Lo, Hi);
1176 break;
1177 case ISD::FMA: case ISD::VP_FMA:
1178 case ISD::FSHL:
1179 case ISD::VP_FSHL:
1180 case ISD::FSHR:
1181 case ISD::VP_FSHR:
1182 SplitVecRes_TernaryOp(N, Lo, Hi);
1183 break;
1184
1185 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1186 case ISD::STRICT_##DAGN:
1187 #include "llvm/IR/ConstrainedOps.def"
1188 SplitVecRes_StrictFPOp(N, Lo, Hi);
1189 break;
1190
1191 case ISD::FP_TO_UINT_SAT:
1192 case ISD::FP_TO_SINT_SAT:
1193 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1194 break;
1195
1196 case ISD::UADDO:
1197 case ISD::SADDO:
1198 case ISD::USUBO:
1199 case ISD::SSUBO:
1200 case ISD::UMULO:
1201 case ISD::SMULO:
1202 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1203 break;
1204 case ISD::SMULFIX:
1205 case ISD::SMULFIXSAT:
1206 case ISD::UMULFIX:
1207 case ISD::UMULFIXSAT:
1208 case ISD::SDIVFIX:
1209 case ISD::SDIVFIXSAT:
1210 case ISD::UDIVFIX:
1211 case ISD::UDIVFIXSAT:
1212 SplitVecRes_FIX(N, Lo, Hi);
1213 break;
1214 case ISD::EXPERIMENTAL_VP_REVERSE:
1215 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1216 break;
1217 }
1218
1219 // If Lo/Hi is null, the sub-method took care of registering results etc.
1220 if (Lo.getNode())
1221 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1222 }
1223
IncrementPointer(MemSDNode * N,EVT MemVT,MachinePointerInfo & MPI,SDValue & Ptr,uint64_t * ScaledOffset)1224 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1225 MachinePointerInfo &MPI, SDValue &Ptr,
1226 uint64_t *ScaledOffset) {
1227 SDLoc DL(N);
1228 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1229
1230 if (MemVT.isScalableVector()) {
1231 SDNodeFlags Flags;
1232 SDValue BytesIncrement = DAG.getVScale(
1233 DL, Ptr.getValueType(),
1234 APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1235 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1236 Flags.setNoUnsignedWrap(true);
1237 if (ScaledOffset)
1238 *ScaledOffset += IncrementSize;
1239 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1240 Flags);
1241 } else {
1242 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1243 // Increment the pointer to the other half.
1244 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::getFixed(IncrementSize));
1245 }
1246 }
1247
SplitMask(SDValue Mask)1248 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1249 return SplitMask(Mask, SDLoc(Mask));
1250 }
1251
SplitMask(SDValue Mask,const SDLoc & DL)1252 std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1253 const SDLoc &DL) {
1254 SDValue MaskLo, MaskHi;
1255 EVT MaskVT = Mask.getValueType();
1256 if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
1257 GetSplitVector(Mask, MaskLo, MaskHi);
1258 else
1259 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
1260 return std::make_pair(MaskLo, MaskHi);
1261 }
1262
SplitVecRes_BinOp(SDNode * N,SDValue & Lo,SDValue & Hi)1263 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1264 SDValue LHSLo, LHSHi;
1265 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1266 SDValue RHSLo, RHSHi;
1267 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1268 SDLoc dl(N);
1269
1270 const SDNodeFlags Flags = N->getFlags();
1271 unsigned Opcode = N->getOpcode();
1272 if (N->getNumOperands() == 2) {
1273 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1274 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1275 return;
1276 }
1277
1278 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1279 assert(N->isVPOpcode() && "Expected VP opcode");
1280
1281 SDValue MaskLo, MaskHi;
1282 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
1283
1284 SDValue EVLLo, EVLHi;
1285 std::tie(EVLLo, EVLHi) =
1286 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
1287
1288 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(),
1289 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1290 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(),
1291 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1292 }
1293
SplitVecRes_TernaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1294 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1295 SDValue &Hi) {
1296 SDValue Op0Lo, Op0Hi;
1297 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1298 SDValue Op1Lo, Op1Hi;
1299 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1300 SDValue Op2Lo, Op2Hi;
1301 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1302 SDLoc dl(N);
1303
1304 const SDNodeFlags Flags = N->getFlags();
1305 unsigned Opcode = N->getOpcode();
1306 if (N->getNumOperands() == 3) {
1307 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, Op2Lo, Flags);
1308 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, Op2Hi, Flags);
1309 return;
1310 }
1311
1312 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
1313 assert(N->isVPOpcode() && "Expected VP opcode");
1314
1315 SDValue MaskLo, MaskHi;
1316 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
1317
1318 SDValue EVLLo, EVLHi;
1319 std::tie(EVLLo, EVLHi) =
1320 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), dl);
1321
1322 Lo = DAG.getNode(Opcode, dl, Op0Lo.getValueType(),
1323 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1324 Hi = DAG.getNode(Opcode, dl, Op0Hi.getValueType(),
1325 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1326 }
1327
SplitVecRes_FIX(SDNode * N,SDValue & Lo,SDValue & Hi)1328 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1329 SDValue LHSLo, LHSHi;
1330 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1331 SDValue RHSLo, RHSHi;
1332 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1333 SDLoc dl(N);
1334 SDValue Op2 = N->getOperand(2);
1335
1336 unsigned Opcode = N->getOpcode();
1337 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1338 N->getFlags());
1339 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1340 N->getFlags());
1341 }
1342
SplitVecRes_BITCAST(SDNode * N,SDValue & Lo,SDValue & Hi)1343 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1344 SDValue &Hi) {
1345 // We know the result is a vector. The input may be either a vector or a
1346 // scalar value.
1347 EVT LoVT, HiVT;
1348 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1349 SDLoc dl(N);
1350
1351 SDValue InOp = N->getOperand(0);
1352 EVT InVT = InOp.getValueType();
1353
1354 // Handle some special cases efficiently.
1355 switch (getTypeAction(InVT)) {
1356 case TargetLowering::TypeLegal:
1357 case TargetLowering::TypePromoteInteger:
1358 case TargetLowering::TypePromoteFloat:
1359 case TargetLowering::TypeSoftPromoteHalf:
1360 case TargetLowering::TypeSoftenFloat:
1361 case TargetLowering::TypeScalarizeVector:
1362 case TargetLowering::TypeWidenVector:
1363 break;
1364 case TargetLowering::TypeExpandInteger:
1365 case TargetLowering::TypeExpandFloat:
1366 // A scalar to vector conversion, where the scalar needs expansion.
1367 // If the vector is being split in two then we can just convert the
1368 // expanded pieces.
1369 if (LoVT == HiVT) {
1370 GetExpandedOp(InOp, Lo, Hi);
1371 if (DAG.getDataLayout().isBigEndian())
1372 std::swap(Lo, Hi);
1373 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1374 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1375 return;
1376 }
1377 break;
1378 case TargetLowering::TypeSplitVector:
1379 // If the input is a vector that needs to be split, convert each split
1380 // piece of the input now.
1381 GetSplitVector(InOp, Lo, Hi);
1382 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1383 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1384 return;
1385 case TargetLowering::TypeScalarizeScalableVector:
1386 report_fatal_error("Scalarization of scalable vectors is not supported.");
1387 }
1388
1389 // In the general case, convert the input to an integer and split it by hand.
1390 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1391 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1392 if (DAG.getDataLayout().isBigEndian())
1393 std::swap(LoIntVT, HiIntVT);
1394
1395 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1396
1397 if (DAG.getDataLayout().isBigEndian())
1398 std::swap(Lo, Hi);
1399 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1400 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1401 }
1402
SplitVecRes_BUILD_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1403 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1404 SDValue &Hi) {
1405 EVT LoVT, HiVT;
1406 SDLoc dl(N);
1407 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1408 unsigned LoNumElts = LoVT.getVectorNumElements();
1409 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1410 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1411
1412 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1413 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1414 }
1415
SplitVecRes_CONCAT_VECTORS(SDNode * N,SDValue & Lo,SDValue & Hi)1416 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1417 SDValue &Hi) {
1418 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1419 SDLoc dl(N);
1420 unsigned NumSubvectors = N->getNumOperands() / 2;
1421 if (NumSubvectors == 1) {
1422 Lo = N->getOperand(0);
1423 Hi = N->getOperand(1);
1424 return;
1425 }
1426
1427 EVT LoVT, HiVT;
1428 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1429
1430 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1431 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1432
1433 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1434 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1435 }
1436
SplitVecRes_EXTRACT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1437 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1438 SDValue &Hi) {
1439 SDValue Vec = N->getOperand(0);
1440 SDValue Idx = N->getOperand(1);
1441 SDLoc dl(N);
1442
1443 EVT LoVT, HiVT;
1444 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1445
1446 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1447 uint64_t IdxVal = Idx->getAsZExtVal();
1448 Hi = DAG.getNode(
1449 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1450 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1451 }
1452
SplitVecRes_INSERT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1453 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1454 SDValue &Hi) {
1455 SDValue Vec = N->getOperand(0);
1456 SDValue SubVec = N->getOperand(1);
1457 SDValue Idx = N->getOperand(2);
1458 SDLoc dl(N);
1459 GetSplitVector(Vec, Lo, Hi);
1460
1461 EVT VecVT = Vec.getValueType();
1462 EVT LoVT = Lo.getValueType();
1463 EVT SubVecVT = SubVec.getValueType();
1464 unsigned VecElems = VecVT.getVectorMinNumElements();
1465 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1466 unsigned LoElems = LoVT.getVectorMinNumElements();
1467
1468 // If we know the index is in the first half, and we know the subvector
1469 // doesn't cross the boundary between the halves, we can avoid spilling the
1470 // vector, and insert into the lower half of the split vector directly.
1471 unsigned IdxVal = Idx->getAsZExtVal();
1472 if (IdxVal + SubElems <= LoElems) {
1473 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1474 return;
1475 }
1476 // Similarly if the subvector is fully in the high half, but mind that we
1477 // can't tell whether a fixed-length subvector is fully within the high half
1478 // of a scalable vector.
1479 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1480 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1481 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1482 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1483 return;
1484 }
1485
1486 // Spill the vector to the stack.
1487 // In cases where the vector is illegal it will be broken down into parts
1488 // and stored in parts - we should use the alignment for the smallest part.
1489 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1490 SDValue StackPtr =
1491 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1492 auto &MF = DAG.getMachineFunction();
1493 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1494 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1495
1496 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1497 SmallestAlign);
1498
1499 // Store the new subvector into the specified index.
1500 SDValue SubVecPtr =
1501 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1502 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1503 MachinePointerInfo::getUnknownStack(MF));
1504
1505 // Load the Lo part from the stack slot.
1506 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1507 SmallestAlign);
1508
1509 // Increment the pointer to the other part.
1510 auto *Load = cast<LoadSDNode>(Lo);
1511 MachinePointerInfo MPI = Load->getPointerInfo();
1512 IncrementPointer(Load, LoVT, MPI, StackPtr);
1513
1514 // Load the Hi part from the stack slot.
1515 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1516 }
1517
1518 // Handle splitting an FP where the second operand does not match the first
1519 // type. The second operand may be a scalar, or a vector that has exactly as
1520 // many elements as the first
SplitVecRes_FPOp_MultiType(SDNode * N,SDValue & Lo,SDValue & Hi)1521 void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
1522 SDValue &Hi) {
1523 SDValue LHSLo, LHSHi;
1524 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1525 SDLoc DL(N);
1526
1527 SDValue RHSLo, RHSHi;
1528 SDValue RHS = N->getOperand(1);
1529 EVT RHSVT = RHS.getValueType();
1530 if (RHSVT.isVector()) {
1531 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1532 GetSplitVector(RHS, RHSLo, RHSHi);
1533 else
1534 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1535
1536 Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo, RHSLo);
1537 Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi, RHSHi);
1538 } else {
1539 Lo = DAG.getNode(N->getOpcode(), DL, LHSLo.getValueType(), LHSLo, RHS);
1540 Hi = DAG.getNode(N->getOpcode(), DL, LHSHi.getValueType(), LHSHi, RHS);
1541 }
1542 }
1543
SplitVecRes_IS_FPCLASS(SDNode * N,SDValue & Lo,SDValue & Hi)1544 void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
1545 SDValue &Hi) {
1546 SDLoc DL(N);
1547 SDValue ArgLo, ArgHi;
1548 SDValue Test = N->getOperand(1);
1549 SDValue FpValue = N->getOperand(0);
1550 if (getTypeAction(FpValue.getValueType()) == TargetLowering::TypeSplitVector)
1551 GetSplitVector(FpValue, ArgLo, ArgHi);
1552 else
1553 std::tie(ArgLo, ArgHi) = DAG.SplitVector(FpValue, SDLoc(FpValue));
1554 EVT LoVT, HiVT;
1555 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1556
1557 Lo = DAG.getNode(ISD::IS_FPCLASS, DL, LoVT, ArgLo, Test, N->getFlags());
1558 Hi = DAG.getNode(ISD::IS_FPCLASS, DL, HiVT, ArgHi, Test, N->getFlags());
1559 }
1560
SplitVecRes_InregOp(SDNode * N,SDValue & Lo,SDValue & Hi)1561 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1562 SDValue &Hi) {
1563 SDValue LHSLo, LHSHi;
1564 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1565 SDLoc dl(N);
1566
1567 EVT LoVT, HiVT;
1568 std::tie(LoVT, HiVT) =
1569 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1570
1571 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1572 DAG.getValueType(LoVT));
1573 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1574 DAG.getValueType(HiVT));
1575 }
1576
SplitVecRes_ExtVecInRegOp(SDNode * N,SDValue & Lo,SDValue & Hi)1577 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1578 SDValue &Hi) {
1579 unsigned Opcode = N->getOpcode();
1580 SDValue N0 = N->getOperand(0);
1581
1582 SDLoc dl(N);
1583 SDValue InLo, InHi;
1584
1585 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1586 GetSplitVector(N0, InLo, InHi);
1587 else
1588 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1589
1590 EVT InLoVT = InLo.getValueType();
1591 unsigned InNumElements = InLoVT.getVectorNumElements();
1592
1593 EVT OutLoVT, OutHiVT;
1594 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1595 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1596 assert((2 * OutNumElements) <= InNumElements &&
1597 "Illegal extend vector in reg split");
1598
1599 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1600 // input vector (i.e. we only use InLo):
1601 // OutLo will extend the first OutNumElements from InLo.
1602 // OutHi will extend the next OutNumElements from InLo.
1603
1604 // Shuffle the elements from InLo for OutHi into the bottom elements to
1605 // create a 'fake' InHi.
1606 SmallVector<int, 8> SplitHi(InNumElements, -1);
1607 for (unsigned i = 0; i != OutNumElements; ++i)
1608 SplitHi[i] = i + OutNumElements;
1609 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1610
1611 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1612 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1613 }
1614
SplitVecRes_StrictFPOp(SDNode * N,SDValue & Lo,SDValue & Hi)1615 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1616 SDValue &Hi) {
1617 unsigned NumOps = N->getNumOperands();
1618 SDValue Chain = N->getOperand(0);
1619 EVT LoVT, HiVT;
1620 SDLoc dl(N);
1621 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1622
1623 SmallVector<SDValue, 4> OpsLo(NumOps);
1624 SmallVector<SDValue, 4> OpsHi(NumOps);
1625
1626 // The Chain is the first operand.
1627 OpsLo[0] = Chain;
1628 OpsHi[0] = Chain;
1629
1630 // Now process the remaining operands.
1631 for (unsigned i = 1; i < NumOps; ++i) {
1632 SDValue Op = N->getOperand(i);
1633 SDValue OpLo = Op;
1634 SDValue OpHi = Op;
1635
1636 EVT InVT = Op.getValueType();
1637 if (InVT.isVector()) {
1638 // If the input also splits, handle it directly for a
1639 // compile time speedup. Otherwise split it by hand.
1640 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1641 GetSplitVector(Op, OpLo, OpHi);
1642 else
1643 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1644 }
1645
1646 OpsLo[i] = OpLo;
1647 OpsHi[i] = OpHi;
1648 }
1649
1650 EVT LoValueVTs[] = {LoVT, MVT::Other};
1651 EVT HiValueVTs[] = {HiVT, MVT::Other};
1652 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1653 N->getFlags());
1654 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1655 N->getFlags());
1656
1657 // Build a factor node to remember that this Op is independent of the
1658 // other one.
1659 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1660 Lo.getValue(1), Hi.getValue(1));
1661
1662 // Legalize the chain result - switch anything that used the old chain to
1663 // use the new one.
1664 ReplaceValueWith(SDValue(N, 1), Chain);
1665 }
1666
UnrollVectorOp_StrictFP(SDNode * N,unsigned ResNE)1667 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1668 SDValue Chain = N->getOperand(0);
1669 EVT VT = N->getValueType(0);
1670 unsigned NE = VT.getVectorNumElements();
1671 EVT EltVT = VT.getVectorElementType();
1672 SDLoc dl(N);
1673
1674 SmallVector<SDValue, 8> Scalars;
1675 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1676
1677 // If ResNE is 0, fully unroll the vector op.
1678 if (ResNE == 0)
1679 ResNE = NE;
1680 else if (NE > ResNE)
1681 NE = ResNE;
1682
1683 //The results of each unrolled operation, including the chain.
1684 EVT ChainVTs[] = {EltVT, MVT::Other};
1685 SmallVector<SDValue, 8> Chains;
1686
1687 unsigned i;
1688 for (i = 0; i != NE; ++i) {
1689 Operands[0] = Chain;
1690 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1691 SDValue Operand = N->getOperand(j);
1692 EVT OperandVT = Operand.getValueType();
1693 if (OperandVT.isVector()) {
1694 EVT OperandEltVT = OperandVT.getVectorElementType();
1695 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1696 Operand, DAG.getVectorIdxConstant(i, dl));
1697 } else {
1698 Operands[j] = Operand;
1699 }
1700 }
1701 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1702 Scalar.getNode()->setFlags(N->getFlags());
1703
1704 //Add in the scalar as well as its chain value to the
1705 //result vectors.
1706 Scalars.push_back(Scalar);
1707 Chains.push_back(Scalar.getValue(1));
1708 }
1709
1710 for (; i < ResNE; ++i)
1711 Scalars.push_back(DAG.getUNDEF(EltVT));
1712
1713 // Build a new factor node to connect the chain back together.
1714 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1715 ReplaceValueWith(SDValue(N, 1), Chain);
1716
1717 // Create a new BUILD_VECTOR node
1718 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1719 return DAG.getBuildVector(VecVT, dl, Scalars);
1720 }
1721
SplitVecRes_OverflowOp(SDNode * N,unsigned ResNo,SDValue & Lo,SDValue & Hi)1722 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1723 SDValue &Lo, SDValue &Hi) {
1724 SDLoc dl(N);
1725 EVT ResVT = N->getValueType(0);
1726 EVT OvVT = N->getValueType(1);
1727 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1728 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1729 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1730
1731 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1732 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1733 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1734 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1735 } else {
1736 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1737 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1738 }
1739
1740 unsigned Opcode = N->getOpcode();
1741 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1742 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1743 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1744 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1745 LoNode->setFlags(N->getFlags());
1746 HiNode->setFlags(N->getFlags());
1747
1748 Lo = SDValue(LoNode, ResNo);
1749 Hi = SDValue(HiNode, ResNo);
1750
1751 // Replace the other vector result not being explicitly split here.
1752 unsigned OtherNo = 1 - ResNo;
1753 EVT OtherVT = N->getValueType(OtherNo);
1754 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1755 SetSplitVector(SDValue(N, OtherNo),
1756 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1757 } else {
1758 SDValue OtherVal = DAG.getNode(
1759 ISD::CONCAT_VECTORS, dl, OtherVT,
1760 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1761 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1762 }
1763 }
1764
SplitVecRes_INSERT_VECTOR_ELT(SDNode * N,SDValue & Lo,SDValue & Hi)1765 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1766 SDValue &Hi) {
1767 SDValue Vec = N->getOperand(0);
1768 SDValue Elt = N->getOperand(1);
1769 SDValue Idx = N->getOperand(2);
1770 SDLoc dl(N);
1771 GetSplitVector(Vec, Lo, Hi);
1772
1773 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1774 unsigned IdxVal = CIdx->getZExtValue();
1775 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1776 if (IdxVal < LoNumElts) {
1777 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1778 Lo.getValueType(), Lo, Elt, Idx);
1779 return;
1780 } else if (!Vec.getValueType().isScalableVector()) {
1781 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1782 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1783 return;
1784 }
1785 }
1786
1787 // See if the target wants to custom expand this node.
1788 if (CustomLowerNode(N, N->getValueType(0), true))
1789 return;
1790
1791 // Make the vector elements byte-addressable if they aren't already.
1792 EVT VecVT = Vec.getValueType();
1793 EVT EltVT = VecVT.getVectorElementType();
1794 if (VecVT.getScalarSizeInBits() < 8) {
1795 EltVT = MVT::i8;
1796 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1797 VecVT.getVectorElementCount());
1798 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1799 // Extend the element type to match if needed.
1800 if (EltVT.bitsGT(Elt.getValueType()))
1801 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1802 }
1803
1804 // Spill the vector to the stack.
1805 // In cases where the vector is illegal it will be broken down into parts
1806 // and stored in parts - we should use the alignment for the smallest part.
1807 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1808 SDValue StackPtr =
1809 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1810 auto &MF = DAG.getMachineFunction();
1811 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1812 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1813
1814 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1815 SmallestAlign);
1816
1817 // Store the new element. This may be larger than the vector element type,
1818 // so use a truncating store.
1819 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1820 Store = DAG.getTruncStore(
1821 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1822 commonAlignment(SmallestAlign,
1823 EltVT.getFixedSizeInBits() / 8));
1824
1825 EVT LoVT, HiVT;
1826 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1827
1828 // Load the Lo part from the stack slot.
1829 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1830
1831 // Increment the pointer to the other part.
1832 auto Load = cast<LoadSDNode>(Lo);
1833 MachinePointerInfo MPI = Load->getPointerInfo();
1834 IncrementPointer(Load, LoVT, MPI, StackPtr);
1835
1836 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1837
1838 // If we adjusted the original type, we need to truncate the results.
1839 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1840 if (LoVT != Lo.getValueType())
1841 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1842 if (HiVT != Hi.getValueType())
1843 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1844 }
1845
SplitVecRes_STEP_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1846 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1847 SDValue &Hi) {
1848 EVT LoVT, HiVT;
1849 SDLoc dl(N);
1850 assert(N->getValueType(0).isScalableVector() &&
1851 "Only scalable vectors are supported for STEP_VECTOR");
1852 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1853 SDValue Step = N->getOperand(0);
1854
1855 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1856
1857 // Hi = Lo + (EltCnt * Step)
1858 EVT EltVT = Step.getValueType();
1859 APInt StepVal = Step->getAsAPIntVal();
1860 SDValue StartOfHi =
1861 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1862 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1863 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1864
1865 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1866 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1867 }
1868
SplitVecRes_ScalarOp(SDNode * N,SDValue & Lo,SDValue & Hi)1869 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1870 SDValue &Hi) {
1871 EVT LoVT, HiVT;
1872 SDLoc dl(N);
1873 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1874 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1875 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1876 Hi = DAG.getUNDEF(HiVT);
1877 } else {
1878 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1879 Hi = Lo;
1880 }
1881 }
1882
SplitVecRes_LOAD(LoadSDNode * LD,SDValue & Lo,SDValue & Hi)1883 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1884 SDValue &Hi) {
1885 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1886 EVT LoVT, HiVT;
1887 SDLoc dl(LD);
1888 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1889
1890 ISD::LoadExtType ExtType = LD->getExtensionType();
1891 SDValue Ch = LD->getChain();
1892 SDValue Ptr = LD->getBasePtr();
1893 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1894 EVT MemoryVT = LD->getMemoryVT();
1895 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1896 AAMDNodes AAInfo = LD->getAAInfo();
1897
1898 EVT LoMemVT, HiMemVT;
1899 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1900
1901 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1902 SDValue Value, NewChain;
1903 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1904 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1905 ReplaceValueWith(SDValue(LD, 1), NewChain);
1906 return;
1907 }
1908
1909 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1910 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1911 MMOFlags, AAInfo);
1912
1913 MachinePointerInfo MPI;
1914 IncrementPointer(LD, LoMemVT, MPI, Ptr);
1915
1916 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1917 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1918
1919 // Build a factor node to remember that this load is independent of the
1920 // other one.
1921 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1922 Hi.getValue(1));
1923
1924 // Legalize the chain result - switch anything that used the old chain to
1925 // use the new one.
1926 ReplaceValueWith(SDValue(LD, 1), Ch);
1927 }
1928
SplitVecRes_VP_LOAD(VPLoadSDNode * LD,SDValue & Lo,SDValue & Hi)1929 void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
1930 SDValue &Hi) {
1931 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
1932 EVT LoVT, HiVT;
1933 SDLoc dl(LD);
1934 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1935
1936 ISD::LoadExtType ExtType = LD->getExtensionType();
1937 SDValue Ch = LD->getChain();
1938 SDValue Ptr = LD->getBasePtr();
1939 SDValue Offset = LD->getOffset();
1940 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
1941 Align Alignment = LD->getOriginalAlign();
1942 SDValue Mask = LD->getMask();
1943 SDValue EVL = LD->getVectorLength();
1944 EVT MemoryVT = LD->getMemoryVT();
1945
1946 EVT LoMemVT, HiMemVT;
1947 bool HiIsEmpty = false;
1948 std::tie(LoMemVT, HiMemVT) =
1949 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1950
1951 // Split Mask operand
1952 SDValue MaskLo, MaskHi;
1953 if (Mask.getOpcode() == ISD::SETCC) {
1954 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1955 } else {
1956 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1957 GetSplitVector(Mask, MaskLo, MaskHi);
1958 else
1959 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1960 }
1961
1962 // Split EVL operand
1963 SDValue EVLLo, EVLHi;
1964 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, LD->getValueType(0), dl);
1965
1966 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1967 LD->getPointerInfo(), MachineMemOperand::MOLoad,
1968 MemoryLocation::UnknownSize, Alignment, LD->getAAInfo(), LD->getRanges());
1969
1970 Lo =
1971 DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
1972 MaskLo, EVLLo, LoMemVT, MMO, LD->isExpandingLoad());
1973
1974 if (HiIsEmpty) {
1975 // The hi vp_load has zero storage size. We therefore simply set it to
1976 // the low vp_load and rely on subsequent removal from the chain.
1977 Hi = Lo;
1978 } else {
1979 // Generate hi vp_load.
1980 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1981 LD->isExpandingLoad());
1982
1983 MachinePointerInfo MPI;
1984 if (LoMemVT.isScalableVector())
1985 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
1986 else
1987 MPI = LD->getPointerInfo().getWithOffset(
1988 LoMemVT.getStoreSize().getFixedValue());
1989
1990 MMO = DAG.getMachineFunction().getMachineMemOperand(
1991 MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
1992 LD->getAAInfo(), LD->getRanges());
1993
1994 Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
1995 Offset, MaskHi, EVLHi, HiMemVT, MMO,
1996 LD->isExpandingLoad());
1997 }
1998
1999 // Build a factor node to remember that this load is independent of the
2000 // other one.
2001 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2002 Hi.getValue(1));
2003
2004 // Legalize the chain result - switch anything that used the old chain to
2005 // use the new one.
2006 ReplaceValueWith(SDValue(LD, 1), Ch);
2007 }
2008
SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode * SLD,SDValue & Lo,SDValue & Hi)2009 void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2010 SDValue &Lo, SDValue &Hi) {
2011 assert(SLD->isUnindexed() &&
2012 "Indexed VP strided load during type legalization!");
2013 assert(SLD->getOffset().isUndef() &&
2014 "Unexpected indexed variable-length load offset");
2015
2016 SDLoc DL(SLD);
2017
2018 EVT LoVT, HiVT;
2019 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(SLD->getValueType(0));
2020
2021 EVT LoMemVT, HiMemVT;
2022 bool HiIsEmpty = false;
2023 std::tie(LoMemVT, HiMemVT) =
2024 DAG.GetDependentSplitDestVTs(SLD->getMemoryVT(), LoVT, &HiIsEmpty);
2025
2026 SDValue Mask = SLD->getMask();
2027 SDValue LoMask, HiMask;
2028 if (Mask.getOpcode() == ISD::SETCC) {
2029 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
2030 } else {
2031 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2032 GetSplitVector(Mask, LoMask, HiMask);
2033 else
2034 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2035 }
2036
2037 SDValue LoEVL, HiEVL;
2038 std::tie(LoEVL, HiEVL) =
2039 DAG.SplitEVL(SLD->getVectorLength(), SLD->getValueType(0), DL);
2040
2041 // Generate the low vp_strided_load
2042 Lo = DAG.getStridedLoadVP(
2043 SLD->getAddressingMode(), SLD->getExtensionType(), LoVT, DL,
2044 SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), SLD->getStride(),
2045 LoMask, LoEVL, LoMemVT, SLD->getMemOperand(), SLD->isExpandingLoad());
2046
2047 if (HiIsEmpty) {
2048 // The high vp_strided_load has zero storage size. We therefore simply set
2049 // it to the low vp_strided_load and rely on subsequent removal from the
2050 // chain.
2051 Hi = Lo;
2052 } else {
2053 // Generate the high vp_strided_load.
2054 // To calculate the high base address, we need to sum to the low base
2055 // address stride number of bytes for each element already loaded by low,
2056 // that is: Ptr = Ptr + (LoEVL * Stride)
2057 EVT PtrVT = SLD->getBasePtr().getValueType();
2058 SDValue Increment =
2059 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
2060 DAG.getSExtOrTrunc(SLD->getStride(), DL, PtrVT));
2061 SDValue Ptr =
2062 DAG.getNode(ISD::ADD, DL, PtrVT, SLD->getBasePtr(), Increment);
2063
2064 Align Alignment = SLD->getOriginalAlign();
2065 if (LoMemVT.isScalableVector())
2066 Alignment = commonAlignment(
2067 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2068
2069 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2070 MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2071 MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
2072 SLD->getAAInfo(), SLD->getRanges());
2073
2074 Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
2075 HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
2076 SLD->getStride(), HiMask, HiEVL, HiMemVT, MMO,
2077 SLD->isExpandingLoad());
2078 }
2079
2080 // Build a factor node to remember that this load is independent of the
2081 // other one.
2082 SDValue Ch = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
2083 Hi.getValue(1));
2084
2085 // Legalize the chain result - switch anything that used the old chain to
2086 // use the new one.
2087 ReplaceValueWith(SDValue(SLD, 1), Ch);
2088 }
2089
SplitVecRes_MLOAD(MaskedLoadSDNode * MLD,SDValue & Lo,SDValue & Hi)2090 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2091 SDValue &Lo, SDValue &Hi) {
2092 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2093 EVT LoVT, HiVT;
2094 SDLoc dl(MLD);
2095 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
2096
2097 SDValue Ch = MLD->getChain();
2098 SDValue Ptr = MLD->getBasePtr();
2099 SDValue Offset = MLD->getOffset();
2100 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2101 SDValue Mask = MLD->getMask();
2102 SDValue PassThru = MLD->getPassThru();
2103 Align Alignment = MLD->getOriginalAlign();
2104 ISD::LoadExtType ExtType = MLD->getExtensionType();
2105
2106 // Split Mask operand
2107 SDValue MaskLo, MaskHi;
2108 if (Mask.getOpcode() == ISD::SETCC) {
2109 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2110 } else {
2111 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2112 GetSplitVector(Mask, MaskLo, MaskHi);
2113 else
2114 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2115 }
2116
2117 EVT MemoryVT = MLD->getMemoryVT();
2118 EVT LoMemVT, HiMemVT;
2119 bool HiIsEmpty = false;
2120 std::tie(LoMemVT, HiMemVT) =
2121 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
2122
2123 SDValue PassThruLo, PassThruHi;
2124 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2125 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2126 else
2127 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2128
2129 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2130 MLD->getPointerInfo(), MachineMemOperand::MOLoad,
2131 MemoryLocation::UnknownSize, Alignment, MLD->getAAInfo(),
2132 MLD->getRanges());
2133
2134 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
2135 MMO, MLD->getAddressingMode(), ExtType,
2136 MLD->isExpandingLoad());
2137
2138 if (HiIsEmpty) {
2139 // The hi masked load has zero storage size. We therefore simply set it to
2140 // the low masked load and rely on subsequent removal from the chain.
2141 Hi = Lo;
2142 } else {
2143 // Generate hi masked load.
2144 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
2145 MLD->isExpandingLoad());
2146
2147 MachinePointerInfo MPI;
2148 if (LoMemVT.isScalableVector())
2149 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2150 else
2151 MPI = MLD->getPointerInfo().getWithOffset(
2152 LoMemVT.getStoreSize().getFixedValue());
2153
2154 MMO = DAG.getMachineFunction().getMachineMemOperand(
2155 MPI, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize, Alignment,
2156 MLD->getAAInfo(), MLD->getRanges());
2157
2158 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
2159 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
2160 MLD->isExpandingLoad());
2161 }
2162
2163 // Build a factor node to remember that this load is independent of the
2164 // other one.
2165 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2166 Hi.getValue(1));
2167
2168 // Legalize the chain result - switch anything that used the old chain to
2169 // use the new one.
2170 ReplaceValueWith(SDValue(MLD, 1), Ch);
2171
2172 }
2173
SplitVecRes_Gather(MemSDNode * N,SDValue & Lo,SDValue & Hi,bool SplitSETCC)2174 void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2175 SDValue &Hi, bool SplitSETCC) {
2176 EVT LoVT, HiVT;
2177 SDLoc dl(N);
2178 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2179
2180 SDValue Ch = N->getChain();
2181 SDValue Ptr = N->getBasePtr();
2182 struct Operands {
2183 SDValue Mask;
2184 SDValue Index;
2185 SDValue Scale;
2186 } Ops = [&]() -> Operands {
2187 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(N)) {
2188 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2189 }
2190 auto *VPSC = cast<VPGatherSDNode>(N);
2191 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2192 }();
2193
2194 EVT MemoryVT = N->getMemoryVT();
2195 Align Alignment = N->getOriginalAlign();
2196
2197 // Split Mask operand
2198 SDValue MaskLo, MaskHi;
2199 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2200 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2201 } else {
2202 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2203 }
2204
2205 EVT LoMemVT, HiMemVT;
2206 // Split MemoryVT
2207 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2208
2209 SDValue IndexHi, IndexLo;
2210 if (getTypeAction(Ops.Index.getValueType()) ==
2211 TargetLowering::TypeSplitVector)
2212 GetSplitVector(Ops.Index, IndexLo, IndexHi);
2213 else
2214 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
2215
2216 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2217 N->getPointerInfo(), MachineMemOperand::MOLoad,
2218 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2219
2220 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
2221 SDValue PassThru = MGT->getPassThru();
2222 SDValue PassThruLo, PassThruHi;
2223 if (getTypeAction(PassThru.getValueType()) ==
2224 TargetLowering::TypeSplitVector)
2225 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2226 else
2227 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2228
2229 ISD::LoadExtType ExtType = MGT->getExtensionType();
2230 ISD::MemIndexType IndexTy = MGT->getIndexType();
2231
2232 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2233 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2234 OpsLo, MMO, IndexTy, ExtType);
2235
2236 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2237 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2238 OpsHi, MMO, IndexTy, ExtType);
2239 } else {
2240 auto *VPGT = cast<VPGatherSDNode>(N);
2241 SDValue EVLLo, EVLHi;
2242 std::tie(EVLLo, EVLHi) =
2243 DAG.SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2244
2245 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2246 Lo = DAG.getGatherVP(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
2247 MMO, VPGT->getIndexType());
2248
2249 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2250 Hi = DAG.getGatherVP(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
2251 MMO, VPGT->getIndexType());
2252 }
2253
2254 // Build a factor node to remember that this load is independent of the
2255 // other one.
2256 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2257 Hi.getValue(1));
2258
2259 // Legalize the chain result - switch anything that used the old chain to
2260 // use the new one.
2261 ReplaceValueWith(SDValue(N, 1), Ch);
2262 }
2263
SplitVecRes_SETCC(SDNode * N,SDValue & Lo,SDValue & Hi)2264 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2265 assert(N->getValueType(0).isVector() &&
2266 N->getOperand(0).getValueType().isVector() &&
2267 "Operand types must be vectors");
2268
2269 EVT LoVT, HiVT;
2270 SDLoc DL(N);
2271 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2272
2273 // If the input also splits, handle it directly. Otherwise split it by hand.
2274 SDValue LL, LH, RL, RH;
2275 if (getTypeAction(N->getOperand(0).getValueType()) ==
2276 TargetLowering::TypeSplitVector)
2277 GetSplitVector(N->getOperand(0), LL, LH);
2278 else
2279 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
2280
2281 if (getTypeAction(N->getOperand(1).getValueType()) ==
2282 TargetLowering::TypeSplitVector)
2283 GetSplitVector(N->getOperand(1), RL, RH);
2284 else
2285 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
2286
2287 if (N->getOpcode() == ISD::SETCC) {
2288 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
2289 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
2290 } else {
2291 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2292 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2293 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
2294 std::tie(EVLLo, EVLHi) =
2295 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
2296 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2), MaskLo,
2297 EVLLo);
2298 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2), MaskHi,
2299 EVLHi);
2300 }
2301 }
2302
SplitVecRes_UnaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)2303 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2304 SDValue &Hi) {
2305 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2306 EVT LoVT, HiVT;
2307 SDLoc dl(N);
2308 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
2309
2310 // If the input also splits, handle it directly for a compile time speedup.
2311 // Otherwise split it by hand.
2312 EVT InVT = N->getOperand(0).getValueType();
2313 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2314 GetSplitVector(N->getOperand(0), Lo, Hi);
2315 else
2316 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2317
2318 const SDNodeFlags Flags = N->getFlags();
2319 unsigned Opcode = N->getOpcode();
2320 if (N->getNumOperands() <= 2) {
2321 if (Opcode == ISD::FP_ROUND) {
2322 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, N->getOperand(1), Flags);
2323 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, N->getOperand(1), Flags);
2324 } else {
2325 Lo = DAG.getNode(Opcode, dl, LoVT, Lo, Flags);
2326 Hi = DAG.getNode(Opcode, dl, HiVT, Hi, Flags);
2327 }
2328 return;
2329 }
2330
2331 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
2332 assert(N->isVPOpcode() && "Expected VP opcode");
2333
2334 SDValue MaskLo, MaskHi;
2335 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2336
2337 SDValue EVLLo, EVLHi;
2338 std::tie(EVLLo, EVLHi) =
2339 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2340
2341 Lo = DAG.getNode(Opcode, dl, LoVT, {Lo, MaskLo, EVLLo}, Flags);
2342 Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
2343 }
2344
SplitVecRes_FFREXP(SDNode * N,unsigned ResNo,SDValue & Lo,SDValue & Hi)2345 void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo,
2346 SDValue &Lo, SDValue &Hi) {
2347 SDLoc dl(N);
2348 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0));
2349 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(N->getValueType(1));
2350
2351 // If the input also splits, handle it directly for a compile time speedup.
2352 // Otherwise split it by hand.
2353 EVT InVT = N->getOperand(0).getValueType();
2354 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
2355 GetSplitVector(N->getOperand(0), Lo, Hi);
2356 else
2357 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
2358
2359 Lo = DAG.getNode(N->getOpcode(), dl, {LoVT, LoVT1}, Lo);
2360 Hi = DAG.getNode(N->getOpcode(), dl, {HiVT, HiVT1}, Hi);
2361 Lo->setFlags(N->getFlags());
2362 Hi->setFlags(N->getFlags());
2363
2364 SDNode *HiNode = Hi.getNode();
2365 SDNode *LoNode = Lo.getNode();
2366
2367 // Replace the other vector result not being explicitly split here.
2368 unsigned OtherNo = 1 - ResNo;
2369 EVT OtherVT = N->getValueType(OtherNo);
2370 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
2371 SetSplitVector(SDValue(N, OtherNo), SDValue(LoNode, OtherNo),
2372 SDValue(HiNode, OtherNo));
2373 } else {
2374 SDValue OtherVal =
2375 DAG.getNode(ISD::CONCAT_VECTORS, dl, OtherVT, SDValue(LoNode, OtherNo),
2376 SDValue(HiNode, OtherNo));
2377 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
2378 }
2379 }
2380
SplitVecRes_ExtendOp(SDNode * N,SDValue & Lo,SDValue & Hi)2381 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
2382 SDValue &Hi) {
2383 SDLoc dl(N);
2384 EVT SrcVT = N->getOperand(0).getValueType();
2385 EVT DestVT = N->getValueType(0);
2386 EVT LoVT, HiVT;
2387 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
2388
2389 // We can do better than a generic split operation if the extend is doing
2390 // more than just doubling the width of the elements and the following are
2391 // true:
2392 // - The number of vector elements is even,
2393 // - the source type is legal,
2394 // - the type of a split source is illegal,
2395 // - the type of an extended (by doubling element size) source is legal, and
2396 // - the type of that extended source when split is legal.
2397 //
2398 // This won't necessarily completely legalize the operation, but it will
2399 // more effectively move in the right direction and prevent falling down
2400 // to scalarization in many cases due to the input vector being split too
2401 // far.
2402 if (SrcVT.getVectorElementCount().isKnownEven() &&
2403 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
2404 LLVMContext &Ctx = *DAG.getContext();
2405 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
2406 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
2407
2408 EVT SplitLoVT, SplitHiVT;
2409 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
2410 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
2411 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
2412 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
2413 N->dump(&DAG); dbgs() << "\n");
2414 if (!N->isVPOpcode()) {
2415 // Extend the source vector by one step.
2416 SDValue NewSrc =
2417 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
2418 // Get the low and high halves of the new, extended one step, vector.
2419 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2420 // Extend those vector halves the rest of the way.
2421 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
2422 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
2423 return;
2424 }
2425
2426 // Extend the source vector by one step.
2427 SDValue NewSrc =
2428 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0),
2429 N->getOperand(1), N->getOperand(2));
2430 // Get the low and high halves of the new, extended one step, vector.
2431 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
2432
2433 SDValue MaskLo, MaskHi;
2434 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
2435
2436 SDValue EVLLo, EVLHi;
2437 std::tie(EVLLo, EVLHi) =
2438 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2439 // Extend those vector halves the rest of the way.
2440 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2441 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2442 return;
2443 }
2444 }
2445 // Fall back to the generic unary operator splitting otherwise.
2446 SplitVecRes_UnaryOp(N, Lo, Hi);
2447 }
2448
SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N,SDValue & Lo,SDValue & Hi)2449 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
2450 SDValue &Lo, SDValue &Hi) {
2451 // The low and high parts of the original input give four input vectors.
2452 SDValue Inputs[4];
2453 SDLoc DL(N);
2454 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
2455 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
2456 EVT NewVT = Inputs[0].getValueType();
2457 unsigned NewElts = NewVT.getVectorNumElements();
2458
2459 auto &&IsConstant = [](const SDValue &N) {
2460 APInt SplatValue;
2461 return N.getResNo() == 0 &&
2462 (ISD::isConstantSplatVector(N.getNode(), SplatValue) ||
2463 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
2464 };
2465 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
2466 SDValue &Input2,
2467 ArrayRef<int> Mask) {
2468 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
2469 Input2->getOpcode() == ISD::BUILD_VECTOR &&
2470 "Expected build vector node.");
2471 EVT EltVT = NewVT.getVectorElementType();
2472 SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
2473 for (unsigned I = 0; I < NewElts; ++I) {
2474 if (Mask[I] == PoisonMaskElem)
2475 continue;
2476 unsigned Idx = Mask[I];
2477 if (Idx >= NewElts)
2478 Ops[I] = Input2.getOperand(Idx - NewElts);
2479 else
2480 Ops[I] = Input1.getOperand(Idx);
2481 // Make the type of all elements the same as the element type.
2482 if (Ops[I].getValueType().bitsGT(EltVT))
2483 Ops[I] = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Ops[I]);
2484 }
2485 return DAG.getBuildVector(NewVT, DL, Ops);
2486 };
2487
2488 // If Lo or Hi uses elements from at most two of the four input vectors, then
2489 // express it as a vector shuffle of those two inputs. Otherwise extract the
2490 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
2491 SmallVector<int> OrigMask(N->getMask());
2492 // Try to pack incoming shuffles/inputs.
2493 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
2494 &DL](SmallVectorImpl<int> &Mask) {
2495 // Check if all inputs are shuffles of the same operands or non-shuffles.
2496 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
2497 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
2498 SDValue Input = Inputs[Idx];
2499 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Input.getNode());
2500 if (!Shuffle ||
2501 Input.getOperand(0).getValueType() != Input.getValueType())
2502 continue;
2503 ShufflesIdxs[std::make_pair(Input.getOperand(0), Input.getOperand(1))]
2504 .push_back(Idx);
2505 ShufflesIdxs[std::make_pair(Input.getOperand(1), Input.getOperand(0))]
2506 .push_back(Idx);
2507 }
2508 for (auto &P : ShufflesIdxs) {
2509 if (P.second.size() < 2)
2510 continue;
2511 // Use shuffles operands instead of shuffles themselves.
2512 // 1. Adjust mask.
2513 for (int &Idx : Mask) {
2514 if (Idx == PoisonMaskElem)
2515 continue;
2516 unsigned SrcRegIdx = Idx / NewElts;
2517 if (Inputs[SrcRegIdx].isUndef()) {
2518 Idx = PoisonMaskElem;
2519 continue;
2520 }
2521 auto *Shuffle =
2522 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2523 if (!Shuffle || !is_contained(P.second, SrcRegIdx))
2524 continue;
2525 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2526 if (MaskElt == PoisonMaskElem) {
2527 Idx = PoisonMaskElem;
2528 continue;
2529 }
2530 Idx = MaskElt % NewElts +
2531 P.second[Shuffle->getOperand(MaskElt / NewElts) == P.first.first
2532 ? 0
2533 : 1] *
2534 NewElts;
2535 }
2536 // 2. Update inputs.
2537 Inputs[P.second[0]] = P.first.first;
2538 Inputs[P.second[1]] = P.first.second;
2539 // Clear the pair data.
2540 P.second.clear();
2541 ShufflesIdxs[std::make_pair(P.first.second, P.first.first)].clear();
2542 }
2543 // Check if any concat_vectors can be simplified.
2544 SmallBitVector UsedSubVector(2 * std::size(Inputs));
2545 for (int &Idx : Mask) {
2546 if (Idx == PoisonMaskElem)
2547 continue;
2548 unsigned SrcRegIdx = Idx / NewElts;
2549 if (Inputs[SrcRegIdx].isUndef()) {
2550 Idx = PoisonMaskElem;
2551 continue;
2552 }
2553 TargetLowering::LegalizeTypeAction TypeAction =
2554 getTypeAction(Inputs[SrcRegIdx].getValueType());
2555 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
2556 Inputs[SrcRegIdx].getNumOperands() == 2 &&
2557 !Inputs[SrcRegIdx].getOperand(1).isUndef() &&
2558 (TypeAction == TargetLowering::TypeLegal ||
2559 TypeAction == TargetLowering::TypeWidenVector))
2560 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
2561 }
2562 if (UsedSubVector.count() > 1) {
2563 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
2564 for (unsigned I = 0; I < std::size(Inputs); ++I) {
2565 if (UsedSubVector.test(2 * I) == UsedSubVector.test(2 * I + 1))
2566 continue;
2567 if (Pairs.empty() || Pairs.back().size() == 2)
2568 Pairs.emplace_back();
2569 if (UsedSubVector.test(2 * I)) {
2570 Pairs.back().emplace_back(I, 0);
2571 } else {
2572 assert(UsedSubVector.test(2 * I + 1) &&
2573 "Expected to be used one of the subvectors.");
2574 Pairs.back().emplace_back(I, 1);
2575 }
2576 }
2577 if (!Pairs.empty() && Pairs.front().size() > 1) {
2578 // Adjust mask.
2579 for (int &Idx : Mask) {
2580 if (Idx == PoisonMaskElem)
2581 continue;
2582 unsigned SrcRegIdx = Idx / NewElts;
2583 auto *It = find_if(
2584 Pairs, [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
2585 return Idxs.front().first == SrcRegIdx ||
2586 Idxs.back().first == SrcRegIdx;
2587 });
2588 if (It == Pairs.end())
2589 continue;
2590 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
2591 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2592 }
2593 // Adjust inputs.
2594 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2595 Inputs[Idxs.front().first] = DAG.getNode(
2596 ISD::CONCAT_VECTORS, DL,
2597 Inputs[Idxs.front().first].getValueType(),
2598 Inputs[Idxs.front().first].getOperand(Idxs.front().second),
2599 Inputs[Idxs.back().first].getOperand(Idxs.back().second));
2600 }
2601 }
2602 }
2603 bool Changed;
2604 do {
2605 // Try to remove extra shuffles (except broadcasts) and shuffles with the
2606 // reused operands.
2607 Changed = false;
2608 for (unsigned I = 0; I < std::size(Inputs); ++I) {
2609 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[I].getNode());
2610 if (!Shuffle)
2611 continue;
2612 if (Shuffle->getOperand(0).getValueType() != NewVT)
2613 continue;
2614 int Op = -1;
2615 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(1).isUndef() &&
2616 !Shuffle->isSplat()) {
2617 Op = 0;
2618 } else if (!Inputs[I].hasOneUse() &&
2619 !Shuffle->getOperand(1).isUndef()) {
2620 // Find the only used operand, if possible.
2621 for (int &Idx : Mask) {
2622 if (Idx == PoisonMaskElem)
2623 continue;
2624 unsigned SrcRegIdx = Idx / NewElts;
2625 if (SrcRegIdx != I)
2626 continue;
2627 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2628 if (MaskElt == PoisonMaskElem) {
2629 Idx = PoisonMaskElem;
2630 continue;
2631 }
2632 int OpIdx = MaskElt / NewElts;
2633 if (Op == -1) {
2634 Op = OpIdx;
2635 continue;
2636 }
2637 if (Op != OpIdx) {
2638 Op = -1;
2639 break;
2640 }
2641 }
2642 }
2643 if (Op < 0) {
2644 // Try to check if one of the shuffle operands is used already.
2645 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2646 if (Shuffle->getOperand(OpIdx).isUndef())
2647 continue;
2648 auto *It = find(Inputs, Shuffle->getOperand(OpIdx));
2649 if (It == std::end(Inputs))
2650 continue;
2651 int FoundOp = std::distance(std::begin(Inputs), It);
2652 // Found that operand is used already.
2653 // 1. Fix the mask for the reused operand.
2654 for (int &Idx : Mask) {
2655 if (Idx == PoisonMaskElem)
2656 continue;
2657 unsigned SrcRegIdx = Idx / NewElts;
2658 if (SrcRegIdx != I)
2659 continue;
2660 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2661 if (MaskElt == PoisonMaskElem) {
2662 Idx = PoisonMaskElem;
2663 continue;
2664 }
2665 int MaskIdx = MaskElt / NewElts;
2666 if (OpIdx == MaskIdx)
2667 Idx = MaskElt % NewElts + FoundOp * NewElts;
2668 }
2669 // 2. Set Op to the unused OpIdx.
2670 Op = (OpIdx + 1) % 2;
2671 break;
2672 }
2673 }
2674 if (Op >= 0) {
2675 Changed = true;
2676 Inputs[I] = Shuffle->getOperand(Op);
2677 // Adjust mask.
2678 for (int &Idx : Mask) {
2679 if (Idx == PoisonMaskElem)
2680 continue;
2681 unsigned SrcRegIdx = Idx / NewElts;
2682 if (SrcRegIdx != I)
2683 continue;
2684 int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
2685 int OpIdx = MaskElt / NewElts;
2686 if (OpIdx != Op)
2687 continue;
2688 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2689 }
2690 }
2691 }
2692 } while (Changed);
2693 };
2694 TryPeekThroughShufflesInputs(OrigMask);
2695 // Proces unique inputs.
2696 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
2697 NewElts](SmallVectorImpl<int> &Mask) {
2698 SetVector<SDValue> UniqueInputs;
2699 SetVector<SDValue> UniqueConstantInputs;
2700 for (const auto &I : Inputs) {
2701 if (IsConstant(I))
2702 UniqueConstantInputs.insert(I);
2703 else if (!I.isUndef())
2704 UniqueInputs.insert(I);
2705 }
2706 // Adjust mask in case of reused inputs. Also, need to insert constant
2707 // inputs at first, otherwise it affects the final outcome.
2708 if (UniqueInputs.size() != std::size(Inputs)) {
2709 auto &&UniqueVec = UniqueInputs.takeVector();
2710 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
2711 unsigned ConstNum = UniqueConstantVec.size();
2712 for (int &Idx : Mask) {
2713 if (Idx == PoisonMaskElem)
2714 continue;
2715 unsigned SrcRegIdx = Idx / NewElts;
2716 if (Inputs[SrcRegIdx].isUndef()) {
2717 Idx = PoisonMaskElem;
2718 continue;
2719 }
2720 const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
2721 if (It != UniqueConstantVec.end()) {
2722 Idx = (Idx % NewElts) +
2723 NewElts * std::distance(UniqueConstantVec.begin(), It);
2724 assert(Idx >= 0 && "Expected defined mask idx.");
2725 continue;
2726 }
2727 const auto RegIt = find(UniqueVec, Inputs[SrcRegIdx]);
2728 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
2729 Idx = (Idx % NewElts) +
2730 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2731 assert(Idx >= 0 && "Expected defined mask idx.");
2732 }
2733 copy(UniqueConstantVec, std::begin(Inputs));
2734 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2735 }
2736 };
2737 MakeUniqueInputs(OrigMask);
2738 SDValue OrigInputs[4];
2739 copy(Inputs, std::begin(OrigInputs));
2740 for (unsigned High = 0; High < 2; ++High) {
2741 SDValue &Output = High ? Hi : Lo;
2742
2743 // Build a shuffle mask for the output, discovering on the fly which
2744 // input vectors to use as shuffle operands.
2745 unsigned FirstMaskIdx = High * NewElts;
2746 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
2747 copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
2748 assert(!Output && "Expected default initialized initial value.");
2749 TryPeekThroughShufflesInputs(Mask);
2750 MakeUniqueInputs(Mask);
2751 SDValue TmpInputs[4];
2752 copy(Inputs, std::begin(TmpInputs));
2753 // Track changes in the output registers.
2754 int UsedIdx = -1;
2755 bool SecondIteration = false;
2756 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
2757 if (UsedIdx < 0) {
2758 UsedIdx = Idx;
2759 return false;
2760 }
2761 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
2762 SecondIteration = true;
2763 return SecondIteration;
2764 };
2765 processShuffleMasks(
2766 Mask, std::size(Inputs), std::size(Inputs),
2767 /*NumOfUsedRegs=*/1,
2768 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2769 [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
2770 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
2771 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
2772 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
2773 else
2774 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx],
2775 DAG.getUNDEF(NewVT), Mask);
2776 Inputs[Idx] = Output;
2777 },
2778 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
2779 &TmpInputs,
2780 &BuildVector](ArrayRef<int> Mask, unsigned Idx1, unsigned Idx2) {
2781 if (AccumulateResults(Idx1)) {
2782 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2783 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2784 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2785 else
2786 Output = DAG.getVectorShuffle(NewVT, DL, Inputs[Idx1],
2787 Inputs[Idx2], Mask);
2788 } else {
2789 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
2790 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
2791 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2792 else
2793 Output = DAG.getVectorShuffle(NewVT, DL, TmpInputs[Idx1],
2794 TmpInputs[Idx2], Mask);
2795 }
2796 Inputs[Idx1] = Output;
2797 });
2798 copy(OrigInputs, std::begin(Inputs));
2799 }
2800 }
2801
SplitVecRes_VAARG(SDNode * N,SDValue & Lo,SDValue & Hi)2802 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2803 EVT OVT = N->getValueType(0);
2804 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2805 SDValue Chain = N->getOperand(0);
2806 SDValue Ptr = N->getOperand(1);
2807 SDValue SV = N->getOperand(2);
2808 SDLoc dl(N);
2809
2810 const Align Alignment =
2811 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2812
2813 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2814 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2815 Chain = Hi.getValue(1);
2816
2817 // Modified the chain - switch anything that used the old chain to use
2818 // the new one.
2819 ReplaceValueWith(SDValue(N, 1), Chain);
2820 }
2821
SplitVecRes_FP_TO_XINT_SAT(SDNode * N,SDValue & Lo,SDValue & Hi)2822 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2823 SDValue &Hi) {
2824 EVT DstVTLo, DstVTHi;
2825 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2826 SDLoc dl(N);
2827
2828 SDValue SrcLo, SrcHi;
2829 EVT SrcVT = N->getOperand(0).getValueType();
2830 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2831 GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2832 else
2833 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2834
2835 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2836 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2837 }
2838
SplitVecRes_VECTOR_REVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)2839 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
2840 SDValue &Hi) {
2841 SDValue InLo, InHi;
2842 GetSplitVector(N->getOperand(0), InLo, InHi);
2843 SDLoc DL(N);
2844
2845 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
2846 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
2847 }
2848
SplitVecRes_VECTOR_SPLICE(SDNode * N,SDValue & Lo,SDValue & Hi)2849 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
2850 SDValue &Hi) {
2851 EVT VT = N->getValueType(0);
2852 SDLoc DL(N);
2853
2854 EVT LoVT, HiVT;
2855 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2856
2857 SDValue Expanded = TLI.expandVectorSplice(N, DAG);
2858 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
2859 DAG.getVectorIdxConstant(0, DL));
2860 Hi =
2861 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
2862 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
2863 }
2864
SplitVecRes_VP_REVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)2865 void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
2866 SDValue &Hi) {
2867 EVT VT = N->getValueType(0);
2868 SDValue Val = N->getOperand(0);
2869 SDValue Mask = N->getOperand(1);
2870 SDValue EVL = N->getOperand(2);
2871 SDLoc DL(N);
2872
2873 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
2874 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
2875
2876 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
2877 VT.getVectorElementCount());
2878 SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
2879 EVT PtrVT = StackPtr.getValueType();
2880 auto &MF = DAG.getMachineFunction();
2881 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2882 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2883
2884 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
2885 PtrInfo, MachineMemOperand::MOStore, MemoryLocation::UnknownSize,
2886 Alignment);
2887 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
2888 PtrInfo, MachineMemOperand::MOLoad, MemoryLocation::UnknownSize,
2889 Alignment);
2890
2891 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
2892 SDValue NumElemMinus1 =
2893 DAG.getNode(ISD::SUB, DL, PtrVT, DAG.getZExtOrTrunc(EVL, DL, PtrVT),
2894 DAG.getConstant(1, DL, PtrVT));
2895 SDValue StartOffset = DAG.getNode(ISD::MUL, DL, PtrVT, NumElemMinus1,
2896 DAG.getConstant(EltWidth, DL, PtrVT));
2897 SDValue StorePtr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, StartOffset);
2898 SDValue Stride = DAG.getConstant(-(int64_t)EltWidth, DL, PtrVT);
2899
2900 SDValue TrueMask = DAG.getBoolConstant(true, DL, Mask.getValueType(), VT);
2901 SDValue Store = DAG.getStridedStoreVP(DAG.getEntryNode(), DL, Val, StorePtr,
2902 DAG.getUNDEF(PtrVT), Stride, TrueMask,
2903 EVL, MemVT, StoreMMO, ISD::UNINDEXED);
2904
2905 SDValue Load = DAG.getLoadVP(VT, DL, Store, StackPtr, Mask, EVL, LoadMMO);
2906
2907 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
2908 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Load,
2909 DAG.getVectorIdxConstant(0, DL));
2910 Hi =
2911 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Load,
2912 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
2913 }
2914
SplitVecRes_VECTOR_DEINTERLEAVE(SDNode * N)2915 void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
2916
2917 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2918 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
2919 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
2920 EVT VT = Op0Lo.getValueType();
2921 SDLoc DL(N);
2922 SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
2923 DAG.getVTList(VT, VT), Op0Lo, Op0Hi);
2924 SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
2925 DAG.getVTList(VT, VT), Op1Lo, Op1Hi);
2926
2927 SetSplitVector(SDValue(N, 0), ResLo.getValue(0), ResHi.getValue(0));
2928 SetSplitVector(SDValue(N, 1), ResLo.getValue(1), ResHi.getValue(1));
2929 }
2930
SplitVecRes_VECTOR_INTERLEAVE(SDNode * N)2931 void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
2932 SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi;
2933 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
2934 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
2935 EVT VT = Op0Lo.getValueType();
2936 SDLoc DL(N);
2937 SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
2938 DAG.getVTList(VT, VT), Op0Lo, Op1Lo),
2939 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
2940 DAG.getVTList(VT, VT), Op0Hi, Op1Hi)};
2941
2942 SetSplitVector(SDValue(N, 0), Res[0].getValue(0), Res[0].getValue(1));
2943 SetSplitVector(SDValue(N, 1), Res[1].getValue(0), Res[1].getValue(1));
2944 }
2945
2946 //===----------------------------------------------------------------------===//
2947 // Operand Vector Splitting
2948 //===----------------------------------------------------------------------===//
2949
2950 /// This method is called when the specified operand of the specified node is
2951 /// found to need vector splitting. At this point, all of the result types of
2952 /// the node are known to be legal, but other operands of the node may need
2953 /// legalization as well as the specified one.
SplitVectorOperand(SDNode * N,unsigned OpNo)2954 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2955 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
2956 SDValue Res = SDValue();
2957
2958 // See if the target wants to custom split this node.
2959 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2960 return false;
2961
2962 switch (N->getOpcode()) {
2963 default:
2964 #ifndef NDEBUG
2965 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2966 N->dump(&DAG);
2967 dbgs() << "\n";
2968 #endif
2969 report_fatal_error("Do not know how to split this operator's "
2970 "operand!\n");
2971
2972 case ISD::VP_SETCC:
2973 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2974 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2975 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2976 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2977 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2978 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2979 case ISD::VP_TRUNCATE:
2980 case ISD::TRUNCATE:
2981 Res = SplitVecOp_TruncateHelper(N);
2982 break;
2983 case ISD::STRICT_FP_ROUND:
2984 case ISD::VP_FP_ROUND:
2985 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2986 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
2987 case ISD::STORE:
2988 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2989 break;
2990 case ISD::VP_STORE:
2991 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
2992 break;
2993 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2994 Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(N), OpNo);
2995 break;
2996 case ISD::MSTORE:
2997 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2998 break;
2999 case ISD::MSCATTER:
3000 case ISD::VP_SCATTER:
3001 Res = SplitVecOp_Scatter(cast<MemSDNode>(N), OpNo);
3002 break;
3003 case ISD::MGATHER:
3004 case ISD::VP_GATHER:
3005 Res = SplitVecOp_Gather(cast<MemSDNode>(N), OpNo);
3006 break;
3007 case ISD::VSELECT:
3008 Res = SplitVecOp_VSELECT(N, OpNo);
3009 break;
3010 case ISD::STRICT_SINT_TO_FP:
3011 case ISD::STRICT_UINT_TO_FP:
3012 case ISD::SINT_TO_FP:
3013 case ISD::UINT_TO_FP:
3014 case ISD::VP_SINT_TO_FP:
3015 case ISD::VP_UINT_TO_FP:
3016 if (N->getValueType(0).bitsLT(
3017 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3018 Res = SplitVecOp_TruncateHelper(N);
3019 else
3020 Res = SplitVecOp_UnaryOp(N);
3021 break;
3022 case ISD::FP_TO_SINT_SAT:
3023 case ISD::FP_TO_UINT_SAT:
3024 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3025 break;
3026 case ISD::FP_TO_SINT:
3027 case ISD::FP_TO_UINT:
3028 case ISD::VP_FP_TO_SINT:
3029 case ISD::VP_FP_TO_UINT:
3030 case ISD::STRICT_FP_TO_SINT:
3031 case ISD::STRICT_FP_TO_UINT:
3032 case ISD::STRICT_FP_EXTEND:
3033 case ISD::FP_EXTEND:
3034 case ISD::SIGN_EXTEND:
3035 case ISD::ZERO_EXTEND:
3036 case ISD::ANY_EXTEND:
3037 case ISD::FTRUNC:
3038 case ISD::LRINT:
3039 case ISD::LLRINT:
3040 Res = SplitVecOp_UnaryOp(N);
3041 break;
3042 case ISD::FLDEXP:
3043 Res = SplitVecOp_FPOpDifferentTypes(N);
3044 break;
3045
3046 case ISD::ANY_EXTEND_VECTOR_INREG:
3047 case ISD::SIGN_EXTEND_VECTOR_INREG:
3048 case ISD::ZERO_EXTEND_VECTOR_INREG:
3049 Res = SplitVecOp_ExtVecInRegOp(N);
3050 break;
3051
3052 case ISD::VECREDUCE_FADD:
3053 case ISD::VECREDUCE_FMUL:
3054 case ISD::VECREDUCE_ADD:
3055 case ISD::VECREDUCE_MUL:
3056 case ISD::VECREDUCE_AND:
3057 case ISD::VECREDUCE_OR:
3058 case ISD::VECREDUCE_XOR:
3059 case ISD::VECREDUCE_SMAX:
3060 case ISD::VECREDUCE_SMIN:
3061 case ISD::VECREDUCE_UMAX:
3062 case ISD::VECREDUCE_UMIN:
3063 case ISD::VECREDUCE_FMAX:
3064 case ISD::VECREDUCE_FMIN:
3065 case ISD::VECREDUCE_FMAXIMUM:
3066 case ISD::VECREDUCE_FMINIMUM:
3067 Res = SplitVecOp_VECREDUCE(N, OpNo);
3068 break;
3069 case ISD::VECREDUCE_SEQ_FADD:
3070 case ISD::VECREDUCE_SEQ_FMUL:
3071 Res = SplitVecOp_VECREDUCE_SEQ(N);
3072 break;
3073 case ISD::VP_REDUCE_FADD:
3074 case ISD::VP_REDUCE_SEQ_FADD:
3075 case ISD::VP_REDUCE_FMUL:
3076 case ISD::VP_REDUCE_SEQ_FMUL:
3077 case ISD::VP_REDUCE_ADD:
3078 case ISD::VP_REDUCE_MUL:
3079 case ISD::VP_REDUCE_AND:
3080 case ISD::VP_REDUCE_OR:
3081 case ISD::VP_REDUCE_XOR:
3082 case ISD::VP_REDUCE_SMAX:
3083 case ISD::VP_REDUCE_SMIN:
3084 case ISD::VP_REDUCE_UMAX:
3085 case ISD::VP_REDUCE_UMIN:
3086 case ISD::VP_REDUCE_FMAX:
3087 case ISD::VP_REDUCE_FMIN:
3088 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3089 break;
3090 }
3091
3092 // If the result is null, the sub-method took care of registering results etc.
3093 if (!Res.getNode()) return false;
3094
3095 // If the result is N, the sub-method updated N in place. Tell the legalizer
3096 // core about this.
3097 if (Res.getNode() == N)
3098 return true;
3099
3100 if (N->isStrictFPOpcode())
3101 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3102 "Invalid operand expansion");
3103 else
3104 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3105 "Invalid operand expansion");
3106
3107 ReplaceValueWith(SDValue(N, 0), Res);
3108 return false;
3109 }
3110
SplitVecOp_VSELECT(SDNode * N,unsigned OpNo)3111 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3112 // The only possibility for an illegal operand is the mask, since result type
3113 // legalization would have handled this node already otherwise.
3114 assert(OpNo == 0 && "Illegal operand must be mask");
3115
3116 SDValue Mask = N->getOperand(0);
3117 SDValue Src0 = N->getOperand(1);
3118 SDValue Src1 = N->getOperand(2);
3119 EVT Src0VT = Src0.getValueType();
3120 SDLoc DL(N);
3121 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
3122
3123 SDValue Lo, Hi;
3124 GetSplitVector(N->getOperand(0), Lo, Hi);
3125 assert(Lo.getValueType() == Hi.getValueType() &&
3126 "Lo and Hi have differing types");
3127
3128 EVT LoOpVT, HiOpVT;
3129 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
3130 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
3131
3132 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
3133 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
3134 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
3135 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
3136
3137 SDValue LoSelect =
3138 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
3139 SDValue HiSelect =
3140 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
3141
3142 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
3143 }
3144
SplitVecOp_VECREDUCE(SDNode * N,unsigned OpNo)3145 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
3146 EVT ResVT = N->getValueType(0);
3147 SDValue Lo, Hi;
3148 SDLoc dl(N);
3149
3150 SDValue VecOp = N->getOperand(OpNo);
3151 EVT VecVT = VecOp.getValueType();
3152 assert(VecVT.isVector() && "Can only split reduce vector operand");
3153 GetSplitVector(VecOp, Lo, Hi);
3154 EVT LoOpVT, HiOpVT;
3155 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3156
3157 // Use the appropriate scalar instruction on the split subvectors before
3158 // reducing the now partially reduced smaller vector.
3159 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
3160 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
3161 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
3162 }
3163
SplitVecOp_VECREDUCE_SEQ(SDNode * N)3164 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
3165 EVT ResVT = N->getValueType(0);
3166 SDValue Lo, Hi;
3167 SDLoc dl(N);
3168
3169 SDValue AccOp = N->getOperand(0);
3170 SDValue VecOp = N->getOperand(1);
3171 SDNodeFlags Flags = N->getFlags();
3172
3173 EVT VecVT = VecOp.getValueType();
3174 assert(VecVT.isVector() && "Can only split reduce vector operand");
3175 GetSplitVector(VecOp, Lo, Hi);
3176 EVT LoOpVT, HiOpVT;
3177 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3178
3179 // Reduce low half.
3180 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
3181
3182 // Reduce high half, using low half result as initial value.
3183 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
3184 }
3185
SplitVecOp_VP_REDUCE(SDNode * N,unsigned OpNo)3186 SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
3187 assert(N->isVPOpcode() && "Expected VP opcode");
3188 assert(OpNo == 1 && "Can only split reduce vector operand");
3189
3190 unsigned Opc = N->getOpcode();
3191 EVT ResVT = N->getValueType(0);
3192 SDValue Lo, Hi;
3193 SDLoc dl(N);
3194
3195 SDValue VecOp = N->getOperand(OpNo);
3196 EVT VecVT = VecOp.getValueType();
3197 assert(VecVT.isVector() && "Can only split reduce vector operand");
3198 GetSplitVector(VecOp, Lo, Hi);
3199
3200 SDValue MaskLo, MaskHi;
3201 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(2));
3202
3203 SDValue EVLLo, EVLHi;
3204 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(N->getOperand(3), VecVT, dl);
3205
3206 const SDNodeFlags Flags = N->getFlags();
3207
3208 SDValue ResLo =
3209 DAG.getNode(Opc, dl, ResVT, {N->getOperand(0), Lo, MaskLo, EVLLo}, Flags);
3210 return DAG.getNode(Opc, dl, ResVT, {ResLo, Hi, MaskHi, EVLHi}, Flags);
3211 }
3212
SplitVecOp_UnaryOp(SDNode * N)3213 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
3214 // The result has a legal vector type, but the input needs splitting.
3215 EVT ResVT = N->getValueType(0);
3216 SDValue Lo, Hi;
3217 SDLoc dl(N);
3218 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3219 EVT InVT = Lo.getValueType();
3220
3221 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3222 InVT.getVectorElementCount());
3223
3224 if (N->isStrictFPOpcode()) {
3225 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3226 { N->getOperand(0), Lo });
3227 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
3228 { N->getOperand(0), Hi });
3229
3230 // Build a factor node to remember that this operation is independent
3231 // of the other one.
3232 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3233 Hi.getValue(1));
3234
3235 // Legalize the chain result - switch anything that used the old chain to
3236 // use the new one.
3237 ReplaceValueWith(SDValue(N, 1), Ch);
3238 } else if (N->getNumOperands() == 3) {
3239 assert(N->isVPOpcode() && "Expected VP opcode");
3240 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3241 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3242 std::tie(EVLLo, EVLHi) =
3243 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
3244 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo, MaskLo, EVLLo);
3245 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi, MaskHi, EVLHi);
3246 } else {
3247 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
3248 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
3249 }
3250
3251 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
3252 }
3253
SplitVecOp_BITCAST(SDNode * N)3254 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
3255 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
3256 // end up being split all the way down to individual components. Convert the
3257 // split pieces into integers and reassemble.
3258 SDValue Lo, Hi;
3259 GetSplitVector(N->getOperand(0), Lo, Hi);
3260 Lo = BitConvertToInteger(Lo);
3261 Hi = BitConvertToInteger(Hi);
3262
3263 if (DAG.getDataLayout().isBigEndian())
3264 std::swap(Lo, Hi);
3265
3266 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
3267 JoinIntegers(Lo, Hi));
3268 }
3269
SplitVecOp_INSERT_SUBVECTOR(SDNode * N,unsigned OpNo)3270 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
3271 unsigned OpNo) {
3272 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
3273 // We know that the result type is legal.
3274 EVT ResVT = N->getValueType(0);
3275
3276 SDValue Vec = N->getOperand(0);
3277 SDValue SubVec = N->getOperand(1);
3278 SDValue Idx = N->getOperand(2);
3279 SDLoc dl(N);
3280
3281 SDValue Lo, Hi;
3282 GetSplitVector(SubVec, Lo, Hi);
3283
3284 uint64_t IdxVal = Idx->getAsZExtVal();
3285 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3286
3287 SDValue FirstInsertion =
3288 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
3289 SDValue SecondInsertion =
3290 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
3291 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
3292
3293 return SecondInsertion;
3294 }
3295
SplitVecOp_EXTRACT_SUBVECTOR(SDNode * N)3296 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
3297 // We know that the extracted result type is legal.
3298 EVT SubVT = N->getValueType(0);
3299 SDValue Idx = N->getOperand(1);
3300 SDLoc dl(N);
3301 SDValue Lo, Hi;
3302
3303 GetSplitVector(N->getOperand(0), Lo, Hi);
3304
3305 uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
3306 uint64_t IdxVal = Idx->getAsZExtVal();
3307
3308 if (IdxVal < LoEltsMin) {
3309 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
3310 "Extracted subvector crosses vector split!");
3311 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
3312 } else if (SubVT.isScalableVector() ==
3313 N->getOperand(0).getValueType().isScalableVector())
3314 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
3315 DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl));
3316
3317 // After this point the DAG node only permits extracting fixed-width
3318 // subvectors from scalable vectors.
3319 assert(SubVT.isFixedLengthVector() &&
3320 "Extracting scalable subvector from fixed-width unsupported");
3321
3322 // If the element type is i1 and we're not promoting the result, then we may
3323 // end up loading the wrong data since the bits are packed tightly into
3324 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
3325 // type at index 4, then we will load a byte starting at index 0.
3326 if (SubVT.getScalarType() == MVT::i1)
3327 report_fatal_error("Don't know how to extract fixed-width predicate "
3328 "subvector from a scalable predicate vector");
3329
3330 // Spill the vector to the stack. We should use the alignment for
3331 // the smallest part.
3332 SDValue Vec = N->getOperand(0);
3333 EVT VecVT = Vec.getValueType();
3334 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3335 SDValue StackPtr =
3336 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3337 auto &MF = DAG.getMachineFunction();
3338 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3339 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3340
3341 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3342 SmallestAlign);
3343
3344 // Extract the subvector by loading the correct part.
3345 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVT, Idx);
3346
3347 return DAG.getLoad(
3348 SubVT, dl, Store, StackPtr,
3349 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3350 }
3351
SplitVecOp_EXTRACT_VECTOR_ELT(SDNode * N)3352 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3353 SDValue Vec = N->getOperand(0);
3354 SDValue Idx = N->getOperand(1);
3355 EVT VecVT = Vec.getValueType();
3356
3357 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Idx)) {
3358 uint64_t IdxVal = Index->getZExtValue();
3359
3360 SDValue Lo, Hi;
3361 GetSplitVector(Vec, Lo, Hi);
3362
3363 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
3364
3365 if (IdxVal < LoElts)
3366 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
3367 else if (!Vec.getValueType().isScalableVector())
3368 return SDValue(DAG.UpdateNodeOperands(N, Hi,
3369 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
3370 Idx.getValueType())), 0);
3371 }
3372
3373 // See if the target wants to custom expand this node.
3374 if (CustomLowerNode(N, N->getValueType(0), true))
3375 return SDValue();
3376
3377 // Make the vector elements byte-addressable if they aren't already.
3378 SDLoc dl(N);
3379 EVT EltVT = VecVT.getVectorElementType();
3380 if (VecVT.getScalarSizeInBits() < 8) {
3381 EltVT = MVT::i8;
3382 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
3383 VecVT.getVectorElementCount());
3384 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
3385 }
3386
3387 // Store the vector to the stack.
3388 // In cases where the vector is illegal it will be broken down into parts
3389 // and stored in parts - we should use the alignment for the smallest part.
3390 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
3391 SDValue StackPtr =
3392 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
3393 auto &MF = DAG.getMachineFunction();
3394 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
3395 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
3396 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3397 SmallestAlign);
3398
3399 // Load back the required element.
3400 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
3401
3402 // FIXME: This is to handle i1 vectors with elements promoted to i8.
3403 // i1 vector handling needs general improvement.
3404 if (N->getValueType(0).bitsLT(EltVT)) {
3405 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3406 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
3407 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
3408 }
3409
3410 return DAG.getExtLoad(
3411 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
3412 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
3413 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
3414 }
3415
SplitVecOp_ExtVecInRegOp(SDNode * N)3416 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
3417 SDValue Lo, Hi;
3418
3419 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
3420 // splitting the result has the same effect as splitting the input operand.
3421 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
3422
3423 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
3424 }
3425
SplitVecOp_Gather(MemSDNode * N,unsigned OpNo)3426 SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
3427 (void)OpNo;
3428 SDValue Lo, Hi;
3429 SplitVecRes_Gather(N, Lo, Hi);
3430
3431 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, N, N->getValueType(0), Lo, Hi);
3432 ReplaceValueWith(SDValue(N, 0), Res);
3433 return SDValue();
3434 }
3435
SplitVecOp_VP_STORE(VPStoreSDNode * N,unsigned OpNo)3436 SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
3437 assert(N->isUnindexed() && "Indexed vp_store of vector?");
3438 SDValue Ch = N->getChain();
3439 SDValue Ptr = N->getBasePtr();
3440 SDValue Offset = N->getOffset();
3441 assert(Offset.isUndef() && "Unexpected VP store offset");
3442 SDValue Mask = N->getMask();
3443 SDValue EVL = N->getVectorLength();
3444 SDValue Data = N->getValue();
3445 Align Alignment = N->getOriginalAlign();
3446 SDLoc DL(N);
3447
3448 SDValue DataLo, DataHi;
3449 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3450 // Split Data operand
3451 GetSplitVector(Data, DataLo, DataHi);
3452 else
3453 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3454
3455 // Split Mask operand
3456 SDValue MaskLo, MaskHi;
3457 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3458 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3459 } else {
3460 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3461 GetSplitVector(Mask, MaskLo, MaskHi);
3462 else
3463 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3464 }
3465
3466 EVT MemoryVT = N->getMemoryVT();
3467 EVT LoMemVT, HiMemVT;
3468 bool HiIsEmpty = false;
3469 std::tie(LoMemVT, HiMemVT) =
3470 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3471
3472 // Split EVL
3473 SDValue EVLLo, EVLHi;
3474 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL, Data.getValueType(), DL);
3475
3476 SDValue Lo, Hi;
3477 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3478 N->getPointerInfo(), MachineMemOperand::MOStore,
3479 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3480
3481 Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
3482 N->getAddressingMode(), N->isTruncatingStore(),
3483 N->isCompressingStore());
3484
3485 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
3486 if (HiIsEmpty)
3487 return Lo;
3488
3489 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3490 N->isCompressingStore());
3491
3492 MachinePointerInfo MPI;
3493 if (LoMemVT.isScalableVector()) {
3494 Alignment = commonAlignment(Alignment,
3495 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3496 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3497 } else
3498 MPI = N->getPointerInfo().getWithOffset(
3499 LoMemVT.getStoreSize().getFixedValue());
3500
3501 MMO = DAG.getMachineFunction().getMachineMemOperand(
3502 MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
3503 N->getAAInfo(), N->getRanges());
3504
3505 Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
3506 N->getAddressingMode(), N->isTruncatingStore(),
3507 N->isCompressingStore());
3508
3509 // Build a factor node to remember that this store is independent of the
3510 // other one.
3511 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3512 }
3513
SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode * N,unsigned OpNo)3514 SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
3515 unsigned OpNo) {
3516 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
3517 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
3518
3519 SDLoc DL(N);
3520
3521 SDValue Data = N->getValue();
3522 SDValue LoData, HiData;
3523 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3524 GetSplitVector(Data, LoData, HiData);
3525 else
3526 std::tie(LoData, HiData) = DAG.SplitVector(Data, DL);
3527
3528 EVT LoMemVT, HiMemVT;
3529 bool HiIsEmpty = false;
3530 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
3531 N->getMemoryVT(), LoData.getValueType(), &HiIsEmpty);
3532
3533 SDValue Mask = N->getMask();
3534 SDValue LoMask, HiMask;
3535 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
3536 SplitVecRes_SETCC(Mask.getNode(), LoMask, HiMask);
3537 else if (getTypeAction(Mask.getValueType()) ==
3538 TargetLowering::TypeSplitVector)
3539 GetSplitVector(Mask, LoMask, HiMask);
3540 else
3541 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
3542
3543 SDValue LoEVL, HiEVL;
3544 std::tie(LoEVL, HiEVL) =
3545 DAG.SplitEVL(N->getVectorLength(), Data.getValueType(), DL);
3546
3547 // Generate the low vp_strided_store
3548 SDValue Lo = DAG.getStridedStoreVP(
3549 N->getChain(), DL, LoData, N->getBasePtr(), N->getOffset(),
3550 N->getStride(), LoMask, LoEVL, LoMemVT, N->getMemOperand(),
3551 N->getAddressingMode(), N->isTruncatingStore(), N->isCompressingStore());
3552
3553 // If the high vp_strided_store has zero storage size, only the low
3554 // vp_strided_store is needed.
3555 if (HiIsEmpty)
3556 return Lo;
3557
3558 // Generate the high vp_strided_store.
3559 // To calculate the high base address, we need to sum to the low base
3560 // address stride number of bytes for each element already stored by low,
3561 // that is: Ptr = Ptr + (LoEVL * Stride)
3562 EVT PtrVT = N->getBasePtr().getValueType();
3563 SDValue Increment =
3564 DAG.getNode(ISD::MUL, DL, PtrVT, LoEVL,
3565 DAG.getSExtOrTrunc(N->getStride(), DL, PtrVT));
3566 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, N->getBasePtr(), Increment);
3567
3568 Align Alignment = N->getOriginalAlign();
3569 if (LoMemVT.isScalableVector())
3570 Alignment = commonAlignment(Alignment,
3571 LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3572
3573 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3574 MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
3575 MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
3576 N->getAAInfo(), N->getRanges());
3577
3578 SDValue Hi = DAG.getStridedStoreVP(
3579 N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
3580 HiEVL, HiMemVT, MMO, N->getAddressingMode(), N->isTruncatingStore(),
3581 N->isCompressingStore());
3582
3583 // Build a factor node to remember that this store is independent of the
3584 // other one.
3585 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3586 }
3587
SplitVecOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)3588 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
3589 unsigned OpNo) {
3590 assert(N->isUnindexed() && "Indexed masked store of vector?");
3591 SDValue Ch = N->getChain();
3592 SDValue Ptr = N->getBasePtr();
3593 SDValue Offset = N->getOffset();
3594 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
3595 SDValue Mask = N->getMask();
3596 SDValue Data = N->getValue();
3597 Align Alignment = N->getOriginalAlign();
3598 SDLoc DL(N);
3599
3600 SDValue DataLo, DataHi;
3601 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
3602 // Split Data operand
3603 GetSplitVector(Data, DataLo, DataHi);
3604 else
3605 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
3606
3607 // Split Mask operand
3608 SDValue MaskLo, MaskHi;
3609 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
3610 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
3611 } else {
3612 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
3613 GetSplitVector(Mask, MaskLo, MaskHi);
3614 else
3615 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
3616 }
3617
3618 EVT MemoryVT = N->getMemoryVT();
3619 EVT LoMemVT, HiMemVT;
3620 bool HiIsEmpty = false;
3621 std::tie(LoMemVT, HiMemVT) =
3622 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
3623
3624 SDValue Lo, Hi, Res;
3625 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3626 N->getPointerInfo(), MachineMemOperand::MOStore,
3627 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3628
3629 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
3630 N->getAddressingMode(), N->isTruncatingStore(),
3631 N->isCompressingStore());
3632
3633 if (HiIsEmpty) {
3634 // The hi masked store has zero storage size.
3635 // Only the lo masked store is needed.
3636 Res = Lo;
3637 } else {
3638
3639 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
3640 N->isCompressingStore());
3641
3642 MachinePointerInfo MPI;
3643 if (LoMemVT.isScalableVector()) {
3644 Alignment = commonAlignment(
3645 Alignment, LoMemVT.getSizeInBits().getKnownMinValue() / 8);
3646 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
3647 } else
3648 MPI = N->getPointerInfo().getWithOffset(
3649 LoMemVT.getStoreSize().getFixedValue());
3650
3651 MMO = DAG.getMachineFunction().getMachineMemOperand(
3652 MPI, MachineMemOperand::MOStore, MemoryLocation::UnknownSize, Alignment,
3653 N->getAAInfo(), N->getRanges());
3654
3655 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
3656 N->getAddressingMode(), N->isTruncatingStore(),
3657 N->isCompressingStore());
3658
3659 // Build a factor node to remember that this store is independent of the
3660 // other one.
3661 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3662 }
3663
3664 return Res;
3665 }
3666
SplitVecOp_Scatter(MemSDNode * N,unsigned OpNo)3667 SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
3668 SDValue Ch = N->getChain();
3669 SDValue Ptr = N->getBasePtr();
3670 EVT MemoryVT = N->getMemoryVT();
3671 Align Alignment = N->getOriginalAlign();
3672 SDLoc DL(N);
3673 struct Operands {
3674 SDValue Mask;
3675 SDValue Index;
3676 SDValue Scale;
3677 SDValue Data;
3678 } Ops = [&]() -> Operands {
3679 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3680 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3681 MSC->getValue()};
3682 }
3683 auto *VPSC = cast<VPScatterSDNode>(N);
3684 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3685 VPSC->getValue()};
3686 }();
3687 // Split all operands
3688
3689 EVT LoMemVT, HiMemVT;
3690 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3691
3692 SDValue DataLo, DataHi;
3693 if (getTypeAction(Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
3694 // Split Data operand
3695 GetSplitVector(Ops.Data, DataLo, DataHi);
3696 else
3697 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data, DL);
3698
3699 // Split Mask operand
3700 SDValue MaskLo, MaskHi;
3701 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
3702 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3703 } else {
3704 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, DL);
3705 }
3706
3707 SDValue IndexHi, IndexLo;
3708 if (getTypeAction(Ops.Index.getValueType()) ==
3709 TargetLowering::TypeSplitVector)
3710 GetSplitVector(Ops.Index, IndexLo, IndexHi);
3711 else
3712 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
3713
3714 SDValue Lo;
3715 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
3716 N->getPointerInfo(), MachineMemOperand::MOStore,
3717 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
3718
3719 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
3720 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
3721 Lo =
3722 DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3723 MSC->getIndexType(), MSC->isTruncatingStore());
3724
3725 // The order of the Scatter operation after split is well defined. The "Hi"
3726 // part comes after the "Lo". So these two operations should be chained one
3727 // after another.
3728 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
3729 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi,
3730 MMO, MSC->getIndexType(),
3731 MSC->isTruncatingStore());
3732 }
3733 auto *VPSC = cast<VPScatterSDNode>(N);
3734 SDValue EVLLo, EVLHi;
3735 std::tie(EVLLo, EVLHi) =
3736 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(), DL);
3737
3738 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3739 Lo = DAG.getScatterVP(DAG.getVTList(MVT::Other), LoMemVT, DL, OpsLo, MMO,
3740 VPSC->getIndexType());
3741
3742 // The order of the Scatter operation after split is well defined. The "Hi"
3743 // part comes after the "Lo". So these two operations should be chained one
3744 // after another.
3745 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3746 return DAG.getScatterVP(DAG.getVTList(MVT::Other), HiMemVT, DL, OpsHi, MMO,
3747 VPSC->getIndexType());
3748 }
3749
SplitVecOp_STORE(StoreSDNode * N,unsigned OpNo)3750 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
3751 assert(N->isUnindexed() && "Indexed store of vector?");
3752 assert(OpNo == 1 && "Can only split the stored value");
3753 SDLoc DL(N);
3754
3755 bool isTruncating = N->isTruncatingStore();
3756 SDValue Ch = N->getChain();
3757 SDValue Ptr = N->getBasePtr();
3758 EVT MemoryVT = N->getMemoryVT();
3759 Align Alignment = N->getOriginalAlign();
3760 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3761 AAMDNodes AAInfo = N->getAAInfo();
3762 SDValue Lo, Hi;
3763 GetSplitVector(N->getOperand(1), Lo, Hi);
3764
3765 EVT LoMemVT, HiMemVT;
3766 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3767
3768 // Scalarize if the split halves are not byte-sized.
3769 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
3770 return TLI.scalarizeVectorStore(N, DAG);
3771
3772 if (isTruncating)
3773 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
3774 Alignment, MMOFlags, AAInfo);
3775 else
3776 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3777 AAInfo);
3778
3779 MachinePointerInfo MPI;
3780 IncrementPointer(N, LoMemVT, MPI, Ptr);
3781
3782 if (isTruncating)
3783 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
3784 HiMemVT, Alignment, MMOFlags, AAInfo);
3785 else
3786 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
3787
3788 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
3789 }
3790
SplitVecOp_CONCAT_VECTORS(SDNode * N)3791 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
3792 SDLoc DL(N);
3793
3794 // The input operands all must have the same type, and we know the result
3795 // type is valid. Convert this to a buildvector which extracts all the
3796 // input elements.
3797 // TODO: If the input elements are power-two vectors, we could convert this to
3798 // a new CONCAT_VECTORS node with elements that are half-wide.
3799 SmallVector<SDValue, 32> Elts;
3800 EVT EltVT = N->getValueType(0).getVectorElementType();
3801 for (const SDValue &Op : N->op_values()) {
3802 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
3803 i != e; ++i) {
3804 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
3805 DAG.getVectorIdxConstant(i, DL)));
3806 }
3807 }
3808
3809 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
3810 }
3811
SplitVecOp_TruncateHelper(SDNode * N)3812 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
3813 // The result type is legal, but the input type is illegal. If splitting
3814 // ends up with the result type of each half still being legal, just
3815 // do that. If, however, that would result in an illegal result type,
3816 // we can try to get more clever with power-two vectors. Specifically,
3817 // split the input type, but also widen the result element size, then
3818 // concatenate the halves and truncate again. For example, consider a target
3819 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
3820 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
3821 // %inlo = v4i32 extract_subvector %in, 0
3822 // %inhi = v4i32 extract_subvector %in, 4
3823 // %lo16 = v4i16 trunc v4i32 %inlo
3824 // %hi16 = v4i16 trunc v4i32 %inhi
3825 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
3826 // %res = v8i8 trunc v8i16 %in16
3827 //
3828 // Without this transform, the original truncate would end up being
3829 // scalarized, which is pretty much always a last resort.
3830 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3831 SDValue InVec = N->getOperand(OpNo);
3832 EVT InVT = InVec->getValueType(0);
3833 EVT OutVT = N->getValueType(0);
3834 ElementCount NumElements = OutVT.getVectorElementCount();
3835 bool IsFloat = OutVT.isFloatingPoint();
3836
3837 unsigned InElementSize = InVT.getScalarSizeInBits();
3838 unsigned OutElementSize = OutVT.getScalarSizeInBits();
3839
3840 // Determine the split output VT. If its legal we can just split dirctly.
3841 EVT LoOutVT, HiOutVT;
3842 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3843 assert(LoOutVT == HiOutVT && "Unequal split?");
3844
3845 // If the input elements are only 1/2 the width of the result elements,
3846 // just use the normal splitting. Our trick only work if there's room
3847 // to split more than once.
3848 if (isTypeLegal(LoOutVT) ||
3849 InElementSize <= OutElementSize * 2)
3850 return SplitVecOp_UnaryOp(N);
3851 SDLoc DL(N);
3852
3853 // Don't touch if this will be scalarized.
3854 EVT FinalVT = InVT;
3855 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3856 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
3857
3858 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
3859 return SplitVecOp_UnaryOp(N);
3860
3861 // Get the split input vector.
3862 SDValue InLoVec, InHiVec;
3863 GetSplitVector(InVec, InLoVec, InHiVec);
3864
3865 // Truncate them to 1/2 the element size.
3866 //
3867 // This assumes the number of elements is a power of two; any vector that
3868 // isn't should be widened, not split.
3869 EVT HalfElementVT = IsFloat ?
3870 EVT::getFloatingPointVT(InElementSize/2) :
3871 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3872 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
3873 NumElements.divideCoefficientBy(2));
3874
3875 SDValue HalfLo;
3876 SDValue HalfHi;
3877 SDValue Chain;
3878 if (N->isStrictFPOpcode()) {
3879 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3880 {N->getOperand(0), InLoVec});
3881 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
3882 {N->getOperand(0), InHiVec});
3883 // Legalize the chain result - switch anything that used the old chain to
3884 // use the new one.
3885 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
3886 HalfHi.getValue(1));
3887 } else {
3888 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
3889 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
3890 }
3891
3892 // Concatenate them to get the full intermediate truncation result.
3893 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
3894 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
3895 HalfHi);
3896 // Now finish up by truncating all the way down to the original result
3897 // type. This should normally be something that ends up being legal directly,
3898 // but in theory if a target has very wide vectors and an annoyingly
3899 // restricted set of legal types, this split can chain to build things up.
3900
3901 if (N->isStrictFPOpcode()) {
3902 SDValue Res = DAG.getNode(
3903 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
3904 {Chain, InterVec,
3905 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
3906 // Relink the chain
3907 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
3908 return Res;
3909 }
3910
3911 return IsFloat
3912 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
3913 DAG.getTargetConstant(
3914 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
3915 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
3916 }
3917
SplitVecOp_VSETCC(SDNode * N)3918 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
3919 assert(N->getValueType(0).isVector() &&
3920 N->getOperand(0).getValueType().isVector() &&
3921 "Operand types must be vectors");
3922 // The result has a legal vector type, but the input needs splitting.
3923 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3924 SDLoc DL(N);
3925 GetSplitVector(N->getOperand(0), Lo0, Hi0);
3926 GetSplitVector(N->getOperand(1), Lo1, Hi1);
3927 auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
3928
3929 LLVMContext &Context = *DAG.getContext();
3930 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
3931 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
3932
3933 if (N->getOpcode() == ISD::SETCC) {
3934 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
3935 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
3936 } else {
3937 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
3938 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3939 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(3));
3940 std::tie(EVLLo, EVLHi) =
3941 DAG.SplitEVL(N->getOperand(4), N->getValueType(0), DL);
3942 LoRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Lo0, Lo1,
3943 N->getOperand(2), MaskLo, EVLLo);
3944 HiRes = DAG.getNode(ISD::VP_SETCC, DL, PartResVT, Hi0, Hi1,
3945 N->getOperand(2), MaskHi, EVLHi);
3946 }
3947 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
3948
3949 EVT OpVT = N->getOperand(0).getValueType();
3950 ISD::NodeType ExtendCode =
3951 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
3952 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
3953 }
3954
3955
SplitVecOp_FP_ROUND(SDNode * N)3956 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
3957 // The result has a legal vector type, but the input needs splitting.
3958 EVT ResVT = N->getValueType(0);
3959 SDValue Lo, Hi;
3960 SDLoc DL(N);
3961 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
3962 EVT InVT = Lo.getValueType();
3963
3964 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
3965 InVT.getVectorElementCount());
3966
3967 if (N->isStrictFPOpcode()) {
3968 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3969 { N->getOperand(0), Lo, N->getOperand(2) });
3970 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
3971 { N->getOperand(0), Hi, N->getOperand(2) });
3972 // Legalize the chain result - switch anything that used the old chain to
3973 // use the new one.
3974 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3975 Lo.getValue(1), Hi.getValue(1));
3976 ReplaceValueWith(SDValue(N, 1), NewChain);
3977 } else if (N->getOpcode() == ISD::VP_FP_ROUND) {
3978 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3979 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
3980 std::tie(EVLLo, EVLHi) =
3981 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), DL);
3982 Lo = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Lo, MaskLo, EVLLo);
3983 Hi = DAG.getNode(ISD::VP_FP_ROUND, DL, OutVT, Hi, MaskHi, EVLHi);
3984 } else {
3985 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
3986 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
3987 }
3988
3989 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
3990 }
3991
3992 // Split a vector type in an FP binary operation where the second operand has a
3993 // different type from the first.
3994 //
3995 // The result (and the first input) has a legal vector type, but the second
3996 // input needs splitting.
SplitVecOp_FPOpDifferentTypes(SDNode * N)3997 SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
3998 SDLoc DL(N);
3999
4000 EVT LHSLoVT, LHSHiVT;
4001 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
4002
4003 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
4004 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
4005
4006 SDValue LHSLo, LHSHi;
4007 std::tie(LHSLo, LHSHi) =
4008 DAG.SplitVector(N->getOperand(0), DL, LHSLoVT, LHSHiVT);
4009
4010 SDValue RHSLo, RHSHi;
4011 std::tie(RHSLo, RHSHi) = DAG.SplitVector(N->getOperand(1), DL);
4012
4013 SDValue Lo = DAG.getNode(N->getOpcode(), DL, LHSLoVT, LHSLo, RHSLo);
4014 SDValue Hi = DAG.getNode(N->getOpcode(), DL, LHSHiVT, LHSHi, RHSHi);
4015
4016 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), Lo, Hi);
4017 }
4018
SplitVecOp_FP_TO_XINT_SAT(SDNode * N)4019 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
4020 EVT ResVT = N->getValueType(0);
4021 SDValue Lo, Hi;
4022 SDLoc dl(N);
4023 GetSplitVector(N->getOperand(0), Lo, Hi);
4024 EVT InVT = Lo.getValueType();
4025
4026 EVT NewResVT =
4027 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
4028 InVT.getVectorElementCount());
4029
4030 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
4031 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
4032
4033 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
4034 }
4035
4036 //===----------------------------------------------------------------------===//
4037 // Result Vector Widening
4038 //===----------------------------------------------------------------------===//
4039
WidenVectorResult(SDNode * N,unsigned ResNo)4040 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
4041 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
4042
4043 // See if the target wants to custom widen this node.
4044 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
4045 return;
4046
4047 SDValue Res = SDValue();
4048
4049 auto unrollExpandedOp = [&]() {
4050 // We're going to widen this vector op to a legal type by padding with undef
4051 // elements. If the wide vector op is eventually going to be expanded to
4052 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
4053 // libcalls on the undef elements.
4054 EVT VT = N->getValueType(0);
4055 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4056 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
4057 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
4058 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
4059 return true;
4060 }
4061 return false;
4062 };
4063
4064 switch (N->getOpcode()) {
4065 default:
4066 #ifndef NDEBUG
4067 dbgs() << "WidenVectorResult #" << ResNo << ": ";
4068 N->dump(&DAG);
4069 dbgs() << "\n";
4070 #endif
4071 report_fatal_error("Do not know how to widen the result of this operator!");
4072
4073 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
4074 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
4075 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
4076 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
4077 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
4078 case ISD::INSERT_SUBVECTOR:
4079 Res = WidenVecRes_INSERT_SUBVECTOR(N);
4080 break;
4081 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
4082 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
4083 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
4084 case ISD::STEP_VECTOR:
4085 case ISD::SPLAT_VECTOR:
4086 case ISD::SCALAR_TO_VECTOR:
4087 Res = WidenVecRes_ScalarOp(N);
4088 break;
4089 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
4090 case ISD::VSELECT:
4091 case ISD::SELECT:
4092 case ISD::VP_SELECT:
4093 case ISD::VP_MERGE:
4094 Res = WidenVecRes_Select(N);
4095 break;
4096 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
4097 case ISD::VP_SETCC:
4098 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
4099 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
4100 case ISD::VECTOR_SHUFFLE:
4101 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
4102 break;
4103 case ISD::VP_LOAD:
4104 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(N));
4105 break;
4106 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
4107 Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(N));
4108 break;
4109 case ISD::MLOAD:
4110 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
4111 break;
4112 case ISD::MGATHER:
4113 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
4114 break;
4115 case ISD::VP_GATHER:
4116 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(N));
4117 break;
4118 case ISD::VECTOR_REVERSE:
4119 Res = WidenVecRes_VECTOR_REVERSE(N);
4120 break;
4121
4122 case ISD::ADD: case ISD::VP_ADD:
4123 case ISD::AND: case ISD::VP_AND:
4124 case ISD::MUL: case ISD::VP_MUL:
4125 case ISD::MULHS:
4126 case ISD::MULHU:
4127 case ISD::OR: case ISD::VP_OR:
4128 case ISD::SUB: case ISD::VP_SUB:
4129 case ISD::XOR: case ISD::VP_XOR:
4130 case ISD::SHL: case ISD::VP_SHL:
4131 case ISD::SRA: case ISD::VP_ASHR:
4132 case ISD::SRL: case ISD::VP_LSHR:
4133 case ISD::FMINNUM: case ISD::VP_FMINNUM:
4134 case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
4135 case ISD::FMINIMUM:
4136 case ISD::VP_FMINIMUM:
4137 case ISD::FMAXIMUM:
4138 case ISD::VP_FMAXIMUM:
4139 case ISD::SMIN: case ISD::VP_SMIN:
4140 case ISD::SMAX: case ISD::VP_SMAX:
4141 case ISD::UMIN: case ISD::VP_UMIN:
4142 case ISD::UMAX: case ISD::VP_UMAX:
4143 case ISD::UADDSAT:
4144 case ISD::SADDSAT:
4145 case ISD::USUBSAT:
4146 case ISD::SSUBSAT:
4147 case ISD::SSHLSAT:
4148 case ISD::USHLSAT:
4149 case ISD::ROTL:
4150 case ISD::ROTR:
4151 case ISD::AVGFLOORS:
4152 case ISD::AVGFLOORU:
4153 case ISD::AVGCEILS:
4154 case ISD::AVGCEILU:
4155 // Vector-predicated binary op widening. Note that -- unlike the
4156 // unpredicated versions -- we don't have to worry about trapping on
4157 // operations like UDIV, FADD, etc., as we pass on the original vector
4158 // length parameter. This means the widened elements containing garbage
4159 // aren't active.
4160 case ISD::VP_SDIV:
4161 case ISD::VP_UDIV:
4162 case ISD::VP_SREM:
4163 case ISD::VP_UREM:
4164 case ISD::VP_FADD:
4165 case ISD::VP_FSUB:
4166 case ISD::VP_FMUL:
4167 case ISD::VP_FDIV:
4168 case ISD::VP_FREM:
4169 case ISD::VP_FCOPYSIGN:
4170 Res = WidenVecRes_Binary(N);
4171 break;
4172
4173 case ISD::FPOW:
4174 case ISD::FREM:
4175 if (unrollExpandedOp())
4176 break;
4177 // If the target has custom/legal support for the scalar FP intrinsic ops
4178 // (they are probably not destined to become libcalls), then widen those
4179 // like any other binary ops.
4180 [[fallthrough]];
4181
4182 case ISD::FADD:
4183 case ISD::FMUL:
4184 case ISD::FSUB:
4185 case ISD::FDIV:
4186 case ISD::SDIV:
4187 case ISD::UDIV:
4188 case ISD::SREM:
4189 case ISD::UREM:
4190 Res = WidenVecRes_BinaryCanTrap(N);
4191 break;
4192
4193 case ISD::SMULFIX:
4194 case ISD::SMULFIXSAT:
4195 case ISD::UMULFIX:
4196 case ISD::UMULFIXSAT:
4197 // These are binary operations, but with an extra operand that shouldn't
4198 // be widened (the scale).
4199 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
4200 break;
4201
4202 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
4203 case ISD::STRICT_##DAGN:
4204 #include "llvm/IR/ConstrainedOps.def"
4205 Res = WidenVecRes_StrictFP(N);
4206 break;
4207
4208 case ISD::UADDO:
4209 case ISD::SADDO:
4210 case ISD::USUBO:
4211 case ISD::SSUBO:
4212 case ISD::UMULO:
4213 case ISD::SMULO:
4214 Res = WidenVecRes_OverflowOp(N, ResNo);
4215 break;
4216
4217 case ISD::FCOPYSIGN:
4218 Res = WidenVecRes_FCOPYSIGN(N);
4219 break;
4220
4221 case ISD::IS_FPCLASS:
4222 Res = WidenVecRes_IS_FPCLASS(N);
4223 break;
4224
4225 case ISD::FLDEXP:
4226 case ISD::FPOWI:
4227 if (!unrollExpandedOp())
4228 Res = WidenVecRes_ExpOp(N);
4229 break;
4230
4231 case ISD::ANY_EXTEND_VECTOR_INREG:
4232 case ISD::SIGN_EXTEND_VECTOR_INREG:
4233 case ISD::ZERO_EXTEND_VECTOR_INREG:
4234 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
4235 break;
4236
4237 case ISD::ANY_EXTEND:
4238 case ISD::FP_EXTEND:
4239 case ISD::VP_FP_EXTEND:
4240 case ISD::FP_ROUND:
4241 case ISD::VP_FP_ROUND:
4242 case ISD::FP_TO_SINT:
4243 case ISD::VP_FP_TO_SINT:
4244 case ISD::FP_TO_UINT:
4245 case ISD::VP_FP_TO_UINT:
4246 case ISD::SIGN_EXTEND:
4247 case ISD::VP_SIGN_EXTEND:
4248 case ISD::SINT_TO_FP:
4249 case ISD::VP_SINT_TO_FP:
4250 case ISD::VP_TRUNCATE:
4251 case ISD::TRUNCATE:
4252 case ISD::UINT_TO_FP:
4253 case ISD::VP_UINT_TO_FP:
4254 case ISD::ZERO_EXTEND:
4255 case ISD::VP_ZERO_EXTEND:
4256 Res = WidenVecRes_Convert(N);
4257 break;
4258
4259 case ISD::FP_TO_SINT_SAT:
4260 case ISD::FP_TO_UINT_SAT:
4261 Res = WidenVecRes_FP_TO_XINT_SAT(N);
4262 break;
4263
4264 case ISD::LRINT:
4265 case ISD::LLRINT:
4266 Res = WidenVecRes_XRINT(N);
4267 break;
4268
4269 case ISD::FABS:
4270 case ISD::FCEIL:
4271 case ISD::FCOS:
4272 case ISD::FEXP:
4273 case ISD::FEXP2:
4274 case ISD::FEXP10:
4275 case ISD::FFLOOR:
4276 case ISD::FLOG:
4277 case ISD::FLOG10:
4278 case ISD::FLOG2:
4279 case ISD::FNEARBYINT:
4280 case ISD::FRINT:
4281 case ISD::FROUND:
4282 case ISD::FROUNDEVEN:
4283 case ISD::FSIN:
4284 case ISD::FSQRT:
4285 case ISD::FTRUNC:
4286 if (unrollExpandedOp())
4287 break;
4288 // If the target has custom/legal support for the scalar FP intrinsic ops
4289 // (they are probably not destined to become libcalls), then widen those
4290 // like any other unary ops.
4291 [[fallthrough]];
4292
4293 case ISD::ABS:
4294 case ISD::VP_ABS:
4295 case ISD::BITREVERSE:
4296 case ISD::VP_BITREVERSE:
4297 case ISD::BSWAP:
4298 case ISD::VP_BSWAP:
4299 case ISD::CTLZ:
4300 case ISD::VP_CTLZ:
4301 case ISD::CTLZ_ZERO_UNDEF:
4302 case ISD::VP_CTLZ_ZERO_UNDEF:
4303 case ISD::CTPOP:
4304 case ISD::VP_CTPOP:
4305 case ISD::CTTZ:
4306 case ISD::VP_CTTZ:
4307 case ISD::CTTZ_ZERO_UNDEF:
4308 case ISD::VP_CTTZ_ZERO_UNDEF:
4309 case ISD::FNEG: case ISD::VP_FNEG:
4310 case ISD::VP_FABS:
4311 case ISD::VP_SQRT:
4312 case ISD::VP_FCEIL:
4313 case ISD::VP_FFLOOR:
4314 case ISD::VP_FRINT:
4315 case ISD::VP_FNEARBYINT:
4316 case ISD::VP_FROUND:
4317 case ISD::VP_FROUNDEVEN:
4318 case ISD::VP_FROUNDTOZERO:
4319 case ISD::FREEZE:
4320 case ISD::ARITH_FENCE:
4321 case ISD::FCANONICALIZE:
4322 Res = WidenVecRes_Unary(N);
4323 break;
4324 case ISD::FMA: case ISD::VP_FMA:
4325 case ISD::FSHL:
4326 case ISD::VP_FSHL:
4327 case ISD::FSHR:
4328 case ISD::VP_FSHR:
4329 Res = WidenVecRes_Ternary(N);
4330 break;
4331 }
4332
4333 // If Res is null, the sub-method took care of registering the result.
4334 if (Res.getNode())
4335 SetWidenedVector(SDValue(N, ResNo), Res);
4336 }
4337
WidenVecRes_Ternary(SDNode * N)4338 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
4339 // Ternary op widening.
4340 SDLoc dl(N);
4341 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4342 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4343 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4344 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
4345 if (N->getNumOperands() == 3)
4346 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
4347
4348 assert(N->getNumOperands() == 5 && "Unexpected number of operands!");
4349 assert(N->isVPOpcode() && "Expected VP opcode");
4350
4351 SDValue Mask =
4352 GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount());
4353 return DAG.getNode(N->getOpcode(), dl, WidenVT,
4354 {InOp1, InOp2, InOp3, Mask, N->getOperand(4)});
4355 }
4356
WidenVecRes_Binary(SDNode * N)4357 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
4358 // Binary op widening.
4359 SDLoc dl(N);
4360 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4361 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4362 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4363 if (N->getNumOperands() == 2)
4364 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2,
4365 N->getFlags());
4366
4367 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
4368 assert(N->isVPOpcode() && "Expected VP opcode");
4369
4370 SDValue Mask =
4371 GetWidenedMask(N->getOperand(2), WidenVT.getVectorElementCount());
4372 return DAG.getNode(N->getOpcode(), dl, WidenVT,
4373 {InOp1, InOp2, Mask, N->getOperand(3)}, N->getFlags());
4374 }
4375
WidenVecRes_BinaryWithExtraScalarOp(SDNode * N)4376 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
4377 // Binary op widening, but with an extra operand that shouldn't be widened.
4378 SDLoc dl(N);
4379 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4380 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4381 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4382 SDValue InOp3 = N->getOperand(2);
4383 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
4384 N->getFlags());
4385 }
4386
4387 // Given a vector of operations that have been broken up to widen, see
4388 // if we can collect them together into the next widest legal VT. This
4389 // implementation is trap-safe.
CollectOpsToWiden(SelectionDAG & DAG,const TargetLowering & TLI,SmallVectorImpl<SDValue> & ConcatOps,unsigned ConcatEnd,EVT VT,EVT MaxVT,EVT WidenVT)4390 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
4391 SmallVectorImpl<SDValue> &ConcatOps,
4392 unsigned ConcatEnd, EVT VT, EVT MaxVT,
4393 EVT WidenVT) {
4394 // Check to see if we have a single operation with the widen type.
4395 if (ConcatEnd == 1) {
4396 VT = ConcatOps[0].getValueType();
4397 if (VT == WidenVT)
4398 return ConcatOps[0];
4399 }
4400
4401 SDLoc dl(ConcatOps[0]);
4402 EVT WidenEltVT = WidenVT.getVectorElementType();
4403
4404 // while (Some element of ConcatOps is not of type MaxVT) {
4405 // From the end of ConcatOps, collect elements of the same type and put
4406 // them into an op of the next larger supported type
4407 // }
4408 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
4409 int Idx = ConcatEnd - 1;
4410 VT = ConcatOps[Idx--].getValueType();
4411 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
4412 Idx--;
4413
4414 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
4415 EVT NextVT;
4416 do {
4417 NextSize *= 2;
4418 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
4419 } while (!TLI.isTypeLegal(NextVT));
4420
4421 if (!VT.isVector()) {
4422 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
4423 SDValue VecOp = DAG.getUNDEF(NextVT);
4424 unsigned NumToInsert = ConcatEnd - Idx - 1;
4425 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
4426 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
4427 ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
4428 }
4429 ConcatOps[Idx+1] = VecOp;
4430 ConcatEnd = Idx + 2;
4431 } else {
4432 // Vector type, create a CONCAT_VECTORS of type NextVT
4433 SDValue undefVec = DAG.getUNDEF(VT);
4434 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
4435 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
4436 unsigned RealVals = ConcatEnd - Idx - 1;
4437 unsigned SubConcatEnd = 0;
4438 unsigned SubConcatIdx = Idx + 1;
4439 while (SubConcatEnd < RealVals)
4440 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
4441 while (SubConcatEnd < OpsToConcat)
4442 SubConcatOps[SubConcatEnd++] = undefVec;
4443 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
4444 NextVT, SubConcatOps);
4445 ConcatEnd = SubConcatIdx + 1;
4446 }
4447 }
4448
4449 // Check to see if we have a single operation with the widen type.
4450 if (ConcatEnd == 1) {
4451 VT = ConcatOps[0].getValueType();
4452 if (VT == WidenVT)
4453 return ConcatOps[0];
4454 }
4455
4456 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
4457 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
4458 if (NumOps != ConcatEnd ) {
4459 SDValue UndefVal = DAG.getUNDEF(MaxVT);
4460 for (unsigned j = ConcatEnd; j < NumOps; ++j)
4461 ConcatOps[j] = UndefVal;
4462 }
4463 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4464 ArrayRef(ConcatOps.data(), NumOps));
4465 }
4466
WidenVecRes_BinaryCanTrap(SDNode * N)4467 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
4468 // Binary op widening for operations that can trap.
4469 unsigned Opcode = N->getOpcode();
4470 SDLoc dl(N);
4471 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4472 EVT WidenEltVT = WidenVT.getVectorElementType();
4473 EVT VT = WidenVT;
4474 unsigned NumElts = VT.getVectorMinNumElements();
4475 const SDNodeFlags Flags = N->getFlags();
4476 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4477 NumElts = NumElts / 2;
4478 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4479 }
4480
4481 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
4482 // Operation doesn't trap so just widen as normal.
4483 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4484 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4485 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
4486 }
4487
4488 // FIXME: Improve support for scalable vectors.
4489 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
4490
4491 // No legal vector version so unroll the vector operation and then widen.
4492 if (NumElts == 1)
4493 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
4494
4495 // Since the operation can trap, apply operation on the original vector.
4496 EVT MaxVT = VT;
4497 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4498 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4499 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4500
4501 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4502 unsigned ConcatEnd = 0; // Current ConcatOps index.
4503 int Idx = 0; // Current Idx into input vectors.
4504
4505 // NumElts := greatest legal vector size (at most WidenVT)
4506 // while (orig. vector has unhandled elements) {
4507 // take munches of size NumElts from the beginning and add to ConcatOps
4508 // NumElts := next smaller supported vector size or 1
4509 // }
4510 while (CurNumElts != 0) {
4511 while (CurNumElts >= NumElts) {
4512 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
4513 DAG.getVectorIdxConstant(Idx, dl));
4514 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
4515 DAG.getVectorIdxConstant(Idx, dl));
4516 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
4517 Idx += NumElts;
4518 CurNumElts -= NumElts;
4519 }
4520 do {
4521 NumElts = NumElts / 2;
4522 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4523 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4524
4525 if (NumElts == 1) {
4526 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4527 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
4528 InOp1, DAG.getVectorIdxConstant(Idx, dl));
4529 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
4530 InOp2, DAG.getVectorIdxConstant(Idx, dl));
4531 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
4532 EOp1, EOp2, Flags);
4533 }
4534 CurNumElts = 0;
4535 }
4536 }
4537
4538 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4539 }
4540
WidenVecRes_StrictFP(SDNode * N)4541 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
4542 switch (N->getOpcode()) {
4543 case ISD::STRICT_FSETCC:
4544 case ISD::STRICT_FSETCCS:
4545 return WidenVecRes_STRICT_FSETCC(N);
4546 case ISD::STRICT_FP_EXTEND:
4547 case ISD::STRICT_FP_ROUND:
4548 case ISD::STRICT_FP_TO_SINT:
4549 case ISD::STRICT_FP_TO_UINT:
4550 case ISD::STRICT_SINT_TO_FP:
4551 case ISD::STRICT_UINT_TO_FP:
4552 return WidenVecRes_Convert_StrictFP(N);
4553 default:
4554 break;
4555 }
4556
4557 // StrictFP op widening for operations that can trap.
4558 unsigned NumOpers = N->getNumOperands();
4559 unsigned Opcode = N->getOpcode();
4560 SDLoc dl(N);
4561 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4562 EVT WidenEltVT = WidenVT.getVectorElementType();
4563 EVT VT = WidenVT;
4564 unsigned NumElts = VT.getVectorNumElements();
4565 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
4566 NumElts = NumElts / 2;
4567 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4568 }
4569
4570 // No legal vector version so unroll the vector operation and then widen.
4571 if (NumElts == 1)
4572 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
4573
4574 // Since the operation can trap, apply operation on the original vector.
4575 EVT MaxVT = VT;
4576 SmallVector<SDValue, 4> InOps;
4577 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
4578
4579 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
4580 SmallVector<SDValue, 16> Chains;
4581 unsigned ConcatEnd = 0; // Current ConcatOps index.
4582 int Idx = 0; // Current Idx into input vectors.
4583
4584 // The Chain is the first operand.
4585 InOps.push_back(N->getOperand(0));
4586
4587 // Now process the remaining operands.
4588 for (unsigned i = 1; i < NumOpers; ++i) {
4589 SDValue Oper = N->getOperand(i);
4590
4591 EVT OpVT = Oper.getValueType();
4592 if (OpVT.isVector()) {
4593 if (getTypeAction(OpVT) == TargetLowering::TypeWidenVector)
4594 Oper = GetWidenedVector(Oper);
4595 else {
4596 EVT WideOpVT =
4597 EVT::getVectorVT(*DAG.getContext(), OpVT.getVectorElementType(),
4598 WidenVT.getVectorElementCount());
4599 Oper = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4600 DAG.getUNDEF(WideOpVT), Oper,
4601 DAG.getVectorIdxConstant(0, dl));
4602 }
4603 }
4604
4605 InOps.push_back(Oper);
4606 }
4607
4608 // NumElts := greatest legal vector size (at most WidenVT)
4609 // while (orig. vector has unhandled elements) {
4610 // take munches of size NumElts from the beginning and add to ConcatOps
4611 // NumElts := next smaller supported vector size or 1
4612 // }
4613 while (CurNumElts != 0) {
4614 while (CurNumElts >= NumElts) {
4615 SmallVector<SDValue, 4> EOps;
4616
4617 for (unsigned i = 0; i < NumOpers; ++i) {
4618 SDValue Op = InOps[i];
4619
4620 EVT OpVT = Op.getValueType();
4621 if (OpVT.isVector()) {
4622 EVT OpExtractVT =
4623 EVT::getVectorVT(*DAG.getContext(), OpVT.getVectorElementType(),
4624 VT.getVectorElementCount());
4625 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpExtractVT, Op,
4626 DAG.getVectorIdxConstant(Idx, dl));
4627 }
4628
4629 EOps.push_back(Op);
4630 }
4631
4632 EVT OperVT[] = {VT, MVT::Other};
4633 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
4634 ConcatOps[ConcatEnd++] = Oper;
4635 Chains.push_back(Oper.getValue(1));
4636 Idx += NumElts;
4637 CurNumElts -= NumElts;
4638 }
4639 do {
4640 NumElts = NumElts / 2;
4641 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
4642 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
4643
4644 if (NumElts == 1) {
4645 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
4646 SmallVector<SDValue, 4> EOps;
4647
4648 for (unsigned i = 0; i < NumOpers; ++i) {
4649 SDValue Op = InOps[i];
4650
4651 EVT OpVT = Op.getValueType();
4652 if (OpVT.isVector())
4653 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4654 OpVT.getVectorElementType(), Op,
4655 DAG.getVectorIdxConstant(Idx, dl));
4656
4657 EOps.push_back(Op);
4658 }
4659
4660 EVT WidenVT[] = {WidenEltVT, MVT::Other};
4661 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
4662 ConcatOps[ConcatEnd++] = Oper;
4663 Chains.push_back(Oper.getValue(1));
4664 }
4665 CurNumElts = 0;
4666 }
4667 }
4668
4669 // Build a factor node to remember all the Ops that have been created.
4670 SDValue NewChain;
4671 if (Chains.size() == 1)
4672 NewChain = Chains[0];
4673 else
4674 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4675 ReplaceValueWith(SDValue(N, 1), NewChain);
4676
4677 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
4678 }
4679
WidenVecRes_OverflowOp(SDNode * N,unsigned ResNo)4680 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
4681 SDLoc DL(N);
4682 EVT ResVT = N->getValueType(0);
4683 EVT OvVT = N->getValueType(1);
4684 EVT WideResVT, WideOvVT;
4685 SDValue WideLHS, WideRHS;
4686
4687 // TODO: This might result in a widen/split loop.
4688 if (ResNo == 0) {
4689 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
4690 WideOvVT = EVT::getVectorVT(
4691 *DAG.getContext(), OvVT.getVectorElementType(),
4692 WideResVT.getVectorNumElements());
4693
4694 WideLHS = GetWidenedVector(N->getOperand(0));
4695 WideRHS = GetWidenedVector(N->getOperand(1));
4696 } else {
4697 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
4698 WideResVT = EVT::getVectorVT(
4699 *DAG.getContext(), ResVT.getVectorElementType(),
4700 WideOvVT.getVectorNumElements());
4701
4702 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4703 WideLHS = DAG.getNode(
4704 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4705 N->getOperand(0), Zero);
4706 WideRHS = DAG.getNode(
4707 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
4708 N->getOperand(1), Zero);
4709 }
4710
4711 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
4712 SDNode *WideNode = DAG.getNode(
4713 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
4714
4715 // Replace the other vector result not being explicitly widened here.
4716 unsigned OtherNo = 1 - ResNo;
4717 EVT OtherVT = N->getValueType(OtherNo);
4718 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
4719 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
4720 } else {
4721 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
4722 SDValue OtherVal = DAG.getNode(
4723 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
4724 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
4725 }
4726
4727 return SDValue(WideNode, ResNo);
4728 }
4729
WidenVecRes_Convert(SDNode * N)4730 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
4731 LLVMContext &Ctx = *DAG.getContext();
4732 SDValue InOp = N->getOperand(0);
4733 SDLoc DL(N);
4734
4735 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
4736 ElementCount WidenEC = WidenVT.getVectorElementCount();
4737
4738 EVT InVT = InOp.getValueType();
4739
4740 unsigned Opcode = N->getOpcode();
4741 const SDNodeFlags Flags = N->getFlags();
4742
4743 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
4744 // not equal that of WidenVT.
4745 if (N->getOpcode() == ISD::ZERO_EXTEND &&
4746 getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
4747 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
4748 WidenVT.getScalarSizeInBits()) {
4749 InOp = ZExtPromotedInteger(InOp);
4750 InVT = InOp.getValueType();
4751 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
4752 Opcode = ISD::TRUNCATE;
4753 }
4754
4755 EVT InEltVT = InVT.getVectorElementType();
4756 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenEC);
4757 ElementCount InVTEC = InVT.getVectorElementCount();
4758
4759 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4760 InOp = GetWidenedVector(N->getOperand(0));
4761 InVT = InOp.getValueType();
4762 InVTEC = InVT.getVectorElementCount();
4763 if (InVTEC == WidenEC) {
4764 if (N->getNumOperands() == 1)
4765 return DAG.getNode(Opcode, DL, WidenVT, InOp);
4766 if (N->getNumOperands() == 3) {
4767 assert(N->isVPOpcode() && "Expected VP opcode");
4768 SDValue Mask =
4769 GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount());
4770 return DAG.getNode(Opcode, DL, WidenVT, InOp, Mask, N->getOperand(2));
4771 }
4772 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
4773 }
4774 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
4775 // If both input and result vector types are of same width, extend
4776 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
4777 // accepts fewer elements in the result than in the input.
4778 if (Opcode == ISD::ANY_EXTEND)
4779 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4780 if (Opcode == ISD::SIGN_EXTEND)
4781 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4782 if (Opcode == ISD::ZERO_EXTEND)
4783 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
4784 }
4785 }
4786
4787 if (TLI.isTypeLegal(InWidenVT)) {
4788 // Because the result and the input are different vector types, widening
4789 // the result could create a legal type but widening the input might make
4790 // it an illegal type that might lead to repeatedly splitting the input
4791 // and then widening it. To avoid this, we widen the input only if
4792 // it results in a legal type.
4793 if (WidenEC.isKnownMultipleOf(InVTEC.getKnownMinValue())) {
4794 // Widen the input and call convert on the widened input vector.
4795 unsigned NumConcat =
4796 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
4797 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
4798 Ops[0] = InOp;
4799 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
4800 if (N->getNumOperands() == 1)
4801 return DAG.getNode(Opcode, DL, WidenVT, InVec);
4802 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
4803 }
4804
4805 if (InVTEC.isKnownMultipleOf(WidenEC.getKnownMinValue())) {
4806 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
4807 DAG.getVectorIdxConstant(0, DL));
4808 // Extract the input and convert the shorten input vector.
4809 if (N->getNumOperands() == 1)
4810 return DAG.getNode(Opcode, DL, WidenVT, InVal);
4811 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
4812 }
4813 }
4814
4815 // Otherwise unroll into some nasty scalar code and rebuild the vector.
4816 EVT EltVT = WidenVT.getVectorElementType();
4817 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getUNDEF(EltVT));
4818 // Use the original element count so we don't do more scalar opts than
4819 // necessary.
4820 unsigned MinElts = N->getValueType(0).getVectorNumElements();
4821 for (unsigned i=0; i < MinElts; ++i) {
4822 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
4823 DAG.getVectorIdxConstant(i, DL));
4824 if (N->getNumOperands() == 1)
4825 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
4826 else
4827 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
4828 }
4829
4830 return DAG.getBuildVector(WidenVT, DL, Ops);
4831 }
4832
WidenVecRes_FP_TO_XINT_SAT(SDNode * N)4833 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
4834 SDLoc dl(N);
4835 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4836 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4837
4838 SDValue Src = N->getOperand(0);
4839 EVT SrcVT = Src.getValueType();
4840
4841 // Also widen the input.
4842 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
4843 Src = GetWidenedVector(Src);
4844 SrcVT = Src.getValueType();
4845 }
4846
4847 // Input and output not widened to the same size, give up.
4848 if (WidenNumElts != SrcVT.getVectorElementCount())
4849 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
4850
4851 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
4852 }
4853
WidenVecRes_XRINT(SDNode * N)4854 SDValue DAGTypeLegalizer::WidenVecRes_XRINT(SDNode *N) {
4855 SDLoc dl(N);
4856 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4857 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
4858
4859 SDValue Src = N->getOperand(0);
4860 EVT SrcVT = Src.getValueType();
4861
4862 // Also widen the input.
4863 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
4864 Src = GetWidenedVector(Src);
4865 SrcVT = Src.getValueType();
4866 }
4867
4868 // Input and output not widened to the same size, give up.
4869 if (WidenNumElts != SrcVT.getVectorElementCount())
4870 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
4871
4872 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src);
4873 }
4874
WidenVecRes_Convert_StrictFP(SDNode * N)4875 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
4876 SDValue InOp = N->getOperand(1);
4877 SDLoc DL(N);
4878 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4879
4880 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4881 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4882
4883 EVT InVT = InOp.getValueType();
4884 EVT InEltVT = InVT.getVectorElementType();
4885
4886 unsigned Opcode = N->getOpcode();
4887
4888 // FIXME: Optimizations need to be implemented here.
4889
4890 // Otherwise unroll into some nasty scalar code and rebuild the vector.
4891 EVT EltVT = WidenVT.getVectorElementType();
4892 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
4893 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
4894 SmallVector<SDValue, 32> OpChains;
4895 // Use the original element count so we don't do more scalar opts than
4896 // necessary.
4897 unsigned MinElts = N->getValueType(0).getVectorNumElements();
4898 for (unsigned i=0; i < MinElts; ++i) {
4899 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
4900 DAG.getVectorIdxConstant(i, DL));
4901 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
4902 OpChains.push_back(Ops[i].getValue(1));
4903 }
4904 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
4905 ReplaceValueWith(SDValue(N, 1), NewChain);
4906
4907 return DAG.getBuildVector(WidenVT, DL, Ops);
4908 }
4909
WidenVecRes_EXTEND_VECTOR_INREG(SDNode * N)4910 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
4911 unsigned Opcode = N->getOpcode();
4912 SDValue InOp = N->getOperand(0);
4913 SDLoc DL(N);
4914
4915 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4916 EVT WidenSVT = WidenVT.getVectorElementType();
4917 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4918
4919 EVT InVT = InOp.getValueType();
4920 EVT InSVT = InVT.getVectorElementType();
4921 unsigned InVTNumElts = InVT.getVectorNumElements();
4922
4923 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4924 InOp = GetWidenedVector(InOp);
4925 InVT = InOp.getValueType();
4926 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
4927 switch (Opcode) {
4928 case ISD::ANY_EXTEND_VECTOR_INREG:
4929 case ISD::SIGN_EXTEND_VECTOR_INREG:
4930 case ISD::ZERO_EXTEND_VECTOR_INREG:
4931 return DAG.getNode(Opcode, DL, WidenVT, InOp);
4932 }
4933 }
4934 }
4935
4936 // Unroll, extend the scalars and rebuild the vector.
4937 SmallVector<SDValue, 16> Ops;
4938 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
4939 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
4940 DAG.getVectorIdxConstant(i, DL));
4941 switch (Opcode) {
4942 case ISD::ANY_EXTEND_VECTOR_INREG:
4943 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
4944 break;
4945 case ISD::SIGN_EXTEND_VECTOR_INREG:
4946 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
4947 break;
4948 case ISD::ZERO_EXTEND_VECTOR_INREG:
4949 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
4950 break;
4951 default:
4952 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
4953 }
4954 Ops.push_back(Val);
4955 }
4956
4957 while (Ops.size() != WidenNumElts)
4958 Ops.push_back(DAG.getUNDEF(WidenSVT));
4959
4960 return DAG.getBuildVector(WidenVT, DL, Ops);
4961 }
4962
WidenVecRes_FCOPYSIGN(SDNode * N)4963 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
4964 // If this is an FCOPYSIGN with same input types, we can treat it as a
4965 // normal (can trap) binary op.
4966 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
4967 return WidenVecRes_BinaryCanTrap(N);
4968
4969 // If the types are different, fall back to unrolling.
4970 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4971 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
4972 }
4973
WidenVecRes_IS_FPCLASS(SDNode * N)4974 SDValue DAGTypeLegalizer::WidenVecRes_IS_FPCLASS(SDNode *N) {
4975 SDValue FpValue = N->getOperand(0);
4976 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4977 if (getTypeAction(FpValue.getValueType()) != TargetLowering::TypeWidenVector)
4978 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
4979 SDValue Arg = GetWidenedVector(FpValue);
4980 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, {Arg, N->getOperand(1)},
4981 N->getFlags());
4982 }
4983
WidenVecRes_ExpOp(SDNode * N)4984 SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
4985 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4986 SDValue InOp = GetWidenedVector(N->getOperand(0));
4987 SDValue RHS = N->getOperand(1);
4988 SDValue ExpOp = RHS.getValueType().isVector() ? GetWidenedVector(RHS) : RHS;
4989
4990 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ExpOp);
4991 }
4992
WidenVecRes_Unary(SDNode * N)4993 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
4994 // Unary op widening.
4995 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4996 SDValue InOp = GetWidenedVector(N->getOperand(0));
4997 if (N->getNumOperands() == 1)
4998 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, N->getFlags());
4999
5000 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
5001 assert(N->isVPOpcode() && "Expected VP opcode");
5002
5003 SDValue Mask =
5004 GetWidenedMask(N->getOperand(1), WidenVT.getVectorElementCount());
5005 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT,
5006 {InOp, Mask, N->getOperand(2)});
5007 }
5008
WidenVecRes_InregOp(SDNode * N)5009 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
5010 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5011 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
5012 cast<VTSDNode>(N->getOperand(1))->getVT()
5013 .getVectorElementType(),
5014 WidenVT.getVectorNumElements());
5015 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
5016 return DAG.getNode(N->getOpcode(), SDLoc(N),
5017 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
5018 }
5019
WidenVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)5020 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
5021 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
5022 return GetWidenedVector(WidenVec);
5023 }
5024
WidenVecRes_BITCAST(SDNode * N)5025 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
5026 SDValue InOp = N->getOperand(0);
5027 EVT InVT = InOp.getValueType();
5028 EVT VT = N->getValueType(0);
5029 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5030 SDLoc dl(N);
5031
5032 switch (getTypeAction(InVT)) {
5033 case TargetLowering::TypeLegal:
5034 break;
5035 case TargetLowering::TypeScalarizeScalableVector:
5036 report_fatal_error("Scalarization of scalable vectors is not supported.");
5037 case TargetLowering::TypePromoteInteger: {
5038 // If the incoming type is a vector that is being promoted, then
5039 // we know that the elements are arranged differently and that we
5040 // must perform the conversion using a stack slot.
5041 if (InVT.isVector())
5042 break;
5043
5044 // If the InOp is promoted to the same size, convert it. Otherwise,
5045 // fall out of the switch and widen the promoted input.
5046 SDValue NInOp = GetPromotedInteger(InOp);
5047 EVT NInVT = NInOp.getValueType();
5048 if (WidenVT.bitsEq(NInVT)) {
5049 // For big endian targets we need to shift the input integer or the
5050 // interesting bits will end up at the wrong place.
5051 if (DAG.getDataLayout().isBigEndian()) {
5052 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
5053 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
5054 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
5055 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
5056 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
5057 }
5058 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
5059 }
5060 InOp = NInOp;
5061 InVT = NInVT;
5062 break;
5063 }
5064 case TargetLowering::TypeSoftenFloat:
5065 case TargetLowering::TypePromoteFloat:
5066 case TargetLowering::TypeSoftPromoteHalf:
5067 case TargetLowering::TypeExpandInteger:
5068 case TargetLowering::TypeExpandFloat:
5069 case TargetLowering::TypeScalarizeVector:
5070 case TargetLowering::TypeSplitVector:
5071 break;
5072 case TargetLowering::TypeWidenVector:
5073 // If the InOp is widened to the same size, convert it. Otherwise, fall
5074 // out of the switch and widen the widened input.
5075 InOp = GetWidenedVector(InOp);
5076 InVT = InOp.getValueType();
5077 if (WidenVT.bitsEq(InVT))
5078 // The input widens to the same size. Convert to the widen value.
5079 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
5080 break;
5081 }
5082
5083 unsigned WidenSize = WidenVT.getSizeInBits();
5084 unsigned InSize = InVT.getSizeInBits();
5085 unsigned InScalarSize = InVT.getScalarSizeInBits();
5086 // x86mmx is not an acceptable vector element type, so don't try.
5087 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
5088 // Determine new input vector type. The new input vector type will use
5089 // the same element type (if its a vector) or use the input type as a
5090 // vector. It is the same size as the type to widen to.
5091 EVT NewInVT;
5092 unsigned NewNumParts = WidenSize / InSize;
5093 if (InVT.isVector()) {
5094 EVT InEltVT = InVT.getVectorElementType();
5095 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
5096 WidenSize / InEltVT.getSizeInBits());
5097 } else {
5098 // For big endian systems, using the promoted input scalar type
5099 // to produce the scalar_to_vector would put the desired bits into
5100 // the least significant byte(s) of the wider element zero. This
5101 // will mean that the users of the result vector are using incorrect
5102 // bits. Use the original input type instead. Although either input
5103 // type can be used on little endian systems, for consistency we
5104 // use the original type there as well.
5105 EVT OrigInVT = N->getOperand(0).getValueType();
5106 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
5107 NewInVT = EVT::getVectorVT(*DAG.getContext(), OrigInVT, NewNumParts);
5108 }
5109
5110 if (TLI.isTypeLegal(NewInVT)) {
5111 SDValue NewVec;
5112 if (InVT.isVector()) {
5113 // Because the result and the input are different vector types, widening
5114 // the result could create a legal type but widening the input might
5115 // make it an illegal type that might lead to repeatedly splitting the
5116 // input and then widening it. To avoid this, we widen the input only if
5117 // it results in a legal type.
5118 if (WidenSize % InSize == 0) {
5119 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getUNDEF(InVT));
5120 Ops[0] = InOp;
5121
5122 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
5123 } else {
5124 SmallVector<SDValue, 16> Ops;
5125 DAG.ExtractVectorElements(InOp, Ops);
5126 Ops.append(WidenSize / InScalarSize - Ops.size(),
5127 DAG.getUNDEF(InVT.getVectorElementType()));
5128
5129 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops);
5130 }
5131 } else {
5132 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
5133 }
5134 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
5135 }
5136 }
5137
5138 return CreateStackStoreLoad(InOp, WidenVT);
5139 }
5140
WidenVecRes_BUILD_VECTOR(SDNode * N)5141 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
5142 SDLoc dl(N);
5143 // Build a vector with undefined for the new nodes.
5144 EVT VT = N->getValueType(0);
5145
5146 // Integer BUILD_VECTOR operands may be larger than the node's vector element
5147 // type. The UNDEFs need to have the same type as the existing operands.
5148 EVT EltVT = N->getOperand(0).getValueType();
5149 unsigned NumElts = VT.getVectorNumElements();
5150
5151 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5152 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5153
5154 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
5155 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
5156 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
5157
5158 return DAG.getBuildVector(WidenVT, dl, NewOps);
5159 }
5160
WidenVecRes_CONCAT_VECTORS(SDNode * N)5161 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
5162 EVT InVT = N->getOperand(0).getValueType();
5163 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5164 SDLoc dl(N);
5165 unsigned NumOperands = N->getNumOperands();
5166
5167 bool InputWidened = false; // Indicates we need to widen the input.
5168 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
5169 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5170 unsigned NumInElts = InVT.getVectorMinNumElements();
5171 if (WidenNumElts % NumInElts == 0) {
5172 // Add undef vectors to widen to correct length.
5173 unsigned NumConcat = WidenNumElts / NumInElts;
5174 SDValue UndefVal = DAG.getUNDEF(InVT);
5175 SmallVector<SDValue, 16> Ops(NumConcat);
5176 for (unsigned i=0; i < NumOperands; ++i)
5177 Ops[i] = N->getOperand(i);
5178 for (unsigned i = NumOperands; i != NumConcat; ++i)
5179 Ops[i] = UndefVal;
5180 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
5181 }
5182 } else {
5183 InputWidened = true;
5184 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
5185 // The inputs and the result are widen to the same value.
5186 unsigned i;
5187 for (i=1; i < NumOperands; ++i)
5188 if (!N->getOperand(i).isUndef())
5189 break;
5190
5191 if (i == NumOperands)
5192 // Everything but the first operand is an UNDEF so just return the
5193 // widened first operand.
5194 return GetWidenedVector(N->getOperand(0));
5195
5196 if (NumOperands == 2) {
5197 assert(!WidenVT.isScalableVector() &&
5198 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
5199 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5200 unsigned NumInElts = InVT.getVectorNumElements();
5201
5202 // Replace concat of two operands with a shuffle.
5203 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
5204 for (unsigned i = 0; i < NumInElts; ++i) {
5205 MaskOps[i] = i;
5206 MaskOps[i + NumInElts] = i + WidenNumElts;
5207 }
5208 return DAG.getVectorShuffle(WidenVT, dl,
5209 GetWidenedVector(N->getOperand(0)),
5210 GetWidenedVector(N->getOperand(1)),
5211 MaskOps);
5212 }
5213 }
5214 }
5215
5216 assert(!WidenVT.isScalableVector() &&
5217 "Cannot use build vectors to widen CONCAT_VECTOR result");
5218 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5219 unsigned NumInElts = InVT.getVectorNumElements();
5220
5221 // Fall back to use extracts and build vector.
5222 EVT EltVT = WidenVT.getVectorElementType();
5223 SmallVector<SDValue, 16> Ops(WidenNumElts);
5224 unsigned Idx = 0;
5225 for (unsigned i=0; i < NumOperands; ++i) {
5226 SDValue InOp = N->getOperand(i);
5227 if (InputWidened)
5228 InOp = GetWidenedVector(InOp);
5229 for (unsigned j = 0; j < NumInElts; ++j)
5230 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5231 DAG.getVectorIdxConstant(j, dl));
5232 }
5233 SDValue UndefVal = DAG.getUNDEF(EltVT);
5234 for (; Idx < WidenNumElts; ++Idx)
5235 Ops[Idx] = UndefVal;
5236 return DAG.getBuildVector(WidenVT, dl, Ops);
5237 }
5238
WidenVecRes_INSERT_SUBVECTOR(SDNode * N)5239 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
5240 EVT VT = N->getValueType(0);
5241 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5242 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5243 SDValue InOp2 = N->getOperand(1);
5244 SDValue Idx = N->getOperand(2);
5245 SDLoc dl(N);
5246 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WidenVT, InOp1, InOp2, Idx);
5247 }
5248
WidenVecRes_EXTRACT_SUBVECTOR(SDNode * N)5249 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
5250 EVT VT = N->getValueType(0);
5251 EVT EltVT = VT.getVectorElementType();
5252 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5253 SDValue InOp = N->getOperand(0);
5254 SDValue Idx = N->getOperand(1);
5255 SDLoc dl(N);
5256
5257 auto InOpTypeAction = getTypeAction(InOp.getValueType());
5258 if (InOpTypeAction == TargetLowering::TypeWidenVector)
5259 InOp = GetWidenedVector(InOp);
5260
5261 EVT InVT = InOp.getValueType();
5262
5263 // Check if we can just return the input vector after widening.
5264 uint64_t IdxVal = Idx->getAsZExtVal();
5265 if (IdxVal == 0 && InVT == WidenVT)
5266 return InOp;
5267
5268 // Check if we can extract from the vector.
5269 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5270 unsigned InNumElts = InVT.getVectorMinNumElements();
5271 unsigned VTNumElts = VT.getVectorMinNumElements();
5272 assert(IdxVal % VTNumElts == 0 &&
5273 "Expected Idx to be a multiple of subvector minimum vector length");
5274 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
5275 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
5276
5277 if (VT.isScalableVector()) {
5278 // Try to split the operation up into smaller extracts and concat the
5279 // results together, e.g.
5280 // nxv6i64 extract_subvector(nxv12i64, 6)
5281 // <->
5282 // nxv8i64 concat(
5283 // nxv2i64 extract_subvector(nxv16i64, 6)
5284 // nxv2i64 extract_subvector(nxv16i64, 8)
5285 // nxv2i64 extract_subvector(nxv16i64, 10)
5286 // undef)
5287 unsigned GCD = std::gcd(VTNumElts, WidenNumElts);
5288 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
5289 "down type's element count");
5290 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
5291 ElementCount::getScalable(GCD));
5292 // Avoid recursion around e.g. nxv1i8.
5293 if (getTypeAction(PartVT) != TargetLowering::TypeWidenVector) {
5294 SmallVector<SDValue> Parts;
5295 unsigned I = 0;
5296 for (; I < VTNumElts / GCD; ++I)
5297 Parts.push_back(
5298 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, PartVT, InOp,
5299 DAG.getVectorIdxConstant(IdxVal + I * GCD, dl)));
5300 for (; I < WidenNumElts / GCD; ++I)
5301 Parts.push_back(DAG.getUNDEF(PartVT));
5302
5303 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts);
5304 }
5305
5306 report_fatal_error("Don't know how to widen the result of "
5307 "EXTRACT_SUBVECTOR for scalable vectors");
5308 }
5309
5310 // We could try widening the input to the right length but for now, extract
5311 // the original elements, fill the rest with undefs and build a vector.
5312 SmallVector<SDValue, 16> Ops(WidenNumElts);
5313 unsigned i;
5314 for (i = 0; i < VTNumElts; ++i)
5315 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5316 DAG.getVectorIdxConstant(IdxVal + i, dl));
5317
5318 SDValue UndefVal = DAG.getUNDEF(EltVT);
5319 for (; i < WidenNumElts; ++i)
5320 Ops[i] = UndefVal;
5321 return DAG.getBuildVector(WidenVT, dl, Ops);
5322 }
5323
WidenVecRes_AssertZext(SDNode * N)5324 SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
5325 SDValue InOp = ModifyToType(
5326 N->getOperand(0),
5327 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), true);
5328 return DAG.getNode(ISD::AssertZext, SDLoc(N), InOp.getValueType(), InOp,
5329 N->getOperand(1));
5330 }
5331
WidenVecRes_INSERT_VECTOR_ELT(SDNode * N)5332 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
5333 SDValue InOp = GetWidenedVector(N->getOperand(0));
5334 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
5335 InOp.getValueType(), InOp,
5336 N->getOperand(1), N->getOperand(2));
5337 }
5338
WidenVecRes_LOAD(SDNode * N)5339 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
5340 LoadSDNode *LD = cast<LoadSDNode>(N);
5341 ISD::LoadExtType ExtType = LD->getExtensionType();
5342
5343 // A vector must always be stored in memory as-is, i.e. without any padding
5344 // between the elements, since various code depend on it, e.g. in the
5345 // handling of a bitcast of a vector type to int, which may be done with a
5346 // vector store followed by an integer load. A vector that does not have
5347 // elements that are byte-sized must therefore be stored as an integer
5348 // built out of the extracted vector elements.
5349 if (!LD->getMemoryVT().isByteSized()) {
5350 SDValue Value, NewChain;
5351 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
5352 ReplaceValueWith(SDValue(LD, 0), Value);
5353 ReplaceValueWith(SDValue(LD, 1), NewChain);
5354 return SDValue();
5355 }
5356
5357 // Generate a vector-predicated load if it is custom/legal on the target. To
5358 // avoid possible recursion, only do this if the widened mask type is legal.
5359 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
5360 // removed from the IR by the ExpandVectorPredication pass but we're
5361 // reintroducing them here.
5362 EVT LdVT = LD->getMemoryVT();
5363 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), LdVT);
5364 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
5365 WideVT.getVectorElementCount());
5366 if (ExtType == ISD::NON_EXTLOAD &&
5367 TLI.isOperationLegalOrCustom(ISD::VP_LOAD, WideVT) &&
5368 TLI.isTypeLegal(WideMaskVT)) {
5369 SDLoc DL(N);
5370 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT);
5371 SDValue EVL = DAG.getElementCount(DL, TLI.getVPExplicitVectorLengthTy(),
5372 LdVT.getVectorElementCount());
5373 const auto *MMO = LD->getMemOperand();
5374 SDValue NewLoad =
5375 DAG.getLoadVP(WideVT, DL, LD->getChain(), LD->getBasePtr(), Mask, EVL,
5376 MMO->getPointerInfo(), MMO->getAlign(), MMO->getFlags(),
5377 MMO->getAAInfo());
5378
5379 // Modified the chain - switch anything that used the old chain to use
5380 // the new one.
5381 ReplaceValueWith(SDValue(N, 1), NewLoad.getValue(1));
5382
5383 return NewLoad;
5384 }
5385
5386 SDValue Result;
5387 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
5388 if (ExtType != ISD::NON_EXTLOAD)
5389 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
5390 else
5391 Result = GenWidenVectorLoads(LdChain, LD);
5392
5393 if (Result) {
5394 // If we generate a single load, we can use that for the chain. Otherwise,
5395 // build a factor node to remember the multiple loads are independent and
5396 // chain to that.
5397 SDValue NewChain;
5398 if (LdChain.size() == 1)
5399 NewChain = LdChain[0];
5400 else
5401 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
5402
5403 // Modified the chain - switch anything that used the old chain to use
5404 // the new one.
5405 ReplaceValueWith(SDValue(N, 1), NewChain);
5406
5407 return Result;
5408 }
5409
5410 report_fatal_error("Unable to widen vector load");
5411 }
5412
WidenVecRes_VP_LOAD(VPLoadSDNode * N)5413 SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
5414 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5415 SDValue Mask = N->getMask();
5416 SDValue EVL = N->getVectorLength();
5417 ISD::LoadExtType ExtType = N->getExtensionType();
5418 SDLoc dl(N);
5419
5420 // The mask should be widened as well
5421 assert(getTypeAction(Mask.getValueType()) ==
5422 TargetLowering::TypeWidenVector &&
5423 "Unable to widen binary VP op");
5424 Mask = GetWidenedVector(Mask);
5425 assert(Mask.getValueType().getVectorElementCount() ==
5426 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
5427 .getVectorElementCount() &&
5428 "Unable to widen vector load");
5429
5430 SDValue Res =
5431 DAG.getLoadVP(N->getAddressingMode(), ExtType, WidenVT, dl, N->getChain(),
5432 N->getBasePtr(), N->getOffset(), Mask, EVL,
5433 N->getMemoryVT(), N->getMemOperand(), N->isExpandingLoad());
5434 // Legalize the chain result - switch anything that used the old chain to
5435 // use the new one.
5436 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
5437 return Res;
5438 }
5439
WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode * N)5440 SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
5441 SDLoc DL(N);
5442
5443 // The mask should be widened as well
5444 SDValue Mask = N->getMask();
5445 assert(getTypeAction(Mask.getValueType()) ==
5446 TargetLowering::TypeWidenVector &&
5447 "Unable to widen VP strided load");
5448 Mask = GetWidenedVector(Mask);
5449
5450 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5451 assert(Mask.getValueType().getVectorElementCount() ==
5452 WidenVT.getVectorElementCount() &&
5453 "Data and mask vectors should have the same number of elements");
5454
5455 SDValue Res = DAG.getStridedLoadVP(
5456 N->getAddressingMode(), N->getExtensionType(), WidenVT, DL, N->getChain(),
5457 N->getBasePtr(), N->getOffset(), N->getStride(), Mask,
5458 N->getVectorLength(), N->getMemoryVT(), N->getMemOperand(),
5459 N->isExpandingLoad());
5460
5461 // Legalize the chain result - switch anything that used the old chain to
5462 // use the new one.
5463 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
5464 return Res;
5465 }
5466
WidenVecRes_MLOAD(MaskedLoadSDNode * N)5467 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
5468
5469 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
5470 SDValue Mask = N->getMask();
5471 EVT MaskVT = Mask.getValueType();
5472 SDValue PassThru = GetWidenedVector(N->getPassThru());
5473 ISD::LoadExtType ExtType = N->getExtensionType();
5474 SDLoc dl(N);
5475
5476 // The mask should be widened as well
5477 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
5478 MaskVT.getVectorElementType(),
5479 WidenVT.getVectorNumElements());
5480 Mask = ModifyToType(Mask, WideMaskVT, true);
5481
5482 SDValue Res = DAG.getMaskedLoad(
5483 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
5484 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
5485 ExtType, N->isExpandingLoad());
5486 // Legalize the chain result - switch anything that used the old chain to
5487 // use the new one.
5488 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
5489 return Res;
5490 }
5491
WidenVecRes_MGATHER(MaskedGatherSDNode * N)5492 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
5493
5494 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5495 SDValue Mask = N->getMask();
5496 EVT MaskVT = Mask.getValueType();
5497 SDValue PassThru = GetWidenedVector(N->getPassThru());
5498 SDValue Scale = N->getScale();
5499 unsigned NumElts = WideVT.getVectorNumElements();
5500 SDLoc dl(N);
5501
5502 // The mask should be widened as well
5503 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
5504 MaskVT.getVectorElementType(),
5505 WideVT.getVectorNumElements());
5506 Mask = ModifyToType(Mask, WideMaskVT, true);
5507
5508 // Widen the Index operand
5509 SDValue Index = N->getIndex();
5510 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
5511 Index.getValueType().getScalarType(),
5512 NumElts);
5513 Index = ModifyToType(Index, WideIndexVT);
5514 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
5515 Scale };
5516
5517 // Widen the MemoryType
5518 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
5519 N->getMemoryVT().getScalarType(), NumElts);
5520 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
5521 WideMemVT, dl, Ops, N->getMemOperand(),
5522 N->getIndexType(), N->getExtensionType());
5523
5524 // Legalize the chain result - switch anything that used the old chain to
5525 // use the new one.
5526 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
5527 return Res;
5528 }
5529
WidenVecRes_VP_GATHER(VPGatherSDNode * N)5530 SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
5531 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5532 SDValue Mask = N->getMask();
5533 SDValue Scale = N->getScale();
5534 ElementCount WideEC = WideVT.getVectorElementCount();
5535 SDLoc dl(N);
5536
5537 SDValue Index = GetWidenedVector(N->getIndex());
5538 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
5539 N->getMemoryVT().getScalarType(), WideEC);
5540 Mask = GetWidenedMask(Mask, WideEC);
5541
5542 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
5543 Mask, N->getVectorLength()};
5544 SDValue Res = DAG.getGatherVP(DAG.getVTList(WideVT, MVT::Other), WideMemVT,
5545 dl, Ops, N->getMemOperand(), N->getIndexType());
5546
5547 // Legalize the chain result - switch anything that used the old chain to
5548 // use the new one.
5549 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
5550 return Res;
5551 }
5552
WidenVecRes_ScalarOp(SDNode * N)5553 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
5554 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5555 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
5556 }
5557
5558 // Return true is this is a SETCC node or a strict version of it.
isSETCCOp(unsigned Opcode)5559 static inline bool isSETCCOp(unsigned Opcode) {
5560 switch (Opcode) {
5561 case ISD::SETCC:
5562 case ISD::STRICT_FSETCC:
5563 case ISD::STRICT_FSETCCS:
5564 return true;
5565 }
5566 return false;
5567 }
5568
5569 // Return true if this is a node that could have two SETCCs as operands.
isLogicalMaskOp(unsigned Opcode)5570 static inline bool isLogicalMaskOp(unsigned Opcode) {
5571 switch (Opcode) {
5572 case ISD::AND:
5573 case ISD::OR:
5574 case ISD::XOR:
5575 return true;
5576 }
5577 return false;
5578 }
5579
5580 // If N is a SETCC or a strict variant of it, return the type
5581 // of the compare operands.
getSETCCOperandType(SDValue N)5582 static inline EVT getSETCCOperandType(SDValue N) {
5583 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
5584 return N->getOperand(OpNo).getValueType();
5585 }
5586
5587 // This is used just for the assert in convertMask(). Check that this either
5588 // a SETCC or a previously handled SETCC by convertMask().
5589 #ifndef NDEBUG
isSETCCorConvertedSETCC(SDValue N)5590 static inline bool isSETCCorConvertedSETCC(SDValue N) {
5591 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
5592 N = N.getOperand(0);
5593 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
5594 for (unsigned i = 1; i < N->getNumOperands(); ++i)
5595 if (!N->getOperand(i)->isUndef())
5596 return false;
5597 N = N.getOperand(0);
5598 }
5599
5600 if (N.getOpcode() == ISD::TRUNCATE)
5601 N = N.getOperand(0);
5602 else if (N.getOpcode() == ISD::SIGN_EXTEND)
5603 N = N.getOperand(0);
5604
5605 if (isLogicalMaskOp(N.getOpcode()))
5606 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
5607 isSETCCorConvertedSETCC(N.getOperand(1));
5608
5609 return (isSETCCOp(N.getOpcode()) ||
5610 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
5611 }
5612 #endif
5613
5614 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
5615 // to ToMaskVT if needed with vector extension or truncation.
convertMask(SDValue InMask,EVT MaskVT,EVT ToMaskVT)5616 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
5617 EVT ToMaskVT) {
5618 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
5619 // FIXME: This code seems to be too restrictive, we might consider
5620 // generalizing it or dropping it.
5621 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
5622
5623 // Make a new Mask node, with a legal result VT.
5624 SDValue Mask;
5625 SmallVector<SDValue, 4> Ops;
5626 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
5627 Ops.push_back(InMask->getOperand(i));
5628 if (InMask->isStrictFPOpcode()) {
5629 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
5630 { MaskVT, MVT::Other }, Ops);
5631 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
5632 }
5633 else
5634 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
5635
5636 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
5637 // extend or truncate is needed.
5638 LLVMContext &Ctx = *DAG.getContext();
5639 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
5640 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
5641 if (MaskScalarBits < ToMaskScalBits) {
5642 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
5643 MaskVT.getVectorNumElements());
5644 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
5645 } else if (MaskScalarBits > ToMaskScalBits) {
5646 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
5647 MaskVT.getVectorNumElements());
5648 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
5649 }
5650
5651 assert(Mask->getValueType(0).getScalarSizeInBits() ==
5652 ToMaskVT.getScalarSizeInBits() &&
5653 "Mask should have the right element size by now.");
5654
5655 // Adjust Mask to the right number of elements.
5656 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
5657 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
5658 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
5659 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
5660 ZeroIdx);
5661 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
5662 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
5663 EVT SubVT = Mask->getValueType(0);
5664 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
5665 SubOps[0] = Mask;
5666 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
5667 }
5668
5669 assert((Mask->getValueType(0) == ToMaskVT) &&
5670 "A mask of ToMaskVT should have been produced by now.");
5671
5672 return Mask;
5673 }
5674
5675 // This method tries to handle some special cases for the vselect mask
5676 // and if needed adjusting the mask vector type to match that of the VSELECT.
5677 // Without it, many cases end up with scalarization of the SETCC, with many
5678 // unnecessary instructions.
WidenVSELECTMask(SDNode * N)5679 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
5680 LLVMContext &Ctx = *DAG.getContext();
5681 SDValue Cond = N->getOperand(0);
5682
5683 if (N->getOpcode() != ISD::VSELECT)
5684 return SDValue();
5685
5686 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
5687 return SDValue();
5688
5689 // If this is a splitted VSELECT that was previously already handled, do
5690 // nothing.
5691 EVT CondVT = Cond->getValueType(0);
5692 if (CondVT.getScalarSizeInBits() != 1)
5693 return SDValue();
5694
5695 EVT VSelVT = N->getValueType(0);
5696
5697 // This method can't handle scalable vector types.
5698 // FIXME: This support could be added in the future.
5699 if (VSelVT.isScalableVector())
5700 return SDValue();
5701
5702 // Only handle vector types which are a power of 2.
5703 if (!isPowerOf2_64(VSelVT.getSizeInBits()))
5704 return SDValue();
5705
5706 // Don't touch if this will be scalarized.
5707 EVT FinalVT = VSelVT;
5708 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
5709 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
5710
5711 if (FinalVT.getVectorNumElements() == 1)
5712 return SDValue();
5713
5714 // If there is support for an i1 vector mask, don't touch.
5715 if (isSETCCOp(Cond.getOpcode())) {
5716 EVT SetCCOpVT = getSETCCOperandType(Cond);
5717 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
5718 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
5719 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
5720 if (SetCCResVT.getScalarSizeInBits() == 1)
5721 return SDValue();
5722 } else if (CondVT.getScalarType() == MVT::i1) {
5723 // If there is support for an i1 vector mask (or only scalar i1 conditions),
5724 // don't touch.
5725 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
5726 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
5727
5728 if (CondVT.getScalarType() == MVT::i1)
5729 return SDValue();
5730 }
5731
5732 // Widen the vselect result type if needed.
5733 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector)
5734 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
5735
5736 // The mask of the VSELECT should have integer elements.
5737 EVT ToMaskVT = VSelVT;
5738 if (!ToMaskVT.getScalarType().isInteger())
5739 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
5740
5741 SDValue Mask;
5742 if (isSETCCOp(Cond->getOpcode())) {
5743 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
5744 Mask = convertMask(Cond, MaskVT, ToMaskVT);
5745 } else if (isLogicalMaskOp(Cond->getOpcode()) &&
5746 isSETCCOp(Cond->getOperand(0).getOpcode()) &&
5747 isSETCCOp(Cond->getOperand(1).getOpcode())) {
5748 // Cond is (AND/OR/XOR (SETCC, SETCC))
5749 SDValue SETCC0 = Cond->getOperand(0);
5750 SDValue SETCC1 = Cond->getOperand(1);
5751 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
5752 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
5753 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
5754 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
5755 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
5756 EVT MaskVT;
5757 // If the two SETCCs have different VTs, either extend/truncate one of
5758 // them to the other "towards" ToMaskVT, or truncate one and extend the
5759 // other to ToMaskVT.
5760 if (ScalarBits0 != ScalarBits1) {
5761 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
5762 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
5763 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
5764 MaskVT = WideVT;
5765 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
5766 MaskVT = NarrowVT;
5767 else
5768 MaskVT = ToMaskVT;
5769 } else
5770 // If the two SETCCs have the same VT, don't change it.
5771 MaskVT = VT0;
5772
5773 // Make new SETCCs and logical nodes.
5774 SETCC0 = convertMask(SETCC0, VT0, MaskVT);
5775 SETCC1 = convertMask(SETCC1, VT1, MaskVT);
5776 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
5777
5778 // Convert the logical op for VSELECT if needed.
5779 Mask = convertMask(Cond, MaskVT, ToMaskVT);
5780 } else
5781 return SDValue();
5782
5783 return Mask;
5784 }
5785
WidenVecRes_Select(SDNode * N)5786 SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
5787 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5788 ElementCount WidenEC = WidenVT.getVectorElementCount();
5789
5790 SDValue Cond1 = N->getOperand(0);
5791 EVT CondVT = Cond1.getValueType();
5792 unsigned Opcode = N->getOpcode();
5793 if (CondVT.isVector()) {
5794 if (SDValue WideCond = WidenVSELECTMask(N)) {
5795 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
5796 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
5797 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
5798 return DAG.getNode(Opcode, SDLoc(N), WidenVT, WideCond, InOp1, InOp2);
5799 }
5800
5801 EVT CondEltVT = CondVT.getVectorElementType();
5802 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenEC);
5803 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
5804 Cond1 = GetWidenedVector(Cond1);
5805
5806 // If we have to split the condition there is no point in widening the
5807 // select. This would result in an cycle of widening the select ->
5808 // widening the condition operand -> splitting the condition operand ->
5809 // splitting the select -> widening the select. Instead split this select
5810 // further and widen the resulting type.
5811 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
5812 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
5813 SDValue Res = ModifyToType(SplitSelect, WidenVT);
5814 return Res;
5815 }
5816
5817 if (Cond1.getValueType() != CondWidenVT)
5818 Cond1 = ModifyToType(Cond1, CondWidenVT);
5819 }
5820
5821 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
5822 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
5823 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
5824 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
5825 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2,
5826 N->getOperand(3));
5827 return DAG.getNode(Opcode, SDLoc(N), WidenVT, Cond1, InOp1, InOp2);
5828 }
5829
WidenVecRes_SELECT_CC(SDNode * N)5830 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
5831 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
5832 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
5833 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
5834 InOp1.getValueType(), N->getOperand(0),
5835 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
5836 }
5837
WidenVecRes_UNDEF(SDNode * N)5838 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
5839 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5840 return DAG.getUNDEF(WidenVT);
5841 }
5842
WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N)5843 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
5844 EVT VT = N->getValueType(0);
5845 SDLoc dl(N);
5846
5847 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5848 unsigned NumElts = VT.getVectorNumElements();
5849 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5850
5851 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
5852 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
5853
5854 // Adjust mask based on new input vector length.
5855 SmallVector<int, 16> NewMask;
5856 for (unsigned i = 0; i != NumElts; ++i) {
5857 int Idx = N->getMaskElt(i);
5858 if (Idx < (int)NumElts)
5859 NewMask.push_back(Idx);
5860 else
5861 NewMask.push_back(Idx - NumElts + WidenNumElts);
5862 }
5863 for (unsigned i = NumElts; i != WidenNumElts; ++i)
5864 NewMask.push_back(-1);
5865 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
5866 }
5867
WidenVecRes_VECTOR_REVERSE(SDNode * N)5868 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
5869 EVT VT = N->getValueType(0);
5870 EVT EltVT = VT.getVectorElementType();
5871 SDLoc dl(N);
5872
5873 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5874 SDValue OpValue = GetWidenedVector(N->getOperand(0));
5875 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
5876
5877 SDValue ReverseVal = DAG.getNode(ISD::VECTOR_REVERSE, dl, WidenVT, OpValue);
5878 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
5879 unsigned VTNumElts = VT.getVectorMinNumElements();
5880 unsigned IdxVal = WidenNumElts - VTNumElts;
5881
5882 if (VT.isScalableVector()) {
5883 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
5884 // results together, e.g.(nxv6i64 -> nxv8i64)
5885 // nxv8i64 vector_reverse
5886 // <->
5887 // nxv8i64 concat(
5888 // nxv2i64 extract_subvector(nxv8i64, 2)
5889 // nxv2i64 extract_subvector(nxv8i64, 4)
5890 // nxv2i64 extract_subvector(nxv8i64, 6)
5891 // nxv2i64 undef)
5892
5893 unsigned GCD = std::gcd(VTNumElts, WidenNumElts);
5894 EVT PartVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
5895 ElementCount::getScalable(GCD));
5896 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
5897 "down type's element count");
5898 SmallVector<SDValue> Parts;
5899 unsigned i = 0;
5900 for (; i < VTNumElts / GCD; ++i)
5901 Parts.push_back(
5902 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, PartVT, ReverseVal,
5903 DAG.getVectorIdxConstant(IdxVal + i * GCD, dl)));
5904 for (; i < WidenNumElts / GCD; ++i)
5905 Parts.push_back(DAG.getUNDEF(PartVT));
5906
5907 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Parts);
5908 }
5909
5910 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
5911 // fixed-vectors.
5912 SmallVector<int, 16> Mask;
5913 for (unsigned i = 0; i != VTNumElts; ++i) {
5914 Mask.push_back(IdxVal + i);
5915 }
5916 for (unsigned i = VTNumElts; i != WidenNumElts; ++i)
5917 Mask.push_back(-1);
5918
5919 return DAG.getVectorShuffle(WidenVT, dl, ReverseVal, DAG.getUNDEF(WidenVT),
5920 Mask);
5921 }
5922
WidenVecRes_SETCC(SDNode * N)5923 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
5924 assert(N->getValueType(0).isVector() &&
5925 N->getOperand(0).getValueType().isVector() &&
5926 "Operands must be vectors");
5927 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5928 ElementCount WidenEC = WidenVT.getVectorElementCount();
5929
5930 SDValue InOp1 = N->getOperand(0);
5931 EVT InVT = InOp1.getValueType();
5932 assert(InVT.isVector() && "can not widen non-vector type");
5933 EVT WidenInVT =
5934 EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenEC);
5935
5936 // The input and output types often differ here, and it could be that while
5937 // we'd prefer to widen the result type, the input operands have been split.
5938 // In this case, we also need to split the result of this node as well.
5939 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
5940 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
5941 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
5942 return Res;
5943 }
5944
5945 // If the inputs also widen, handle them directly. Otherwise widen by hand.
5946 SDValue InOp2 = N->getOperand(1);
5947 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
5948 InOp1 = GetWidenedVector(InOp1);
5949 InOp2 = GetWidenedVector(InOp2);
5950 } else {
5951 InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
5952 InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
5953 }
5954
5955 // Assume that the input and output will be widen appropriately. If not,
5956 // we will have to unroll it at some point.
5957 assert(InOp1.getValueType() == WidenInVT &&
5958 InOp2.getValueType() == WidenInVT &&
5959 "Input not widened to expected type!");
5960 (void)WidenInVT;
5961 if (N->getOpcode() == ISD::VP_SETCC) {
5962 SDValue Mask =
5963 GetWidenedMask(N->getOperand(3), WidenVT.getVectorElementCount());
5964 return DAG.getNode(ISD::VP_SETCC, SDLoc(N), WidenVT, InOp1, InOp2,
5965 N->getOperand(2), Mask, N->getOperand(4));
5966 }
5967 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, InOp1, InOp2,
5968 N->getOperand(2));
5969 }
5970
WidenVecRes_STRICT_FSETCC(SDNode * N)5971 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
5972 assert(N->getValueType(0).isVector() &&
5973 N->getOperand(1).getValueType().isVector() &&
5974 "Operands must be vectors");
5975 EVT VT = N->getValueType(0);
5976 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5977 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5978 unsigned NumElts = VT.getVectorNumElements();
5979 EVT EltVT = VT.getVectorElementType();
5980
5981 SDLoc dl(N);
5982 SDValue Chain = N->getOperand(0);
5983 SDValue LHS = N->getOperand(1);
5984 SDValue RHS = N->getOperand(2);
5985 SDValue CC = N->getOperand(3);
5986 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
5987
5988 // Fully unroll and reassemble.
5989 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT));
5990 SmallVector<SDValue, 8> Chains(NumElts);
5991 for (unsigned i = 0; i != NumElts; ++i) {
5992 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
5993 DAG.getVectorIdxConstant(i, dl));
5994 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
5995 DAG.getVectorIdxConstant(i, dl));
5996
5997 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
5998 {Chain, LHSElem, RHSElem, CC});
5999 Chains[i] = Scalars[i].getValue(1);
6000 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
6001 DAG.getBoolConstant(true, dl, EltVT, VT),
6002 DAG.getBoolConstant(false, dl, EltVT, VT));
6003 }
6004
6005 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
6006 ReplaceValueWith(SDValue(N, 1), NewChain);
6007
6008 return DAG.getBuildVector(WidenVT, dl, Scalars);
6009 }
6010
6011 //===----------------------------------------------------------------------===//
6012 // Widen Vector Operand
6013 //===----------------------------------------------------------------------===//
WidenVectorOperand(SDNode * N,unsigned OpNo)6014 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
6015 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
6016 SDValue Res = SDValue();
6017
6018 // See if the target wants to custom widen this node.
6019 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
6020 return false;
6021
6022 switch (N->getOpcode()) {
6023 default:
6024 #ifndef NDEBUG
6025 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
6026 N->dump(&DAG);
6027 dbgs() << "\n";
6028 #endif
6029 report_fatal_error("Do not know how to widen this operator's operand!");
6030
6031 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
6032 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
6033 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
6034 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
6035 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
6036 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
6037 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
6038 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
6039 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
6040 break;
6041 case ISD::ANY_EXTEND_VECTOR_INREG:
6042 case ISD::SIGN_EXTEND_VECTOR_INREG:
6043 case ISD::ZERO_EXTEND_VECTOR_INREG:
6044 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
6045 break;
6046 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
6047 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
6048 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
6049 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
6050 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
6051 case ISD::STRICT_FSETCC:
6052 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
6053 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
6054 case ISD::FLDEXP:
6055 case ISD::FCOPYSIGN:
6056 case ISD::LRINT:
6057 case ISD::LLRINT:
6058 Res = WidenVecOp_UnrollVectorOp(N);
6059 break;
6060 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
6061
6062 case ISD::ANY_EXTEND:
6063 case ISD::SIGN_EXTEND:
6064 case ISD::ZERO_EXTEND:
6065 Res = WidenVecOp_EXTEND(N);
6066 break;
6067
6068 case ISD::FP_EXTEND:
6069 case ISD::STRICT_FP_EXTEND:
6070 case ISD::FP_ROUND:
6071 case ISD::STRICT_FP_ROUND:
6072 case ISD::FP_TO_SINT:
6073 case ISD::STRICT_FP_TO_SINT:
6074 case ISD::FP_TO_UINT:
6075 case ISD::STRICT_FP_TO_UINT:
6076 case ISD::SINT_TO_FP:
6077 case ISD::STRICT_SINT_TO_FP:
6078 case ISD::UINT_TO_FP:
6079 case ISD::STRICT_UINT_TO_FP:
6080 case ISD::TRUNCATE:
6081 Res = WidenVecOp_Convert(N);
6082 break;
6083
6084 case ISD::FP_TO_SINT_SAT:
6085 case ISD::FP_TO_UINT_SAT:
6086 Res = WidenVecOp_FP_TO_XINT_SAT(N);
6087 break;
6088
6089 case ISD::VECREDUCE_FADD:
6090 case ISD::VECREDUCE_FMUL:
6091 case ISD::VECREDUCE_ADD:
6092 case ISD::VECREDUCE_MUL:
6093 case ISD::VECREDUCE_AND:
6094 case ISD::VECREDUCE_OR:
6095 case ISD::VECREDUCE_XOR:
6096 case ISD::VECREDUCE_SMAX:
6097 case ISD::VECREDUCE_SMIN:
6098 case ISD::VECREDUCE_UMAX:
6099 case ISD::VECREDUCE_UMIN:
6100 case ISD::VECREDUCE_FMAX:
6101 case ISD::VECREDUCE_FMIN:
6102 case ISD::VECREDUCE_FMAXIMUM:
6103 case ISD::VECREDUCE_FMINIMUM:
6104 Res = WidenVecOp_VECREDUCE(N);
6105 break;
6106 case ISD::VECREDUCE_SEQ_FADD:
6107 case ISD::VECREDUCE_SEQ_FMUL:
6108 Res = WidenVecOp_VECREDUCE_SEQ(N);
6109 break;
6110 case ISD::VP_REDUCE_FADD:
6111 case ISD::VP_REDUCE_SEQ_FADD:
6112 case ISD::VP_REDUCE_FMUL:
6113 case ISD::VP_REDUCE_SEQ_FMUL:
6114 case ISD::VP_REDUCE_ADD:
6115 case ISD::VP_REDUCE_MUL:
6116 case ISD::VP_REDUCE_AND:
6117 case ISD::VP_REDUCE_OR:
6118 case ISD::VP_REDUCE_XOR:
6119 case ISD::VP_REDUCE_SMAX:
6120 case ISD::VP_REDUCE_SMIN:
6121 case ISD::VP_REDUCE_UMAX:
6122 case ISD::VP_REDUCE_UMIN:
6123 case ISD::VP_REDUCE_FMAX:
6124 case ISD::VP_REDUCE_FMIN:
6125 Res = WidenVecOp_VP_REDUCE(N);
6126 break;
6127 }
6128
6129 // If Res is null, the sub-method took care of registering the result.
6130 if (!Res.getNode()) return false;
6131
6132 // If the result is N, the sub-method updated N in place. Tell the legalizer
6133 // core about this.
6134 if (Res.getNode() == N)
6135 return true;
6136
6137
6138 if (N->isStrictFPOpcode())
6139 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
6140 "Invalid operand expansion");
6141 else
6142 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
6143 "Invalid operand expansion");
6144
6145 ReplaceValueWith(SDValue(N, 0), Res);
6146 return false;
6147 }
6148
WidenVecOp_EXTEND(SDNode * N)6149 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
6150 SDLoc DL(N);
6151 EVT VT = N->getValueType(0);
6152
6153 SDValue InOp = N->getOperand(0);
6154 assert(getTypeAction(InOp.getValueType()) ==
6155 TargetLowering::TypeWidenVector &&
6156 "Unexpected type action");
6157 InOp = GetWidenedVector(InOp);
6158 assert(VT.getVectorNumElements() <
6159 InOp.getValueType().getVectorNumElements() &&
6160 "Input wasn't widened!");
6161
6162 // We may need to further widen the operand until it has the same total
6163 // vector size as the result.
6164 EVT InVT = InOp.getValueType();
6165 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
6166 EVT InEltVT = InVT.getVectorElementType();
6167 for (EVT FixedVT : MVT::vector_valuetypes()) {
6168 EVT FixedEltVT = FixedVT.getVectorElementType();
6169 if (TLI.isTypeLegal(FixedVT) &&
6170 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
6171 FixedEltVT == InEltVT) {
6172 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
6173 "Not enough elements in the fixed type for the operand!");
6174 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
6175 "We can't have the same type as we started with!");
6176 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
6177 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT,
6178 DAG.getUNDEF(FixedVT), InOp,
6179 DAG.getVectorIdxConstant(0, DL));
6180 else
6181 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
6182 DAG.getVectorIdxConstant(0, DL));
6183 break;
6184 }
6185 }
6186 InVT = InOp.getValueType();
6187 if (InVT.getSizeInBits() != VT.getSizeInBits())
6188 // We couldn't find a legal vector type that was a widening of the input
6189 // and could be extended in-register to the result type, so we have to
6190 // scalarize.
6191 return WidenVecOp_Convert(N);
6192 }
6193
6194 // Use special DAG nodes to represent the operation of extending the
6195 // low lanes.
6196 switch (N->getOpcode()) {
6197 default:
6198 llvm_unreachable("Extend legalization on extend operation!");
6199 case ISD::ANY_EXTEND:
6200 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
6201 case ISD::SIGN_EXTEND:
6202 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
6203 case ISD::ZERO_EXTEND:
6204 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
6205 }
6206 }
6207
WidenVecOp_UnrollVectorOp(SDNode * N)6208 SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
6209 // The result (and first input) is legal, but the second input is illegal.
6210 // We can't do much to fix that, so just unroll and let the extracts off of
6211 // the second input be widened as needed later.
6212 return DAG.UnrollVectorOp(N);
6213 }
6214
WidenVecOp_IS_FPCLASS(SDNode * N)6215 SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
6216 SDLoc DL(N);
6217 EVT ResultVT = N->getValueType(0);
6218 SDValue Test = N->getOperand(1);
6219 SDValue WideArg = GetWidenedVector(N->getOperand(0));
6220
6221 // Process this node similarly to SETCC.
6222 EVT WideResultVT = getSetCCResultType(WideArg.getValueType());
6223 if (ResultVT.getScalarType() == MVT::i1)
6224 WideResultVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6225 WideResultVT.getVectorNumElements());
6226
6227 SDValue WideNode = DAG.getNode(ISD::IS_FPCLASS, DL, WideResultVT,
6228 {WideArg, Test}, N->getFlags());
6229
6230 // Extract the needed results from the result vector.
6231 EVT ResVT =
6232 EVT::getVectorVT(*DAG.getContext(), WideResultVT.getVectorElementType(),
6233 ResultVT.getVectorNumElements());
6234 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, WideNode,
6235 DAG.getVectorIdxConstant(0, DL));
6236
6237 EVT OpVT = N->getOperand(0).getValueType();
6238 ISD::NodeType ExtendCode =
6239 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
6240 return DAG.getNode(ExtendCode, DL, ResultVT, CC);
6241 }
6242
WidenVecOp_Convert(SDNode * N)6243 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
6244 // Since the result is legal and the input is illegal.
6245 EVT VT = N->getValueType(0);
6246 EVT EltVT = VT.getVectorElementType();
6247 SDLoc dl(N);
6248 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
6249 assert(getTypeAction(InOp.getValueType()) ==
6250 TargetLowering::TypeWidenVector &&
6251 "Unexpected type action");
6252 InOp = GetWidenedVector(InOp);
6253 EVT InVT = InOp.getValueType();
6254 unsigned Opcode = N->getOpcode();
6255
6256 // See if a widened result type would be legal, if so widen the node.
6257 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
6258 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
6259 InVT.getVectorElementCount());
6260 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
6261 SDValue Res;
6262 if (N->isStrictFPOpcode()) {
6263 if (Opcode == ISD::STRICT_FP_ROUND)
6264 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
6265 { N->getOperand(0), InOp, N->getOperand(2) });
6266 else
6267 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
6268 { N->getOperand(0), InOp });
6269 // Legalize the chain result - switch anything that used the old chain to
6270 // use the new one.
6271 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
6272 } else {
6273 if (Opcode == ISD::FP_ROUND)
6274 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1));
6275 else
6276 Res = DAG.getNode(Opcode, dl, WideVT, InOp);
6277 }
6278 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
6279 DAG.getVectorIdxConstant(0, dl));
6280 }
6281
6282 EVT InEltVT = InVT.getVectorElementType();
6283
6284 // Unroll the convert into some scalar code and create a nasty build vector.
6285 unsigned NumElts = VT.getVectorNumElements();
6286 SmallVector<SDValue, 16> Ops(NumElts);
6287 if (N->isStrictFPOpcode()) {
6288 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
6289 SmallVector<SDValue, 32> OpChains;
6290 for (unsigned i=0; i < NumElts; ++i) {
6291 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
6292 DAG.getVectorIdxConstant(i, dl));
6293 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
6294 OpChains.push_back(Ops[i].getValue(1));
6295 }
6296 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
6297 ReplaceValueWith(SDValue(N, 1), NewChain);
6298 } else {
6299 for (unsigned i = 0; i < NumElts; ++i)
6300 Ops[i] = DAG.getNode(Opcode, dl, EltVT,
6301 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT,
6302 InOp, DAG.getVectorIdxConstant(i, dl)));
6303 }
6304
6305 return DAG.getBuildVector(VT, dl, Ops);
6306 }
6307
WidenVecOp_FP_TO_XINT_SAT(SDNode * N)6308 SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
6309 EVT DstVT = N->getValueType(0);
6310 SDValue Src = GetWidenedVector(N->getOperand(0));
6311 EVT SrcVT = Src.getValueType();
6312 ElementCount WideNumElts = SrcVT.getVectorElementCount();
6313 SDLoc dl(N);
6314
6315 // See if a widened result type would be legal, if so widen the node.
6316 EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(),
6317 DstVT.getVectorElementType(), WideNumElts);
6318 if (TLI.isTypeLegal(WideDstVT)) {
6319 SDValue Res =
6320 DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1));
6321 return DAG.getNode(
6322 ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res,
6323 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
6324 }
6325
6326 // Give up and unroll.
6327 return DAG.UnrollVectorOp(N);
6328 }
6329
WidenVecOp_BITCAST(SDNode * N)6330 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
6331 EVT VT = N->getValueType(0);
6332 SDValue InOp = GetWidenedVector(N->getOperand(0));
6333 EVT InWidenVT = InOp.getValueType();
6334 SDLoc dl(N);
6335
6336 // Check if we can convert between two legal vector types and extract.
6337 TypeSize InWidenSize = InWidenVT.getSizeInBits();
6338 TypeSize Size = VT.getSizeInBits();
6339 // x86mmx is not an acceptable vector element type, so don't try.
6340 if (!VT.isVector() && VT != MVT::x86mmx &&
6341 InWidenSize.hasKnownScalarFactor(Size)) {
6342 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(Size);
6343 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
6344 if (TLI.isTypeLegal(NewVT)) {
6345 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
6346 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
6347 DAG.getVectorIdxConstant(0, dl));
6348 }
6349 }
6350
6351 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
6352 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
6353 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
6354 // having to copy via memory.
6355 if (VT.isVector()) {
6356 EVT EltVT = VT.getVectorElementType();
6357 unsigned EltSize = EltVT.getFixedSizeInBits();
6358 if (InWidenSize.isKnownMultipleOf(EltSize)) {
6359 ElementCount NewNumElts =
6360 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
6361 .divideCoefficientBy(EltSize);
6362 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
6363 if (TLI.isTypeLegal(NewVT)) {
6364 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
6365 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
6366 DAG.getVectorIdxConstant(0, dl));
6367 }
6368 }
6369 }
6370
6371 return CreateStackStoreLoad(InOp, VT);
6372 }
6373
WidenVecOp_CONCAT_VECTORS(SDNode * N)6374 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
6375 EVT VT = N->getValueType(0);
6376 EVT EltVT = VT.getVectorElementType();
6377 EVT InVT = N->getOperand(0).getValueType();
6378 SDLoc dl(N);
6379
6380 // If the widen width for this operand is the same as the width of the concat
6381 // and all but the first operand is undef, just use the widened operand.
6382 unsigned NumOperands = N->getNumOperands();
6383 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
6384 unsigned i;
6385 for (i = 1; i < NumOperands; ++i)
6386 if (!N->getOperand(i).isUndef())
6387 break;
6388
6389 if (i == NumOperands)
6390 return GetWidenedVector(N->getOperand(0));
6391 }
6392
6393 // Otherwise, fall back to a nasty build vector.
6394 unsigned NumElts = VT.getVectorNumElements();
6395 SmallVector<SDValue, 16> Ops(NumElts);
6396
6397 unsigned NumInElts = InVT.getVectorNumElements();
6398
6399 unsigned Idx = 0;
6400 for (unsigned i=0; i < NumOperands; ++i) {
6401 SDValue InOp = N->getOperand(i);
6402 assert(getTypeAction(InOp.getValueType()) ==
6403 TargetLowering::TypeWidenVector &&
6404 "Unexpected type action");
6405 InOp = GetWidenedVector(InOp);
6406 for (unsigned j = 0; j < NumInElts; ++j)
6407 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
6408 DAG.getVectorIdxConstant(j, dl));
6409 }
6410 return DAG.getBuildVector(VT, dl, Ops);
6411 }
6412
WidenVecOp_INSERT_SUBVECTOR(SDNode * N)6413 SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
6414 EVT VT = N->getValueType(0);
6415 SDValue SubVec = N->getOperand(1);
6416 SDValue InVec = N->getOperand(0);
6417
6418 if (getTypeAction(SubVec.getValueType()) == TargetLowering::TypeWidenVector)
6419 SubVec = GetWidenedVector(SubVec);
6420
6421 EVT SubVT = SubVec.getValueType();
6422
6423 // Whether or not all the elements of the widened SubVec will be inserted into
6424 // valid indices of VT.
6425 bool IndicesValid = false;
6426 // If we statically know that VT can fit SubVT, the indices are valid.
6427 if (VT.knownBitsGE(SubVT))
6428 IndicesValid = true;
6429 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
6430 // Otherwise, if we're inserting a fixed vector into a scalable vector and
6431 // we know the minimum vscale we can work out if it's valid ourselves.
6432 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
6433 Attribute::VScaleRange);
6434 if (Attr.isValid()) {
6435 unsigned VScaleMin = Attr.getVScaleRangeMin();
6436 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
6437 SubVT.getFixedSizeInBits())
6438 IndicesValid = true;
6439 }
6440 }
6441
6442 // We need to make sure that the indices are still valid, otherwise we might
6443 // widen what was previously well-defined to something undefined.
6444 if (IndicesValid && InVec.isUndef() && N->getConstantOperandVal(2) == 0)
6445 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, InVec, SubVec,
6446 N->getOperand(2));
6447
6448 report_fatal_error("Don't know how to widen the operands for "
6449 "INSERT_SUBVECTOR");
6450 }
6451
WidenVecOp_EXTRACT_SUBVECTOR(SDNode * N)6452 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
6453 SDValue InOp = GetWidenedVector(N->getOperand(0));
6454 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
6455 N->getValueType(0), InOp, N->getOperand(1));
6456 }
6457
WidenVecOp_EXTRACT_VECTOR_ELT(SDNode * N)6458 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
6459 SDValue InOp = GetWidenedVector(N->getOperand(0));
6460 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
6461 N->getValueType(0), InOp, N->getOperand(1));
6462 }
6463
WidenVecOp_EXTEND_VECTOR_INREG(SDNode * N)6464 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
6465 SDValue InOp = GetWidenedVector(N->getOperand(0));
6466 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), InOp);
6467 }
6468
WidenVecOp_STORE(SDNode * N)6469 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
6470 // We have to widen the value, but we want only to store the original
6471 // vector type.
6472 StoreSDNode *ST = cast<StoreSDNode>(N);
6473
6474 if (!ST->getMemoryVT().getScalarType().isByteSized())
6475 return TLI.scalarizeVectorStore(ST, DAG);
6476
6477 if (ST->isTruncatingStore())
6478 return TLI.scalarizeVectorStore(ST, DAG);
6479
6480 // Generate a vector-predicated store if it is custom/legal on the target.
6481 // To avoid possible recursion, only do this if the widened mask type is
6482 // legal.
6483 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
6484 // removed from the IR by the ExpandVectorPredication pass but we're
6485 // reintroducing them here.
6486 SDValue StVal = ST->getValue();
6487 EVT StVT = StVal.getValueType();
6488 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StVT);
6489 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6490 WideVT.getVectorElementCount());
6491
6492 if (TLI.isOperationLegalOrCustom(ISD::VP_STORE, WideVT) &&
6493 TLI.isTypeLegal(WideMaskVT)) {
6494 // Widen the value.
6495 SDLoc DL(N);
6496 StVal = GetWidenedVector(StVal);
6497 SDValue Mask = DAG.getAllOnesConstant(DL, WideMaskVT);
6498 SDValue EVL = DAG.getElementCount(DL, TLI.getVPExplicitVectorLengthTy(),
6499 StVT.getVectorElementCount());
6500 return DAG.getStoreVP(ST->getChain(), DL, StVal, ST->getBasePtr(),
6501 DAG.getUNDEF(ST->getBasePtr().getValueType()), Mask,
6502 EVL, StVT, ST->getMemOperand(),
6503 ST->getAddressingMode());
6504 }
6505
6506 SmallVector<SDValue, 16> StChain;
6507 if (GenWidenVectorStores(StChain, ST)) {
6508 if (StChain.size() == 1)
6509 return StChain[0];
6510
6511 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
6512 }
6513
6514 report_fatal_error("Unable to widen vector store");
6515 }
6516
WidenVecOp_VP_STORE(SDNode * N,unsigned OpNo)6517 SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
6518 assert((OpNo == 1 || OpNo == 3) &&
6519 "Can widen only data or mask operand of vp_store");
6520 VPStoreSDNode *ST = cast<VPStoreSDNode>(N);
6521 SDValue Mask = ST->getMask();
6522 SDValue StVal = ST->getValue();
6523 SDLoc dl(N);
6524
6525 if (OpNo == 1) {
6526 // Widen the value.
6527 StVal = GetWidenedVector(StVal);
6528
6529 // We only handle the case where the mask needs widening to an
6530 // identically-sized type as the vector inputs.
6531 assert(getTypeAction(Mask.getValueType()) ==
6532 TargetLowering::TypeWidenVector &&
6533 "Unable to widen VP store");
6534 Mask = GetWidenedVector(Mask);
6535 } else {
6536 Mask = GetWidenedVector(Mask);
6537
6538 // We only handle the case where the stored value needs widening to an
6539 // identically-sized type as the mask.
6540 assert(getTypeAction(StVal.getValueType()) ==
6541 TargetLowering::TypeWidenVector &&
6542 "Unable to widen VP store");
6543 StVal = GetWidenedVector(StVal);
6544 }
6545
6546 assert(Mask.getValueType().getVectorElementCount() ==
6547 StVal.getValueType().getVectorElementCount() &&
6548 "Mask and data vectors should have the same number of elements");
6549 return DAG.getStoreVP(ST->getChain(), dl, StVal, ST->getBasePtr(),
6550 ST->getOffset(), Mask, ST->getVectorLength(),
6551 ST->getMemoryVT(), ST->getMemOperand(),
6552 ST->getAddressingMode(), ST->isTruncatingStore(),
6553 ST->isCompressingStore());
6554 }
6555
WidenVecOp_VP_STRIDED_STORE(SDNode * N,unsigned OpNo)6556 SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
6557 unsigned OpNo) {
6558 assert((OpNo == 1 || OpNo == 4) &&
6559 "Can widen only data or mask operand of vp_strided_store");
6560 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N);
6561 SDValue Mask = SST->getMask();
6562 SDValue StVal = SST->getValue();
6563 SDLoc DL(N);
6564
6565 if (OpNo == 1)
6566 assert(getTypeAction(Mask.getValueType()) ==
6567 TargetLowering::TypeWidenVector &&
6568 "Unable to widen VP strided store");
6569 else
6570 assert(getTypeAction(StVal.getValueType()) ==
6571 TargetLowering::TypeWidenVector &&
6572 "Unable to widen VP strided store");
6573
6574 StVal = GetWidenedVector(StVal);
6575 Mask = GetWidenedVector(Mask);
6576
6577 assert(StVal.getValueType().getVectorElementCount() ==
6578 Mask.getValueType().getVectorElementCount() &&
6579 "Data and mask vectors should have the same number of elements");
6580
6581 return DAG.getStridedStoreVP(
6582 SST->getChain(), DL, StVal, SST->getBasePtr(), SST->getOffset(),
6583 SST->getStride(), Mask, SST->getVectorLength(), SST->getMemoryVT(),
6584 SST->getMemOperand(), SST->getAddressingMode(), SST->isTruncatingStore(),
6585 SST->isCompressingStore());
6586 }
6587
WidenVecOp_MSTORE(SDNode * N,unsigned OpNo)6588 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
6589 assert((OpNo == 1 || OpNo == 4) &&
6590 "Can widen only data or mask operand of mstore");
6591 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
6592 SDValue Mask = MST->getMask();
6593 EVT MaskVT = Mask.getValueType();
6594 SDValue StVal = MST->getValue();
6595 SDLoc dl(N);
6596
6597 if (OpNo == 1) {
6598 // Widen the value.
6599 StVal = GetWidenedVector(StVal);
6600
6601 // The mask should be widened as well.
6602 EVT WideVT = StVal.getValueType();
6603 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
6604 MaskVT.getVectorElementType(),
6605 WideVT.getVectorNumElements());
6606 Mask = ModifyToType(Mask, WideMaskVT, true);
6607 } else {
6608 // Widen the mask.
6609 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
6610 Mask = ModifyToType(Mask, WideMaskVT, true);
6611
6612 EVT ValueVT = StVal.getValueType();
6613 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
6614 ValueVT.getVectorElementType(),
6615 WideMaskVT.getVectorNumElements());
6616 StVal = ModifyToType(StVal, WideVT);
6617 }
6618
6619 assert(Mask.getValueType().getVectorNumElements() ==
6620 StVal.getValueType().getVectorNumElements() &&
6621 "Mask and data vectors should have the same number of elements");
6622 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
6623 MST->getOffset(), Mask, MST->getMemoryVT(),
6624 MST->getMemOperand(), MST->getAddressingMode(),
6625 false, MST->isCompressingStore());
6626 }
6627
WidenVecOp_MGATHER(SDNode * N,unsigned OpNo)6628 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
6629 assert(OpNo == 4 && "Can widen only the index of mgather");
6630 auto *MG = cast<MaskedGatherSDNode>(N);
6631 SDValue DataOp = MG->getPassThru();
6632 SDValue Mask = MG->getMask();
6633 SDValue Scale = MG->getScale();
6634
6635 // Just widen the index. It's allowed to have extra elements.
6636 SDValue Index = GetWidenedVector(MG->getIndex());
6637
6638 SDLoc dl(N);
6639 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
6640 Scale};
6641 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
6642 MG->getMemOperand(), MG->getIndexType(),
6643 MG->getExtensionType());
6644 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
6645 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
6646 return SDValue();
6647 }
6648
WidenVecOp_MSCATTER(SDNode * N,unsigned OpNo)6649 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
6650 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
6651 SDValue DataOp = MSC->getValue();
6652 SDValue Mask = MSC->getMask();
6653 SDValue Index = MSC->getIndex();
6654 SDValue Scale = MSC->getScale();
6655 EVT WideMemVT = MSC->getMemoryVT();
6656
6657 if (OpNo == 1) {
6658 DataOp = GetWidenedVector(DataOp);
6659 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
6660
6661 // Widen index.
6662 EVT IndexVT = Index.getValueType();
6663 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
6664 IndexVT.getVectorElementType(), NumElts);
6665 Index = ModifyToType(Index, WideIndexVT);
6666
6667 // The mask should be widened as well.
6668 EVT MaskVT = Mask.getValueType();
6669 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
6670 MaskVT.getVectorElementType(), NumElts);
6671 Mask = ModifyToType(Mask, WideMaskVT, true);
6672
6673 // Widen the MemoryType
6674 WideMemVT = EVT::getVectorVT(*DAG.getContext(),
6675 MSC->getMemoryVT().getScalarType(), NumElts);
6676 } else if (OpNo == 4) {
6677 // Just widen the index. It's allowed to have extra elements.
6678 Index = GetWidenedVector(Index);
6679 } else
6680 llvm_unreachable("Can't widen this operand of mscatter");
6681
6682 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
6683 Scale};
6684 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N),
6685 Ops, MSC->getMemOperand(), MSC->getIndexType(),
6686 MSC->isTruncatingStore());
6687 }
6688
WidenVecOp_VP_SCATTER(SDNode * N,unsigned OpNo)6689 SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
6690 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(N);
6691 SDValue DataOp = VPSC->getValue();
6692 SDValue Mask = VPSC->getMask();
6693 SDValue Index = VPSC->getIndex();
6694 SDValue Scale = VPSC->getScale();
6695 EVT WideMemVT = VPSC->getMemoryVT();
6696
6697 if (OpNo == 1) {
6698 DataOp = GetWidenedVector(DataOp);
6699 Index = GetWidenedVector(Index);
6700 const auto WideEC = DataOp.getValueType().getVectorElementCount();
6701 Mask = GetWidenedMask(Mask, WideEC);
6702 WideMemVT = EVT::getVectorVT(*DAG.getContext(),
6703 VPSC->getMemoryVT().getScalarType(), WideEC);
6704 } else if (OpNo == 3) {
6705 // Just widen the index. It's allowed to have extra elements.
6706 Index = GetWidenedVector(Index);
6707 } else
6708 llvm_unreachable("Can't widen this operand of VP_SCATTER");
6709
6710 SDValue Ops[] = {
6711 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
6712 VPSC->getVectorLength()};
6713 return DAG.getScatterVP(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), Ops,
6714 VPSC->getMemOperand(), VPSC->getIndexType());
6715 }
6716
WidenVecOp_SETCC(SDNode * N)6717 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
6718 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
6719 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
6720 SDLoc dl(N);
6721 EVT VT = N->getValueType(0);
6722
6723 // WARNING: In this code we widen the compare instruction with garbage.
6724 // This garbage may contain denormal floats which may be slow. Is this a real
6725 // concern ? Should we zero the unused lanes if this is a float compare ?
6726
6727 // Get a new SETCC node to compare the newly widened operands.
6728 // Only some of the compared elements are legal.
6729 EVT SVT = getSetCCResultType(InOp0.getValueType());
6730 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
6731 if (VT.getScalarType() == MVT::i1)
6732 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
6733 SVT.getVectorElementCount());
6734
6735 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
6736 SVT, InOp0, InOp1, N->getOperand(2));
6737
6738 // Extract the needed results from the result vector.
6739 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
6740 SVT.getVectorElementType(),
6741 VT.getVectorElementCount());
6742 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
6743 DAG.getVectorIdxConstant(0, dl));
6744
6745 EVT OpVT = N->getOperand(0).getValueType();
6746 ISD::NodeType ExtendCode =
6747 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
6748 return DAG.getNode(ExtendCode, dl, VT, CC);
6749 }
6750
WidenVecOp_STRICT_FSETCC(SDNode * N)6751 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
6752 SDValue Chain = N->getOperand(0);
6753 SDValue LHS = GetWidenedVector(N->getOperand(1));
6754 SDValue RHS = GetWidenedVector(N->getOperand(2));
6755 SDValue CC = N->getOperand(3);
6756 SDLoc dl(N);
6757
6758 EVT VT = N->getValueType(0);
6759 EVT EltVT = VT.getVectorElementType();
6760 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
6761 unsigned NumElts = VT.getVectorNumElements();
6762
6763 // Unroll into a build vector.
6764 SmallVector<SDValue, 8> Scalars(NumElts);
6765 SmallVector<SDValue, 8> Chains(NumElts);
6766
6767 for (unsigned i = 0; i != NumElts; ++i) {
6768 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
6769 DAG.getVectorIdxConstant(i, dl));
6770 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
6771 DAG.getVectorIdxConstant(i, dl));
6772
6773 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
6774 {Chain, LHSElem, RHSElem, CC});
6775 Chains[i] = Scalars[i].getValue(1);
6776 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
6777 DAG.getBoolConstant(true, dl, EltVT, VT),
6778 DAG.getBoolConstant(false, dl, EltVT, VT));
6779 }
6780
6781 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
6782 ReplaceValueWith(SDValue(N, 1), NewChain);
6783
6784 return DAG.getBuildVector(VT, dl, Scalars);
6785 }
6786
WidenVecOp_VECREDUCE(SDNode * N)6787 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
6788 SDLoc dl(N);
6789 SDValue Op = GetWidenedVector(N->getOperand(0));
6790 EVT OrigVT = N->getOperand(0).getValueType();
6791 EVT WideVT = Op.getValueType();
6792 EVT ElemVT = OrigVT.getVectorElementType();
6793 SDNodeFlags Flags = N->getFlags();
6794
6795 unsigned Opc = N->getOpcode();
6796 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
6797 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
6798 assert(NeutralElem && "Neutral element must exist");
6799
6800 // Pad the vector with the neutral element.
6801 unsigned OrigElts = OrigVT.getVectorMinNumElements();
6802 unsigned WideElts = WideVT.getVectorMinNumElements();
6803
6804 if (WideVT.isScalableVector()) {
6805 unsigned GCD = std::gcd(OrigElts, WideElts);
6806 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
6807 ElementCount::getScalable(GCD));
6808 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
6809 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
6810 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral,
6811 DAG.getVectorIdxConstant(Idx, dl));
6812 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
6813 }
6814
6815 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
6816 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
6817 DAG.getVectorIdxConstant(Idx, dl));
6818
6819 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
6820 }
6821
WidenVecOp_VECREDUCE_SEQ(SDNode * N)6822 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
6823 SDLoc dl(N);
6824 SDValue AccOp = N->getOperand(0);
6825 SDValue VecOp = N->getOperand(1);
6826 SDValue Op = GetWidenedVector(VecOp);
6827
6828 EVT OrigVT = VecOp.getValueType();
6829 EVT WideVT = Op.getValueType();
6830 EVT ElemVT = OrigVT.getVectorElementType();
6831 SDNodeFlags Flags = N->getFlags();
6832
6833 unsigned Opc = N->getOpcode();
6834 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
6835 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
6836
6837 // Pad the vector with the neutral element.
6838 unsigned OrigElts = OrigVT.getVectorMinNumElements();
6839 unsigned WideElts = WideVT.getVectorMinNumElements();
6840
6841 if (WideVT.isScalableVector()) {
6842 unsigned GCD = std::gcd(OrigElts, WideElts);
6843 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), ElemVT,
6844 ElementCount::getScalable(GCD));
6845 SDValue SplatNeutral = DAG.getSplatVector(SplatVT, dl, NeutralElem);
6846 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
6847 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral,
6848 DAG.getVectorIdxConstant(Idx, dl));
6849 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags);
6850 }
6851
6852 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
6853 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
6854 DAG.getVectorIdxConstant(Idx, dl));
6855
6856 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags);
6857 }
6858
WidenVecOp_VP_REDUCE(SDNode * N)6859 SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
6860 assert(N->isVPOpcode() && "Expected VP opcode");
6861
6862 SDLoc dl(N);
6863 SDValue Op = GetWidenedVector(N->getOperand(1));
6864 SDValue Mask = GetWidenedMask(N->getOperand(2),
6865 Op.getValueType().getVectorElementCount());
6866
6867 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0),
6868 {N->getOperand(0), Op, Mask, N->getOperand(3)},
6869 N->getFlags());
6870 }
6871
WidenVecOp_VSELECT(SDNode * N)6872 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
6873 // This only gets called in the case that the left and right inputs and
6874 // result are of a legal odd vector type, and the condition is illegal i1 of
6875 // the same odd width that needs widening.
6876 EVT VT = N->getValueType(0);
6877 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
6878
6879 SDValue Cond = GetWidenedVector(N->getOperand(0));
6880 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
6881 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
6882 SDLoc DL(N);
6883
6884 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
6885 LeftIn, RightIn);
6886 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
6887 DAG.getVectorIdxConstant(0, DL));
6888 }
6889
6890 //===----------------------------------------------------------------------===//
6891 // Vector Widening Utilities
6892 //===----------------------------------------------------------------------===//
6893
6894 // Utility function to find the type to chop up a widen vector for load/store
6895 // TLI: Target lowering used to determine legal types.
6896 // Width: Width left need to load/store.
6897 // WidenVT: The widen vector type to load to/store from
6898 // Align: If 0, don't allow use of a wider type
6899 // WidenEx: If Align is not 0, the amount additional we can load/store from.
6900
findMemType(SelectionDAG & DAG,const TargetLowering & TLI,unsigned Width,EVT WidenVT,unsigned Align=0,unsigned WidenEx=0)6901 static std::optional<EVT> findMemType(SelectionDAG &DAG,
6902 const TargetLowering &TLI, unsigned Width,
6903 EVT WidenVT, unsigned Align = 0,
6904 unsigned WidenEx = 0) {
6905 EVT WidenEltVT = WidenVT.getVectorElementType();
6906 const bool Scalable = WidenVT.isScalableVector();
6907 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
6908 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
6909 unsigned AlignInBits = Align*8;
6910
6911 // If we have one element to load/store, return it.
6912 EVT RetVT = WidenEltVT;
6913 if (!Scalable && Width == WidenEltWidth)
6914 return RetVT;
6915
6916 // Don't bother looking for an integer type if the vector is scalable, skip
6917 // to vector types.
6918 if (!Scalable) {
6919 // See if there is larger legal integer than the element type to load/store.
6920 for (EVT MemVT : reverse(MVT::integer_valuetypes())) {
6921 unsigned MemVTWidth = MemVT.getSizeInBits();
6922 if (MemVT.getSizeInBits() <= WidenEltWidth)
6923 break;
6924 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
6925 if ((Action == TargetLowering::TypeLegal ||
6926 Action == TargetLowering::TypePromoteInteger) &&
6927 (WidenWidth % MemVTWidth) == 0 &&
6928 isPowerOf2_32(WidenWidth / MemVTWidth) &&
6929 (MemVTWidth <= Width ||
6930 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
6931 if (MemVTWidth == WidenWidth)
6932 return MemVT;
6933 RetVT = MemVT;
6934 break;
6935 }
6936 }
6937 }
6938
6939 // See if there is a larger vector type to load/store that has the same vector
6940 // element type and is evenly divisible with the WidenVT.
6941 for (EVT MemVT : reverse(MVT::vector_valuetypes())) {
6942 // Skip vector MVTs which don't match the scalable property of WidenVT.
6943 if (Scalable != MemVT.isScalableVector())
6944 continue;
6945 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
6946 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
6947 if ((Action == TargetLowering::TypeLegal ||
6948 Action == TargetLowering::TypePromoteInteger) &&
6949 WidenEltVT == MemVT.getVectorElementType() &&
6950 (WidenWidth % MemVTWidth) == 0 &&
6951 isPowerOf2_32(WidenWidth / MemVTWidth) &&
6952 (MemVTWidth <= Width ||
6953 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
6954 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
6955 return MemVT;
6956 }
6957 }
6958
6959 // Using element-wise loads and stores for widening operations is not
6960 // supported for scalable vectors
6961 if (Scalable)
6962 return std::nullopt;
6963
6964 return RetVT;
6965 }
6966
6967 // Builds a vector type from scalar loads
6968 // VecTy: Resulting Vector type
6969 // LDOps: Load operators to build a vector type
6970 // [Start,End) the list of loads to use.
BuildVectorFromScalar(SelectionDAG & DAG,EVT VecTy,SmallVectorImpl<SDValue> & LdOps,unsigned Start,unsigned End)6971 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
6972 SmallVectorImpl<SDValue> &LdOps,
6973 unsigned Start, unsigned End) {
6974 SDLoc dl(LdOps[Start]);
6975 EVT LdTy = LdOps[Start].getValueType();
6976 unsigned Width = VecTy.getSizeInBits();
6977 unsigned NumElts = Width / LdTy.getSizeInBits();
6978 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
6979
6980 unsigned Idx = 1;
6981 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
6982
6983 for (unsigned i = Start + 1; i != End; ++i) {
6984 EVT NewLdTy = LdOps[i].getValueType();
6985 if (NewLdTy != LdTy) {
6986 NumElts = Width / NewLdTy.getSizeInBits();
6987 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
6988 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
6989 // Readjust position and vector position based on new load type.
6990 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
6991 LdTy = NewLdTy;
6992 }
6993 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
6994 DAG.getVectorIdxConstant(Idx++, dl));
6995 }
6996 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
6997 }
6998
GenWidenVectorLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD)6999 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
7000 LoadSDNode *LD) {
7001 // The strategy assumes that we can efficiently load power-of-two widths.
7002 // The routine chops the vector into the largest vector loads with the same
7003 // element type or scalar loads and then recombines it to the widen vector
7004 // type.
7005 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
7006 EVT LdVT = LD->getMemoryVT();
7007 SDLoc dl(LD);
7008 assert(LdVT.isVector() && WidenVT.isVector());
7009 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
7010 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
7011
7012 // Load information
7013 SDValue Chain = LD->getChain();
7014 SDValue BasePtr = LD->getBasePtr();
7015 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
7016 AAMDNodes AAInfo = LD->getAAInfo();
7017
7018 TypeSize LdWidth = LdVT.getSizeInBits();
7019 TypeSize WidenWidth = WidenVT.getSizeInBits();
7020 TypeSize WidthDiff = WidenWidth - LdWidth;
7021 // Allow wider loads if they are sufficiently aligned to avoid memory faults
7022 // and if the original load is simple.
7023 unsigned LdAlign =
7024 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
7025
7026 // Find the vector type that can load from.
7027 std::optional<EVT> FirstVT =
7028 findMemType(DAG, TLI, LdWidth.getKnownMinValue(), WidenVT, LdAlign,
7029 WidthDiff.getKnownMinValue());
7030
7031 if (!FirstVT)
7032 return SDValue();
7033
7034 SmallVector<EVT, 8> MemVTs;
7035 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
7036
7037 // Unless we're able to load in one instruction we must work out how to load
7038 // the remainder.
7039 if (!TypeSize::isKnownLE(LdWidth, FirstVTWidth)) {
7040 std::optional<EVT> NewVT = FirstVT;
7041 TypeSize RemainingWidth = LdWidth;
7042 TypeSize NewVTWidth = FirstVTWidth;
7043 do {
7044 RemainingWidth -= NewVTWidth;
7045 if (TypeSize::isKnownLT(RemainingWidth, NewVTWidth)) {
7046 // The current type we are using is too large. Find a better size.
7047 NewVT = findMemType(DAG, TLI, RemainingWidth.getKnownMinValue(),
7048 WidenVT, LdAlign, WidthDiff.getKnownMinValue());
7049 if (!NewVT)
7050 return SDValue();
7051 NewVTWidth = NewVT->getSizeInBits();
7052 }
7053 MemVTs.push_back(*NewVT);
7054 } while (TypeSize::isKnownGT(RemainingWidth, NewVTWidth));
7055 }
7056
7057 SDValue LdOp = DAG.getLoad(*FirstVT, dl, Chain, BasePtr, LD->getPointerInfo(),
7058 LD->getOriginalAlign(), MMOFlags, AAInfo);
7059 LdChain.push_back(LdOp.getValue(1));
7060
7061 // Check if we can load the element with one instruction.
7062 if (MemVTs.empty()) {
7063 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
7064 if (!FirstVT->isVector()) {
7065 unsigned NumElts =
7066 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
7067 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), *FirstVT, NumElts);
7068 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
7069 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
7070 }
7071 if (FirstVT == WidenVT)
7072 return LdOp;
7073
7074 // TODO: We don't currently have any tests that exercise this code path.
7075 assert(WidenWidth.getFixedValue() % FirstVTWidth.getFixedValue() == 0);
7076 unsigned NumConcat =
7077 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
7078 SmallVector<SDValue, 16> ConcatOps(NumConcat);
7079 SDValue UndefVal = DAG.getUNDEF(*FirstVT);
7080 ConcatOps[0] = LdOp;
7081 for (unsigned i = 1; i != NumConcat; ++i)
7082 ConcatOps[i] = UndefVal;
7083 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
7084 }
7085
7086 // Load vector by using multiple loads from largest vector to scalar.
7087 SmallVector<SDValue, 16> LdOps;
7088 LdOps.push_back(LdOp);
7089
7090 uint64_t ScaledOffset = 0;
7091 MachinePointerInfo MPI = LD->getPointerInfo();
7092
7093 // First incremement past the first load.
7094 IncrementPointer(cast<LoadSDNode>(LdOp), *FirstVT, MPI, BasePtr,
7095 &ScaledOffset);
7096
7097 for (EVT MemVT : MemVTs) {
7098 Align NewAlign = ScaledOffset == 0
7099 ? LD->getOriginalAlign()
7100 : commonAlignment(LD->getAlign(), ScaledOffset);
7101 SDValue L =
7102 DAG.getLoad(MemVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags, AAInfo);
7103
7104 LdOps.push_back(L);
7105 LdChain.push_back(L.getValue(1));
7106 IncrementPointer(cast<LoadSDNode>(L), MemVT, MPI, BasePtr, &ScaledOffset);
7107 }
7108
7109 // Build the vector from the load operations.
7110 unsigned End = LdOps.size();
7111 if (!LdOps[0].getValueType().isVector())
7112 // All the loads are scalar loads.
7113 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
7114
7115 // If the load contains vectors, build the vector using concat vector.
7116 // All of the vectors used to load are power-of-2, and the scalar loads can be
7117 // combined to make a power-of-2 vector.
7118 SmallVector<SDValue, 16> ConcatOps(End);
7119 int i = End - 1;
7120 int Idx = End;
7121 EVT LdTy = LdOps[i].getValueType();
7122 // First, combine the scalar loads to a vector.
7123 if (!LdTy.isVector()) {
7124 for (--i; i >= 0; --i) {
7125 LdTy = LdOps[i].getValueType();
7126 if (LdTy.isVector())
7127 break;
7128 }
7129 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
7130 }
7131
7132 ConcatOps[--Idx] = LdOps[i];
7133 for (--i; i >= 0; --i) {
7134 EVT NewLdTy = LdOps[i].getValueType();
7135 if (NewLdTy != LdTy) {
7136 // Create a larger vector.
7137 TypeSize LdTySize = LdTy.getSizeInBits();
7138 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
7139 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
7140 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
7141 unsigned NumOps =
7142 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
7143 SmallVector<SDValue, 16> WidenOps(NumOps);
7144 unsigned j = 0;
7145 for (; j != End-Idx; ++j)
7146 WidenOps[j] = ConcatOps[Idx+j];
7147 for (; j != NumOps; ++j)
7148 WidenOps[j] = DAG.getUNDEF(LdTy);
7149
7150 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
7151 WidenOps);
7152 Idx = End - 1;
7153 LdTy = NewLdTy;
7154 }
7155 ConcatOps[--Idx] = LdOps[i];
7156 }
7157
7158 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
7159 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
7160 ArrayRef(&ConcatOps[Idx], End - Idx));
7161
7162 // We need to fill the rest with undefs to build the vector.
7163 unsigned NumOps =
7164 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
7165 SmallVector<SDValue, 16> WidenOps(NumOps);
7166 SDValue UndefVal = DAG.getUNDEF(LdTy);
7167 {
7168 unsigned i = 0;
7169 for (; i != End-Idx; ++i)
7170 WidenOps[i] = ConcatOps[Idx+i];
7171 for (; i != NumOps; ++i)
7172 WidenOps[i] = UndefVal;
7173 }
7174 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
7175 }
7176
7177 SDValue
GenWidenVectorExtLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD,ISD::LoadExtType ExtType)7178 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
7179 LoadSDNode *LD,
7180 ISD::LoadExtType ExtType) {
7181 // For extension loads, it may not be more efficient to chop up the vector
7182 // and then extend it. Instead, we unroll the load and build a new vector.
7183 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
7184 EVT LdVT = LD->getMemoryVT();
7185 SDLoc dl(LD);
7186 assert(LdVT.isVector() && WidenVT.isVector());
7187 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
7188
7189 // Load information
7190 SDValue Chain = LD->getChain();
7191 SDValue BasePtr = LD->getBasePtr();
7192 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
7193 AAMDNodes AAInfo = LD->getAAInfo();
7194
7195 if (LdVT.isScalableVector())
7196 report_fatal_error("Generating widen scalable extending vector loads is "
7197 "not yet supported");
7198
7199 EVT EltVT = WidenVT.getVectorElementType();
7200 EVT LdEltVT = LdVT.getVectorElementType();
7201 unsigned NumElts = LdVT.getVectorNumElements();
7202
7203 // Load each element and widen.
7204 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7205 SmallVector<SDValue, 16> Ops(WidenNumElts);
7206 unsigned Increment = LdEltVT.getSizeInBits() / 8;
7207 Ops[0] =
7208 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
7209 LdEltVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
7210 LdChain.push_back(Ops[0].getValue(1));
7211 unsigned i = 0, Offset = Increment;
7212 for (i=1; i < NumElts; ++i, Offset += Increment) {
7213 SDValue NewBasePtr =
7214 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::getFixed(Offset));
7215 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
7216 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
7217 LD->getOriginalAlign(), MMOFlags, AAInfo);
7218 LdChain.push_back(Ops[i].getValue(1));
7219 }
7220
7221 // Fill the rest with undefs.
7222 SDValue UndefVal = DAG.getUNDEF(EltVT);
7223 for (; i != WidenNumElts; ++i)
7224 Ops[i] = UndefVal;
7225
7226 return DAG.getBuildVector(WidenVT, dl, Ops);
7227 }
7228
GenWidenVectorStores(SmallVectorImpl<SDValue> & StChain,StoreSDNode * ST)7229 bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
7230 StoreSDNode *ST) {
7231 // The strategy assumes that we can efficiently store power-of-two widths.
7232 // The routine chops the vector into the largest vector stores with the same
7233 // element type or scalar stores.
7234 SDValue Chain = ST->getChain();
7235 SDValue BasePtr = ST->getBasePtr();
7236 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
7237 AAMDNodes AAInfo = ST->getAAInfo();
7238 SDValue ValOp = GetWidenedVector(ST->getValue());
7239 SDLoc dl(ST);
7240
7241 EVT StVT = ST->getMemoryVT();
7242 TypeSize StWidth = StVT.getSizeInBits();
7243 EVT ValVT = ValOp.getValueType();
7244 TypeSize ValWidth = ValVT.getSizeInBits();
7245 EVT ValEltVT = ValVT.getVectorElementType();
7246 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
7247 assert(StVT.getVectorElementType() == ValEltVT);
7248 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
7249 "Mismatch between store and value types");
7250
7251 int Idx = 0; // current index to store
7252
7253 MachinePointerInfo MPI = ST->getPointerInfo();
7254 uint64_t ScaledOffset = 0;
7255
7256 // A breakdown of how to widen this vector store. Each element of the vector
7257 // is a memory VT combined with the number of times it is to be stored to,
7258 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
7259 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
7260
7261 while (StWidth.isNonZero()) {
7262 // Find the largest vector type we can store with.
7263 std::optional<EVT> NewVT =
7264 findMemType(DAG, TLI, StWidth.getKnownMinValue(), ValVT);
7265 if (!NewVT)
7266 return false;
7267 MemVTs.push_back({*NewVT, 0});
7268 TypeSize NewVTWidth = NewVT->getSizeInBits();
7269
7270 do {
7271 StWidth -= NewVTWidth;
7272 MemVTs.back().second++;
7273 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
7274 }
7275
7276 for (const auto &Pair : MemVTs) {
7277 EVT NewVT = Pair.first;
7278 unsigned Count = Pair.second;
7279 TypeSize NewVTWidth = NewVT.getSizeInBits();
7280
7281 if (NewVT.isVector()) {
7282 unsigned NumVTElts = NewVT.getVectorMinNumElements();
7283 do {
7284 Align NewAlign = ScaledOffset == 0
7285 ? ST->getOriginalAlign()
7286 : commonAlignment(ST->getAlign(), ScaledOffset);
7287 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
7288 DAG.getVectorIdxConstant(Idx, dl));
7289 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign,
7290 MMOFlags, AAInfo);
7291 StChain.push_back(PartStore);
7292
7293 Idx += NumVTElts;
7294 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr,
7295 &ScaledOffset);
7296 } while (--Count);
7297 } else {
7298 // Cast the vector to the scalar type we can store.
7299 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
7300 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
7301 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
7302 // Readjust index position based on new vector type.
7303 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
7304 do {
7305 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
7306 DAG.getVectorIdxConstant(Idx++, dl));
7307 SDValue PartStore =
7308 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getOriginalAlign(),
7309 MMOFlags, AAInfo);
7310 StChain.push_back(PartStore);
7311
7312 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr);
7313 } while (--Count);
7314 // Restore index back to be relative to the original widen element type.
7315 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
7316 }
7317 }
7318
7319 return true;
7320 }
7321
7322 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
7323 /// input vector must have the same element type as NVT.
7324 /// FillWithZeroes specifies that the vector should be widened with zeroes.
ModifyToType(SDValue InOp,EVT NVT,bool FillWithZeroes)7325 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
7326 bool FillWithZeroes) {
7327 // Note that InOp might have been widened so it might already have
7328 // the right width or it might need be narrowed.
7329 EVT InVT = InOp.getValueType();
7330 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
7331 "input and widen element type must match");
7332 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
7333 "cannot modify scalable vectors in this way");
7334 SDLoc dl(InOp);
7335
7336 // Check if InOp already has the right width.
7337 if (InVT == NVT)
7338 return InOp;
7339
7340 ElementCount InEC = InVT.getVectorElementCount();
7341 ElementCount WidenEC = NVT.getVectorElementCount();
7342 if (WidenEC.hasKnownScalarFactor(InEC)) {
7343 unsigned NumConcat = WidenEC.getKnownScalarFactor(InEC);
7344 SmallVector<SDValue, 16> Ops(NumConcat);
7345 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
7346 DAG.getUNDEF(InVT);
7347 Ops[0] = InOp;
7348 for (unsigned i = 1; i != NumConcat; ++i)
7349 Ops[i] = FillVal;
7350
7351 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
7352 }
7353
7354 if (InEC.hasKnownScalarFactor(WidenEC))
7355 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
7356 DAG.getVectorIdxConstant(0, dl));
7357
7358 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
7359 "Scalable vectors should have been handled already.");
7360
7361 unsigned InNumElts = InEC.getFixedValue();
7362 unsigned WidenNumElts = WidenEC.getFixedValue();
7363
7364 // Fall back to extract and build (+ mask, if padding with zeros).
7365 SmallVector<SDValue, 16> Ops(WidenNumElts);
7366 EVT EltVT = NVT.getVectorElementType();
7367 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
7368 unsigned Idx;
7369 for (Idx = 0; Idx < MinNumElts; ++Idx)
7370 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
7371 DAG.getVectorIdxConstant(Idx, dl));
7372
7373 SDValue UndefVal = DAG.getUNDEF(EltVT);
7374 for (; Idx < WidenNumElts; ++Idx)
7375 Ops[Idx] = UndefVal;
7376
7377 SDValue Widened = DAG.getBuildVector(NVT, dl, Ops);
7378 if (!FillWithZeroes)
7379 return Widened;
7380
7381 assert(NVT.isInteger() &&
7382 "We expect to never want to FillWithZeroes for non-integral types.");
7383
7384 SmallVector<SDValue, 16> MaskOps;
7385 MaskOps.append(MinNumElts, DAG.getAllOnesConstant(dl, EltVT));
7386 MaskOps.append(WidenNumElts - MinNumElts, DAG.getConstant(0, dl, EltVT));
7387
7388 return DAG.getNode(ISD::AND, dl, NVT, Widened,
7389 DAG.getBuildVector(NVT, dl, MaskOps));
7390 }
7391