1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "LegalizeTypes.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/Support/ErrorHandling.h"
26 #include "llvm/Support/TypeSize.h"
27 #include "llvm/Support/raw_ostream.h"
28 using namespace llvm;
29
30 #define DEBUG_TYPE "legalize-types"
31
32 //===----------------------------------------------------------------------===//
33 // Result Vector Scalarization: <1 x ty> -> ty.
34 //===----------------------------------------------------------------------===//
35
ScalarizeVectorResult(SDNode * N,unsigned ResNo)36 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
37 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
38 dbgs() << "\n");
39 SDValue R = SDValue();
40
41 switch (N->getOpcode()) {
42 default:
43 #ifndef NDEBUG
44 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45 N->dump(&DAG);
46 dbgs() << "\n";
47 #endif
48 report_fatal_error("Do not know how to scalarize the result of this "
49 "operator!\n");
50
51 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
67 case ISD::ANY_EXTEND_VECTOR_INREG:
68 case ISD::SIGN_EXTEND_VECTOR_INREG:
69 case ISD::ZERO_EXTEND_VECTOR_INREG:
70 R = ScalarizeVecRes_VecInregOp(N);
71 break;
72 case ISD::ABS:
73 case ISD::ANY_EXTEND:
74 case ISD::BITREVERSE:
75 case ISD::BSWAP:
76 case ISD::CTLZ:
77 case ISD::CTLZ_ZERO_UNDEF:
78 case ISD::CTPOP:
79 case ISD::CTTZ:
80 case ISD::CTTZ_ZERO_UNDEF:
81 case ISD::FABS:
82 case ISD::FCEIL:
83 case ISD::FCOS:
84 case ISD::FEXP:
85 case ISD::FEXP2:
86 case ISD::FFLOOR:
87 case ISD::FLOG:
88 case ISD::FLOG10:
89 case ISD::FLOG2:
90 case ISD::FNEARBYINT:
91 case ISD::FNEG:
92 case ISD::FREEZE:
93 case ISD::ARITH_FENCE:
94 case ISD::FP_EXTEND:
95 case ISD::FP_TO_SINT:
96 case ISD::FP_TO_UINT:
97 case ISD::FRINT:
98 case ISD::FROUND:
99 case ISD::FROUNDEVEN:
100 case ISD::FSIN:
101 case ISD::FSQRT:
102 case ISD::FTRUNC:
103 case ISD::SIGN_EXTEND:
104 case ISD::SINT_TO_FP:
105 case ISD::TRUNCATE:
106 case ISD::UINT_TO_FP:
107 case ISD::ZERO_EXTEND:
108 case ISD::FCANONICALIZE:
109 R = ScalarizeVecRes_UnaryOp(N);
110 break;
111
112 case ISD::ADD:
113 case ISD::AND:
114 case ISD::FADD:
115 case ISD::FCOPYSIGN:
116 case ISD::FDIV:
117 case ISD::FMUL:
118 case ISD::FMINNUM:
119 case ISD::FMAXNUM:
120 case ISD::FMINNUM_IEEE:
121 case ISD::FMAXNUM_IEEE:
122 case ISD::FMINIMUM:
123 case ISD::FMAXIMUM:
124 case ISD::SMIN:
125 case ISD::SMAX:
126 case ISD::UMIN:
127 case ISD::UMAX:
128
129 case ISD::SADDSAT:
130 case ISD::UADDSAT:
131 case ISD::SSUBSAT:
132 case ISD::USUBSAT:
133 case ISD::SSHLSAT:
134 case ISD::USHLSAT:
135
136 case ISD::FPOW:
137 case ISD::FREM:
138 case ISD::FSUB:
139 case ISD::MUL:
140 case ISD::OR:
141 case ISD::SDIV:
142 case ISD::SREM:
143 case ISD::SUB:
144 case ISD::UDIV:
145 case ISD::UREM:
146 case ISD::XOR:
147 case ISD::SHL:
148 case ISD::SRA:
149 case ISD::SRL:
150 case ISD::ROTL:
151 case ISD::ROTR:
152 R = ScalarizeVecRes_BinOp(N);
153 break;
154 case ISD::FMA:
155 case ISD::FSHL:
156 case ISD::FSHR:
157 R = ScalarizeVecRes_TernaryOp(N);
158 break;
159
160 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
161 case ISD::STRICT_##DAGN:
162 #include "llvm/IR/ConstrainedOps.def"
163 R = ScalarizeVecRes_StrictFPOp(N);
164 break;
165
166 case ISD::FP_TO_UINT_SAT:
167 case ISD::FP_TO_SINT_SAT:
168 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
169 break;
170
171 case ISD::UADDO:
172 case ISD::SADDO:
173 case ISD::USUBO:
174 case ISD::SSUBO:
175 case ISD::UMULO:
176 case ISD::SMULO:
177 R = ScalarizeVecRes_OverflowOp(N, ResNo);
178 break;
179 case ISD::SMULFIX:
180 case ISD::SMULFIXSAT:
181 case ISD::UMULFIX:
182 case ISD::UMULFIXSAT:
183 case ISD::SDIVFIX:
184 case ISD::SDIVFIXSAT:
185 case ISD::UDIVFIX:
186 case ISD::UDIVFIXSAT:
187 R = ScalarizeVecRes_FIX(N);
188 break;
189 }
190
191 // If R is null, the sub-method took care of registering the result.
192 if (R.getNode())
193 SetScalarizedVector(SDValue(N, ResNo), R);
194 }
195
ScalarizeVecRes_BinOp(SDNode * N)196 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
197 SDValue LHS = GetScalarizedVector(N->getOperand(0));
198 SDValue RHS = GetScalarizedVector(N->getOperand(1));
199 return DAG.getNode(N->getOpcode(), SDLoc(N),
200 LHS.getValueType(), LHS, RHS, N->getFlags());
201 }
202
ScalarizeVecRes_TernaryOp(SDNode * N)203 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
204 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
205 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
206 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
207 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
208 Op2, N->getFlags());
209 }
210
ScalarizeVecRes_FIX(SDNode * N)211 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
212 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
213 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
214 SDValue Op2 = N->getOperand(2);
215 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
216 Op2, N->getFlags());
217 }
218
ScalarizeVecRes_StrictFPOp(SDNode * N)219 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
220 EVT VT = N->getValueType(0).getVectorElementType();
221 unsigned NumOpers = N->getNumOperands();
222 SDValue Chain = N->getOperand(0);
223 EVT ValueVTs[] = {VT, MVT::Other};
224 SDLoc dl(N);
225
226 SmallVector<SDValue, 4> Opers(NumOpers);
227
228 // The Chain is the first operand.
229 Opers[0] = Chain;
230
231 // Now process the remaining operands.
232 for (unsigned i = 1; i < NumOpers; ++i) {
233 SDValue Oper = N->getOperand(i);
234
235 if (Oper.getValueType().isVector())
236 Oper = GetScalarizedVector(Oper);
237
238 Opers[i] = Oper;
239 }
240
241 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
242 Opers, N->getFlags());
243
244 // Legalize the chain result - switch anything that used the old chain to
245 // use the new one.
246 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
247 return Result;
248 }
249
ScalarizeVecRes_OverflowOp(SDNode * N,unsigned ResNo)250 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
251 unsigned ResNo) {
252 SDLoc DL(N);
253 EVT ResVT = N->getValueType(0);
254 EVT OvVT = N->getValueType(1);
255
256 SDValue ScalarLHS, ScalarRHS;
257 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
258 ScalarLHS = GetScalarizedVector(N->getOperand(0));
259 ScalarRHS = GetScalarizedVector(N->getOperand(1));
260 } else {
261 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
262 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
263 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
264 ScalarLHS = ElemsLHS[0];
265 ScalarRHS = ElemsRHS[0];
266 }
267
268 SDVTList ScalarVTs = DAG.getVTList(
269 ResVT.getVectorElementType(), OvVT.getVectorElementType());
270 SDNode *ScalarNode = DAG.getNode(
271 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
272 ScalarNode->setFlags(N->getFlags());
273
274 // Replace the other vector result not being explicitly scalarized here.
275 unsigned OtherNo = 1 - ResNo;
276 EVT OtherVT = N->getValueType(OtherNo);
277 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
278 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
279 } else {
280 SDValue OtherVal = DAG.getNode(
281 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
282 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
283 }
284
285 return SDValue(ScalarNode, ResNo);
286 }
287
ScalarizeVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)288 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
289 unsigned ResNo) {
290 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
291 return GetScalarizedVector(Op);
292 }
293
ScalarizeVecRes_BITCAST(SDNode * N)294 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
295 SDValue Op = N->getOperand(0);
296 if (Op.getValueType().isVector()
297 && Op.getValueType().getVectorNumElements() == 1
298 && !isSimpleLegalType(Op.getValueType()))
299 Op = GetScalarizedVector(Op);
300 EVT NewVT = N->getValueType(0).getVectorElementType();
301 return DAG.getNode(ISD::BITCAST, SDLoc(N),
302 NewVT, Op);
303 }
304
ScalarizeVecRes_BUILD_VECTOR(SDNode * N)305 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
306 EVT EltVT = N->getValueType(0).getVectorElementType();
307 SDValue InOp = N->getOperand(0);
308 // The BUILD_VECTOR operands may be of wider element types and
309 // we may need to truncate them back to the requested return type.
310 if (EltVT.isInteger())
311 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
312 return InOp;
313 }
314
ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode * N)315 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
316 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
317 N->getValueType(0).getVectorElementType(),
318 N->getOperand(0), N->getOperand(1));
319 }
320
ScalarizeVecRes_FP_ROUND(SDNode * N)321 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
322 SDLoc DL(N);
323 SDValue Op = N->getOperand(0);
324 EVT OpVT = Op.getValueType();
325 // The result needs scalarizing, but it's not a given that the source does.
326 // See similar logic in ScalarizeVecRes_UnaryOp.
327 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
328 Op = GetScalarizedVector(Op);
329 } else {
330 EVT VT = OpVT.getVectorElementType();
331 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
332 DAG.getVectorIdxConstant(0, DL));
333 }
334 return DAG.getNode(ISD::FP_ROUND, DL,
335 N->getValueType(0).getVectorElementType(), Op,
336 N->getOperand(1));
337 }
338
ScalarizeVecRes_FPOWI(SDNode * N)339 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
340 SDValue Op = GetScalarizedVector(N->getOperand(0));
341 return DAG.getNode(ISD::FPOWI, SDLoc(N),
342 Op.getValueType(), Op, N->getOperand(1));
343 }
344
ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode * N)345 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
346 // The value to insert may have a wider type than the vector element type,
347 // so be sure to truncate it to the element type if necessary.
348 SDValue Op = N->getOperand(1);
349 EVT EltVT = N->getValueType(0).getVectorElementType();
350 if (Op.getValueType() != EltVT)
351 // FIXME: Can this happen for floating point types?
352 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
353 return Op;
354 }
355
ScalarizeVecRes_LOAD(LoadSDNode * N)356 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
357 assert(N->isUnindexed() && "Indexed vector load?");
358
359 SDValue Result = DAG.getLoad(
360 ISD::UNINDEXED, N->getExtensionType(),
361 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
362 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
363 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
364 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
365
366 // Legalize the chain result - switch anything that used the old chain to
367 // use the new one.
368 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
369 return Result;
370 }
371
ScalarizeVecRes_UnaryOp(SDNode * N)372 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
373 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
374 EVT DestVT = N->getValueType(0).getVectorElementType();
375 SDValue Op = N->getOperand(0);
376 EVT OpVT = Op.getValueType();
377 SDLoc DL(N);
378 // The result needs scalarizing, but it's not a given that the source does.
379 // This is a workaround for targets where it's impossible to scalarize the
380 // result of a conversion, because the source type is legal.
381 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
382 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
383 // legal and was not scalarized.
384 // See the similar logic in ScalarizeVecRes_SETCC
385 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
386 Op = GetScalarizedVector(Op);
387 } else {
388 EVT VT = OpVT.getVectorElementType();
389 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
390 DAG.getVectorIdxConstant(0, DL));
391 }
392 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
393 }
394
ScalarizeVecRes_InregOp(SDNode * N)395 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
396 EVT EltVT = N->getValueType(0).getVectorElementType();
397 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
398 SDValue LHS = GetScalarizedVector(N->getOperand(0));
399 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
400 LHS, DAG.getValueType(ExtVT));
401 }
402
ScalarizeVecRes_VecInregOp(SDNode * N)403 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
404 SDLoc DL(N);
405 SDValue Op = N->getOperand(0);
406
407 EVT OpVT = Op.getValueType();
408 EVT OpEltVT = OpVT.getVectorElementType();
409 EVT EltVT = N->getValueType(0).getVectorElementType();
410
411 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
412 Op = GetScalarizedVector(Op);
413 } else {
414 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
415 DAG.getVectorIdxConstant(0, DL));
416 }
417
418 switch (N->getOpcode()) {
419 case ISD::ANY_EXTEND_VECTOR_INREG:
420 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
421 case ISD::SIGN_EXTEND_VECTOR_INREG:
422 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
423 case ISD::ZERO_EXTEND_VECTOR_INREG:
424 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
425 }
426
427 llvm_unreachable("Illegal extend_vector_inreg opcode");
428 }
429
ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode * N)430 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
431 // If the operand is wider than the vector element type then it is implicitly
432 // truncated. Make that explicit here.
433 EVT EltVT = N->getValueType(0).getVectorElementType();
434 SDValue InOp = N->getOperand(0);
435 if (InOp.getValueType() != EltVT)
436 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
437 return InOp;
438 }
439
ScalarizeVecRes_VSELECT(SDNode * N)440 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
441 SDValue Cond = N->getOperand(0);
442 EVT OpVT = Cond.getValueType();
443 SDLoc DL(N);
444 // The vselect result and true/value operands needs scalarizing, but it's
445 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
446 // See the similar logic in ScalarizeVecRes_SETCC
447 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
448 Cond = GetScalarizedVector(Cond);
449 } else {
450 EVT VT = OpVT.getVectorElementType();
451 Cond = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
452 DAG.getVectorIdxConstant(0, DL));
453 }
454
455 SDValue LHS = GetScalarizedVector(N->getOperand(1));
456 TargetLowering::BooleanContent ScalarBool =
457 TLI.getBooleanContents(false, false);
458 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
459
460 // If integer and float booleans have different contents then we can't
461 // reliably optimize in all cases. There is a full explanation for this in
462 // DAGCombiner::visitSELECT() where the same issue affects folding
463 // (select C, 0, 1) to (xor C, 1).
464 if (TLI.getBooleanContents(false, false) !=
465 TLI.getBooleanContents(false, true)) {
466 // At least try the common case where the boolean is generated by a
467 // comparison.
468 if (Cond->getOpcode() == ISD::SETCC) {
469 EVT OpVT = Cond->getOperand(0).getValueType();
470 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
471 VecBool = TLI.getBooleanContents(OpVT);
472 } else
473 ScalarBool = TargetLowering::UndefinedBooleanContent;
474 }
475
476 EVT CondVT = Cond.getValueType();
477 if (ScalarBool != VecBool) {
478 switch (ScalarBool) {
479 case TargetLowering::UndefinedBooleanContent:
480 break;
481 case TargetLowering::ZeroOrOneBooleanContent:
482 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
483 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
484 // Vector read from all ones, scalar expects a single 1 so mask.
485 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
486 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
487 break;
488 case TargetLowering::ZeroOrNegativeOneBooleanContent:
489 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
490 VecBool == TargetLowering::ZeroOrOneBooleanContent);
491 // Vector reads from a one, scalar from all ones so sign extend.
492 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
493 Cond, DAG.getValueType(MVT::i1));
494 break;
495 }
496 }
497
498 // Truncate the condition if needed
499 auto BoolVT = getSetCCResultType(CondVT);
500 if (BoolVT.bitsLT(CondVT))
501 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
502
503 return DAG.getSelect(SDLoc(N),
504 LHS.getValueType(), Cond, LHS,
505 GetScalarizedVector(N->getOperand(2)));
506 }
507
ScalarizeVecRes_SELECT(SDNode * N)508 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
509 SDValue LHS = GetScalarizedVector(N->getOperand(1));
510 return DAG.getSelect(SDLoc(N),
511 LHS.getValueType(), N->getOperand(0), LHS,
512 GetScalarizedVector(N->getOperand(2)));
513 }
514
ScalarizeVecRes_SELECT_CC(SDNode * N)515 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
516 SDValue LHS = GetScalarizedVector(N->getOperand(2));
517 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
518 N->getOperand(0), N->getOperand(1),
519 LHS, GetScalarizedVector(N->getOperand(3)),
520 N->getOperand(4));
521 }
522
ScalarizeVecRes_UNDEF(SDNode * N)523 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
524 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
525 }
526
ScalarizeVecRes_VECTOR_SHUFFLE(SDNode * N)527 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
528 // Figure out if the scalar is the LHS or RHS and return it.
529 SDValue Arg = N->getOperand(2).getOperand(0);
530 if (Arg.isUndef())
531 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
532 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
533 return GetScalarizedVector(N->getOperand(Op));
534 }
535
ScalarizeVecRes_FP_TO_XINT_SAT(SDNode * N)536 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
537 SDValue Src = N->getOperand(0);
538 EVT SrcVT = Src.getValueType();
539 SDLoc dl(N);
540
541 // Handle case where result is scalarized but operand is not
542 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
543 Src = GetScalarizedVector(Src);
544 else
545 Src = DAG.getNode(
546 ISD::EXTRACT_VECTOR_ELT, dl, SrcVT.getVectorElementType(), Src,
547 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
548
549 EVT DstVT = N->getValueType(0).getVectorElementType();
550 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
551 }
552
ScalarizeVecRes_SETCC(SDNode * N)553 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
554 assert(N->getValueType(0).isVector() &&
555 N->getOperand(0).getValueType().isVector() &&
556 "Operand types must be vectors");
557 SDValue LHS = N->getOperand(0);
558 SDValue RHS = N->getOperand(1);
559 EVT OpVT = LHS.getValueType();
560 EVT NVT = N->getValueType(0).getVectorElementType();
561 SDLoc DL(N);
562
563 // The result needs scalarizing, but it's not a given that the source does.
564 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
565 LHS = GetScalarizedVector(LHS);
566 RHS = GetScalarizedVector(RHS);
567 } else {
568 EVT VT = OpVT.getVectorElementType();
569 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
570 DAG.getVectorIdxConstant(0, DL));
571 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
572 DAG.getVectorIdxConstant(0, DL));
573 }
574
575 // Turn it into a scalar SETCC.
576 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
577 N->getOperand(2));
578 // Vectors may have a different boolean contents to scalars. Promote the
579 // value appropriately.
580 ISD::NodeType ExtendCode =
581 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
582 return DAG.getNode(ExtendCode, DL, NVT, Res);
583 }
584
585
586 //===----------------------------------------------------------------------===//
587 // Operand Vector Scalarization <1 x ty> -> ty.
588 //===----------------------------------------------------------------------===//
589
ScalarizeVectorOperand(SDNode * N,unsigned OpNo)590 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
591 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
592 dbgs() << "\n");
593 SDValue Res = SDValue();
594
595 switch (N->getOpcode()) {
596 default:
597 #ifndef NDEBUG
598 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
599 N->dump(&DAG);
600 dbgs() << "\n";
601 #endif
602 report_fatal_error("Do not know how to scalarize this operator's "
603 "operand!\n");
604 case ISD::BITCAST:
605 Res = ScalarizeVecOp_BITCAST(N);
606 break;
607 case ISD::ANY_EXTEND:
608 case ISD::ZERO_EXTEND:
609 case ISD::SIGN_EXTEND:
610 case ISD::TRUNCATE:
611 case ISD::FP_TO_SINT:
612 case ISD::FP_TO_UINT:
613 case ISD::SINT_TO_FP:
614 case ISD::UINT_TO_FP:
615 Res = ScalarizeVecOp_UnaryOp(N);
616 break;
617 case ISD::STRICT_SINT_TO_FP:
618 case ISD::STRICT_UINT_TO_FP:
619 case ISD::STRICT_FP_TO_SINT:
620 case ISD::STRICT_FP_TO_UINT:
621 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
622 break;
623 case ISD::CONCAT_VECTORS:
624 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
625 break;
626 case ISD::EXTRACT_VECTOR_ELT:
627 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
628 break;
629 case ISD::VSELECT:
630 Res = ScalarizeVecOp_VSELECT(N);
631 break;
632 case ISD::SETCC:
633 Res = ScalarizeVecOp_VSETCC(N);
634 break;
635 case ISD::STORE:
636 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
637 break;
638 case ISD::STRICT_FP_ROUND:
639 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
640 break;
641 case ISD::FP_ROUND:
642 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
643 break;
644 case ISD::STRICT_FP_EXTEND:
645 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
646 break;
647 case ISD::FP_EXTEND:
648 Res = ScalarizeVecOp_FP_EXTEND(N);
649 break;
650 case ISD::VECREDUCE_FADD:
651 case ISD::VECREDUCE_FMUL:
652 case ISD::VECREDUCE_ADD:
653 case ISD::VECREDUCE_MUL:
654 case ISD::VECREDUCE_AND:
655 case ISD::VECREDUCE_OR:
656 case ISD::VECREDUCE_XOR:
657 case ISD::VECREDUCE_SMAX:
658 case ISD::VECREDUCE_SMIN:
659 case ISD::VECREDUCE_UMAX:
660 case ISD::VECREDUCE_UMIN:
661 case ISD::VECREDUCE_FMAX:
662 case ISD::VECREDUCE_FMIN:
663 Res = ScalarizeVecOp_VECREDUCE(N);
664 break;
665 case ISD::VECREDUCE_SEQ_FADD:
666 case ISD::VECREDUCE_SEQ_FMUL:
667 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
668 break;
669 }
670
671 // If the result is null, the sub-method took care of registering results etc.
672 if (!Res.getNode()) return false;
673
674 // If the result is N, the sub-method updated N in place. Tell the legalizer
675 // core about this.
676 if (Res.getNode() == N)
677 return true;
678
679 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
680 "Invalid operand expansion");
681
682 ReplaceValueWith(SDValue(N, 0), Res);
683 return false;
684 }
685
686 /// If the value to convert is a vector that needs to be scalarized, it must be
687 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_BITCAST(SDNode * N)688 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
689 SDValue Elt = GetScalarizedVector(N->getOperand(0));
690 return DAG.getNode(ISD::BITCAST, SDLoc(N),
691 N->getValueType(0), Elt);
692 }
693
694 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
695 /// Do the operation on the element instead.
ScalarizeVecOp_UnaryOp(SDNode * N)696 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
697 assert(N->getValueType(0).getVectorNumElements() == 1 &&
698 "Unexpected vector type!");
699 SDValue Elt = GetScalarizedVector(N->getOperand(0));
700 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
701 N->getValueType(0).getScalarType(), Elt);
702 // Revectorize the result so the types line up with what the uses of this
703 // expression expect.
704 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
705 }
706
707 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
708 /// Do the strict FP operation on the element instead.
ScalarizeVecOp_UnaryOp_StrictFP(SDNode * N)709 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
710 assert(N->getValueType(0).getVectorNumElements() == 1 &&
711 "Unexpected vector type!");
712 SDValue Elt = GetScalarizedVector(N->getOperand(1));
713 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
714 { N->getValueType(0).getScalarType(), MVT::Other },
715 { N->getOperand(0), Elt });
716 // Legalize the chain result - switch anything that used the old chain to
717 // use the new one.
718 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
719 // Revectorize the result so the types line up with what the uses of this
720 // expression expect.
721 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
722
723 // Do our own replacement and return SDValue() to tell the caller that we
724 // handled all replacements since caller can only handle a single result.
725 ReplaceValueWith(SDValue(N, 0), Res);
726 return SDValue();
727 }
728
729 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
ScalarizeVecOp_CONCAT_VECTORS(SDNode * N)730 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
731 SmallVector<SDValue, 8> Ops(N->getNumOperands());
732 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
733 Ops[i] = GetScalarizedVector(N->getOperand(i));
734 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
735 }
736
737 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
738 /// so just return the element, ignoring the index.
ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode * N)739 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
740 EVT VT = N->getValueType(0);
741 SDValue Res = GetScalarizedVector(N->getOperand(0));
742 if (Res.getValueType() != VT)
743 Res = VT.isFloatingPoint()
744 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
745 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
746 return Res;
747 }
748
749 /// If the input condition is a vector that needs to be scalarized, it must be
750 /// <1 x i1>, so just convert to a normal ISD::SELECT
751 /// (still with vector output type since that was acceptable if we got here).
ScalarizeVecOp_VSELECT(SDNode * N)752 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
753 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
754 EVT VT = N->getValueType(0);
755
756 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
757 N->getOperand(2));
758 }
759
760 /// If the operand is a vector that needs to be scalarized then the
761 /// result must be v1i1, so just convert to a scalar SETCC and wrap
762 /// with a scalar_to_vector since the res type is legal if we got here
ScalarizeVecOp_VSETCC(SDNode * N)763 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
764 assert(N->getValueType(0).isVector() &&
765 N->getOperand(0).getValueType().isVector() &&
766 "Operand types must be vectors");
767 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
768
769 EVT VT = N->getValueType(0);
770 SDValue LHS = GetScalarizedVector(N->getOperand(0));
771 SDValue RHS = GetScalarizedVector(N->getOperand(1));
772
773 EVT OpVT = N->getOperand(0).getValueType();
774 EVT NVT = VT.getVectorElementType();
775 SDLoc DL(N);
776 // Turn it into a scalar SETCC.
777 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
778 N->getOperand(2));
779
780 // Vectors may have a different boolean contents to scalars. Promote the
781 // value appropriately.
782 ISD::NodeType ExtendCode =
783 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
784
785 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
786
787 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
788 }
789
790 /// If the value to store is a vector that needs to be scalarized, it must be
791 /// <1 x ty>. Just store the element.
ScalarizeVecOp_STORE(StoreSDNode * N,unsigned OpNo)792 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
793 assert(N->isUnindexed() && "Indexed store of one-element vector?");
794 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
795 SDLoc dl(N);
796
797 if (N->isTruncatingStore())
798 return DAG.getTruncStore(
799 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
800 N->getBasePtr(), N->getPointerInfo(),
801 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
802 N->getMemOperand()->getFlags(), N->getAAInfo());
803
804 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
805 N->getBasePtr(), N->getPointerInfo(),
806 N->getOriginalAlign(), N->getMemOperand()->getFlags(),
807 N->getAAInfo());
808 }
809
810 /// If the value to round is a vector that needs to be scalarized, it must be
811 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_ROUND(SDNode * N,unsigned OpNo)812 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
813 assert(OpNo == 0 && "Wrong operand for scalarization!");
814 SDValue Elt = GetScalarizedVector(N->getOperand(0));
815 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
816 N->getValueType(0).getVectorElementType(), Elt,
817 N->getOperand(1));
818 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
819 }
820
ScalarizeVecOp_STRICT_FP_ROUND(SDNode * N,unsigned OpNo)821 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
822 unsigned OpNo) {
823 assert(OpNo == 1 && "Wrong operand for scalarization!");
824 SDValue Elt = GetScalarizedVector(N->getOperand(1));
825 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
826 { N->getValueType(0).getVectorElementType(),
827 MVT::Other },
828 { N->getOperand(0), Elt, N->getOperand(2) });
829 // Legalize the chain result - switch anything that used the old chain to
830 // use the new one.
831 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
832
833 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
834
835 // Do our own replacement and return SDValue() to tell the caller that we
836 // handled all replacements since caller can only handle a single result.
837 ReplaceValueWith(SDValue(N, 0), Res);
838 return SDValue();
839 }
840
841 /// If the value to extend is a vector that needs to be scalarized, it must be
842 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_EXTEND(SDNode * N)843 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
844 SDValue Elt = GetScalarizedVector(N->getOperand(0));
845 SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
846 N->getValueType(0).getVectorElementType(), Elt);
847 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
848 }
849
850 /// If the value to extend is a vector that needs to be scalarized, it must be
851 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_STRICT_FP_EXTEND(SDNode * N)852 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
853 SDValue Elt = GetScalarizedVector(N->getOperand(1));
854 SDValue Res =
855 DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
856 {N->getValueType(0).getVectorElementType(), MVT::Other},
857 {N->getOperand(0), Elt});
858 // Legalize the chain result - switch anything that used the old chain to
859 // use the new one.
860 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
861
862 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
863
864 // Do our own replacement and return SDValue() to tell the caller that we
865 // handled all replacements since caller can only handle a single result.
866 ReplaceValueWith(SDValue(N, 0), Res);
867 return SDValue();
868 }
869
ScalarizeVecOp_VECREDUCE(SDNode * N)870 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
871 SDValue Res = GetScalarizedVector(N->getOperand(0));
872 // Result type may be wider than element type.
873 if (Res.getValueType() != N->getValueType(0))
874 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
875 return Res;
876 }
877
ScalarizeVecOp_VECREDUCE_SEQ(SDNode * N)878 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
879 SDValue AccOp = N->getOperand(0);
880 SDValue VecOp = N->getOperand(1);
881
882 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
883
884 SDValue Op = GetScalarizedVector(VecOp);
885 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
886 AccOp, Op, N->getFlags());
887 }
888
889 //===----------------------------------------------------------------------===//
890 // Result Vector Splitting
891 //===----------------------------------------------------------------------===//
892
893 /// This method is called when the specified result of the specified node is
894 /// found to need vector splitting. At this point, the node may also have
895 /// invalid operands or may have other results that need legalization, we just
896 /// know that (at least) one result needs vector splitting.
SplitVectorResult(SDNode * N,unsigned ResNo)897 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
898 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
899 SDValue Lo, Hi;
900
901 // See if the target wants to custom expand this node.
902 if (CustomLowerNode(N, N->getValueType(ResNo), true))
903 return;
904
905 switch (N->getOpcode()) {
906 default:
907 #ifndef NDEBUG
908 dbgs() << "SplitVectorResult #" << ResNo << ": ";
909 N->dump(&DAG);
910 dbgs() << "\n";
911 #endif
912 report_fatal_error("Do not know how to split the result of this "
913 "operator!\n");
914
915 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
916 case ISD::VSELECT:
917 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
918 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
919 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
920 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
921 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
922 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
923 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
924 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
925 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
926 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
927 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
928 case ISD::SPLAT_VECTOR:
929 case ISD::SCALAR_TO_VECTOR:
930 SplitVecRes_ScalarOp(N, Lo, Hi);
931 break;
932 case ISD::STEP_VECTOR:
933 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
934 break;
935 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
936 case ISD::LOAD:
937 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
938 break;
939 case ISD::MLOAD:
940 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
941 break;
942 case ISD::MGATHER:
943 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
944 break;
945 case ISD::SETCC:
946 SplitVecRes_SETCC(N, Lo, Hi);
947 break;
948 case ISD::VECTOR_REVERSE:
949 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
950 break;
951 case ISD::VECTOR_SHUFFLE:
952 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
953 break;
954 case ISD::VECTOR_SPLICE:
955 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
956 break;
957 case ISD::VAARG:
958 SplitVecRes_VAARG(N, Lo, Hi);
959 break;
960
961 case ISD::ANY_EXTEND_VECTOR_INREG:
962 case ISD::SIGN_EXTEND_VECTOR_INREG:
963 case ISD::ZERO_EXTEND_VECTOR_INREG:
964 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
965 break;
966
967 case ISD::ABS:
968 case ISD::BITREVERSE:
969 case ISD::BSWAP:
970 case ISD::CTLZ:
971 case ISD::CTTZ:
972 case ISD::CTLZ_ZERO_UNDEF:
973 case ISD::CTTZ_ZERO_UNDEF:
974 case ISD::CTPOP:
975 case ISD::FABS:
976 case ISD::FCEIL:
977 case ISD::FCOS:
978 case ISD::FEXP:
979 case ISD::FEXP2:
980 case ISD::FFLOOR:
981 case ISD::FLOG:
982 case ISD::FLOG10:
983 case ISD::FLOG2:
984 case ISD::FNEARBYINT:
985 case ISD::FNEG:
986 case ISD::FREEZE:
987 case ISD::ARITH_FENCE:
988 case ISD::FP_EXTEND:
989 case ISD::FP_ROUND:
990 case ISD::FP_TO_SINT:
991 case ISD::FP_TO_UINT:
992 case ISD::FRINT:
993 case ISD::FROUND:
994 case ISD::FROUNDEVEN:
995 case ISD::FSIN:
996 case ISD::FSQRT:
997 case ISD::FTRUNC:
998 case ISD::SINT_TO_FP:
999 case ISD::TRUNCATE:
1000 case ISD::UINT_TO_FP:
1001 case ISD::FCANONICALIZE:
1002 SplitVecRes_UnaryOp(N, Lo, Hi);
1003 break;
1004
1005 case ISD::ANY_EXTEND:
1006 case ISD::SIGN_EXTEND:
1007 case ISD::ZERO_EXTEND:
1008 SplitVecRes_ExtendOp(N, Lo, Hi);
1009 break;
1010
1011 case ISD::ADD:
1012 case ISD::SUB:
1013 case ISD::MUL:
1014 case ISD::MULHS:
1015 case ISD::MULHU:
1016 case ISD::FADD:
1017 case ISD::FSUB:
1018 case ISD::FMUL:
1019 case ISD::FMINNUM:
1020 case ISD::FMAXNUM:
1021 case ISD::FMINIMUM:
1022 case ISD::FMAXIMUM:
1023 case ISD::SDIV:
1024 case ISD::UDIV:
1025 case ISD::FDIV:
1026 case ISD::FPOW:
1027 case ISD::AND:
1028 case ISD::OR:
1029 case ISD::XOR:
1030 case ISD::SHL:
1031 case ISD::SRA:
1032 case ISD::SRL:
1033 case ISD::UREM:
1034 case ISD::SREM:
1035 case ISD::FREM:
1036 case ISD::SMIN:
1037 case ISD::SMAX:
1038 case ISD::UMIN:
1039 case ISD::UMAX:
1040 case ISD::SADDSAT:
1041 case ISD::UADDSAT:
1042 case ISD::SSUBSAT:
1043 case ISD::USUBSAT:
1044 case ISD::SSHLSAT:
1045 case ISD::USHLSAT:
1046 case ISD::ROTL:
1047 case ISD::ROTR:
1048 SplitVecRes_BinOp(N, Lo, Hi);
1049 break;
1050 case ISD::FMA:
1051 case ISD::FSHL:
1052 case ISD::FSHR:
1053 SplitVecRes_TernaryOp(N, Lo, Hi);
1054 break;
1055
1056 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1057 case ISD::STRICT_##DAGN:
1058 #include "llvm/IR/ConstrainedOps.def"
1059 SplitVecRes_StrictFPOp(N, Lo, Hi);
1060 break;
1061
1062 case ISD::FP_TO_UINT_SAT:
1063 case ISD::FP_TO_SINT_SAT:
1064 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1065 break;
1066
1067 case ISD::UADDO:
1068 case ISD::SADDO:
1069 case ISD::USUBO:
1070 case ISD::SSUBO:
1071 case ISD::UMULO:
1072 case ISD::SMULO:
1073 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1074 break;
1075 case ISD::SMULFIX:
1076 case ISD::SMULFIXSAT:
1077 case ISD::UMULFIX:
1078 case ISD::UMULFIXSAT:
1079 case ISD::SDIVFIX:
1080 case ISD::SDIVFIXSAT:
1081 case ISD::UDIVFIX:
1082 case ISD::UDIVFIXSAT:
1083 SplitVecRes_FIX(N, Lo, Hi);
1084 break;
1085 }
1086
1087 // If Lo/Hi is null, the sub-method took care of registering results etc.
1088 if (Lo.getNode())
1089 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1090 }
1091
IncrementPointer(MemSDNode * N,EVT MemVT,MachinePointerInfo & MPI,SDValue & Ptr,uint64_t * ScaledOffset)1092 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1093 MachinePointerInfo &MPI, SDValue &Ptr,
1094 uint64_t *ScaledOffset) {
1095 SDLoc DL(N);
1096 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1097
1098 if (MemVT.isScalableVector()) {
1099 SDNodeFlags Flags;
1100 SDValue BytesIncrement = DAG.getVScale(
1101 DL, Ptr.getValueType(),
1102 APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1103 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1104 Flags.setNoUnsignedWrap(true);
1105 if (ScaledOffset)
1106 *ScaledOffset += IncrementSize;
1107 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1108 Flags);
1109 } else {
1110 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1111 // Increment the pointer to the other half.
1112 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1113 }
1114 }
1115
SplitVecRes_BinOp(SDNode * N,SDValue & Lo,SDValue & Hi)1116 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
1117 SDValue &Hi) {
1118 SDValue LHSLo, LHSHi;
1119 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1120 SDValue RHSLo, RHSHi;
1121 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1122 SDLoc dl(N);
1123
1124 const SDNodeFlags Flags = N->getFlags();
1125 unsigned Opcode = N->getOpcode();
1126 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1127 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1128 }
1129
SplitVecRes_TernaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1130 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1131 SDValue &Hi) {
1132 SDValue Op0Lo, Op0Hi;
1133 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1134 SDValue Op1Lo, Op1Hi;
1135 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1136 SDValue Op2Lo, Op2Hi;
1137 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1138 SDLoc dl(N);
1139
1140 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo,
1141 Op2Lo, N->getFlags());
1142 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi,
1143 Op2Hi, N->getFlags());
1144 }
1145
SplitVecRes_FIX(SDNode * N,SDValue & Lo,SDValue & Hi)1146 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1147 SDValue LHSLo, LHSHi;
1148 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1149 SDValue RHSLo, RHSHi;
1150 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1151 SDLoc dl(N);
1152 SDValue Op2 = N->getOperand(2);
1153
1154 unsigned Opcode = N->getOpcode();
1155 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1156 N->getFlags());
1157 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1158 N->getFlags());
1159 }
1160
SplitVecRes_BITCAST(SDNode * N,SDValue & Lo,SDValue & Hi)1161 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1162 SDValue &Hi) {
1163 // We know the result is a vector. The input may be either a vector or a
1164 // scalar value.
1165 EVT LoVT, HiVT;
1166 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1167 SDLoc dl(N);
1168
1169 SDValue InOp = N->getOperand(0);
1170 EVT InVT = InOp.getValueType();
1171
1172 // Handle some special cases efficiently.
1173 switch (getTypeAction(InVT)) {
1174 case TargetLowering::TypeLegal:
1175 case TargetLowering::TypePromoteInteger:
1176 case TargetLowering::TypePromoteFloat:
1177 case TargetLowering::TypeSoftPromoteHalf:
1178 case TargetLowering::TypeSoftenFloat:
1179 case TargetLowering::TypeScalarizeVector:
1180 case TargetLowering::TypeWidenVector:
1181 break;
1182 case TargetLowering::TypeExpandInteger:
1183 case TargetLowering::TypeExpandFloat:
1184 // A scalar to vector conversion, where the scalar needs expansion.
1185 // If the vector is being split in two then we can just convert the
1186 // expanded pieces.
1187 if (LoVT == HiVT) {
1188 GetExpandedOp(InOp, Lo, Hi);
1189 if (DAG.getDataLayout().isBigEndian())
1190 std::swap(Lo, Hi);
1191 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1192 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1193 return;
1194 }
1195 break;
1196 case TargetLowering::TypeSplitVector:
1197 // If the input is a vector that needs to be split, convert each split
1198 // piece of the input now.
1199 GetSplitVector(InOp, Lo, Hi);
1200 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1201 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1202 return;
1203 case TargetLowering::TypeScalarizeScalableVector:
1204 report_fatal_error("Scalarization of scalable vectors is not supported.");
1205 }
1206
1207 // In the general case, convert the input to an integer and split it by hand.
1208 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1209 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1210 if (DAG.getDataLayout().isBigEndian())
1211 std::swap(LoIntVT, HiIntVT);
1212
1213 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1214
1215 if (DAG.getDataLayout().isBigEndian())
1216 std::swap(Lo, Hi);
1217 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1218 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1219 }
1220
SplitVecRes_BUILD_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1221 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1222 SDValue &Hi) {
1223 EVT LoVT, HiVT;
1224 SDLoc dl(N);
1225 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1226 unsigned LoNumElts = LoVT.getVectorNumElements();
1227 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1228 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1229
1230 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1231 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1232 }
1233
SplitVecRes_CONCAT_VECTORS(SDNode * N,SDValue & Lo,SDValue & Hi)1234 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1235 SDValue &Hi) {
1236 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1237 SDLoc dl(N);
1238 unsigned NumSubvectors = N->getNumOperands() / 2;
1239 if (NumSubvectors == 1) {
1240 Lo = N->getOperand(0);
1241 Hi = N->getOperand(1);
1242 return;
1243 }
1244
1245 EVT LoVT, HiVT;
1246 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1247
1248 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1249 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1250
1251 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1252 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1253 }
1254
SplitVecRes_EXTRACT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1255 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1256 SDValue &Hi) {
1257 SDValue Vec = N->getOperand(0);
1258 SDValue Idx = N->getOperand(1);
1259 SDLoc dl(N);
1260
1261 EVT LoVT, HiVT;
1262 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1263
1264 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1265 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1266 Hi = DAG.getNode(
1267 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1268 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1269 }
1270
SplitVecRes_INSERT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1271 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1272 SDValue &Hi) {
1273 SDValue Vec = N->getOperand(0);
1274 SDValue SubVec = N->getOperand(1);
1275 SDValue Idx = N->getOperand(2);
1276 SDLoc dl(N);
1277 GetSplitVector(Vec, Lo, Hi);
1278
1279 EVT VecVT = Vec.getValueType();
1280 EVT LoVT = Lo.getValueType();
1281 EVT SubVecVT = SubVec.getValueType();
1282 unsigned VecElems = VecVT.getVectorMinNumElements();
1283 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1284 unsigned LoElems = LoVT.getVectorMinNumElements();
1285
1286 // If we know the index is in the first half, and we know the subvector
1287 // doesn't cross the boundary between the halves, we can avoid spilling the
1288 // vector, and insert into the lower half of the split vector directly.
1289 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1290 if (IdxVal + SubElems <= LoElems) {
1291 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1292 return;
1293 }
1294 // Similarly if the subvector is fully in the high half, but mind that we
1295 // can't tell whether a fixed-length subvector is fully within the high half
1296 // of a scalable vector.
1297 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1298 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1299 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1300 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1301 return;
1302 }
1303
1304 // Spill the vector to the stack.
1305 // In cases where the vector is illegal it will be broken down into parts
1306 // and stored in parts - we should use the alignment for the smallest part.
1307 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1308 SDValue StackPtr =
1309 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1310 auto &MF = DAG.getMachineFunction();
1311 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1312 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1313
1314 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1315 SmallestAlign);
1316
1317 // Store the new subvector into the specified index.
1318 SDValue SubVecPtr =
1319 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1320 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1321 MachinePointerInfo::getUnknownStack(MF));
1322
1323 // Load the Lo part from the stack slot.
1324 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1325 SmallestAlign);
1326
1327 // Increment the pointer to the other part.
1328 auto *Load = cast<LoadSDNode>(Lo);
1329 MachinePointerInfo MPI = Load->getPointerInfo();
1330 IncrementPointer(Load, LoVT, MPI, StackPtr);
1331
1332 // Load the Hi part from the stack slot.
1333 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1334 }
1335
SplitVecRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)1336 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1337 SDValue &Hi) {
1338 SDLoc dl(N);
1339 GetSplitVector(N->getOperand(0), Lo, Hi);
1340 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1341 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1342 }
1343
SplitVecRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1344 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1345 SDValue &Hi) {
1346 SDValue LHSLo, LHSHi;
1347 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1348 SDLoc DL(N);
1349
1350 SDValue RHSLo, RHSHi;
1351 SDValue RHS = N->getOperand(1);
1352 EVT RHSVT = RHS.getValueType();
1353 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1354 GetSplitVector(RHS, RHSLo, RHSHi);
1355 else
1356 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1357
1358
1359 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1360 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1361 }
1362
SplitVecRes_InregOp(SDNode * N,SDValue & Lo,SDValue & Hi)1363 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1364 SDValue &Hi) {
1365 SDValue LHSLo, LHSHi;
1366 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1367 SDLoc dl(N);
1368
1369 EVT LoVT, HiVT;
1370 std::tie(LoVT, HiVT) =
1371 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1372
1373 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1374 DAG.getValueType(LoVT));
1375 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1376 DAG.getValueType(HiVT));
1377 }
1378
SplitVecRes_ExtVecInRegOp(SDNode * N,SDValue & Lo,SDValue & Hi)1379 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1380 SDValue &Hi) {
1381 unsigned Opcode = N->getOpcode();
1382 SDValue N0 = N->getOperand(0);
1383
1384 SDLoc dl(N);
1385 SDValue InLo, InHi;
1386
1387 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1388 GetSplitVector(N0, InLo, InHi);
1389 else
1390 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1391
1392 EVT InLoVT = InLo.getValueType();
1393 unsigned InNumElements = InLoVT.getVectorNumElements();
1394
1395 EVT OutLoVT, OutHiVT;
1396 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1397 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1398 assert((2 * OutNumElements) <= InNumElements &&
1399 "Illegal extend vector in reg split");
1400
1401 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1402 // input vector (i.e. we only use InLo):
1403 // OutLo will extend the first OutNumElements from InLo.
1404 // OutHi will extend the next OutNumElements from InLo.
1405
1406 // Shuffle the elements from InLo for OutHi into the bottom elements to
1407 // create a 'fake' InHi.
1408 SmallVector<int, 8> SplitHi(InNumElements, -1);
1409 for (unsigned i = 0; i != OutNumElements; ++i)
1410 SplitHi[i] = i + OutNumElements;
1411 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1412
1413 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1414 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1415 }
1416
SplitVecRes_StrictFPOp(SDNode * N,SDValue & Lo,SDValue & Hi)1417 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1418 SDValue &Hi) {
1419 unsigned NumOps = N->getNumOperands();
1420 SDValue Chain = N->getOperand(0);
1421 EVT LoVT, HiVT;
1422 SDLoc dl(N);
1423 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1424
1425 SmallVector<SDValue, 4> OpsLo(NumOps);
1426 SmallVector<SDValue, 4> OpsHi(NumOps);
1427
1428 // The Chain is the first operand.
1429 OpsLo[0] = Chain;
1430 OpsHi[0] = Chain;
1431
1432 // Now process the remaining operands.
1433 for (unsigned i = 1; i < NumOps; ++i) {
1434 SDValue Op = N->getOperand(i);
1435 SDValue OpLo = Op;
1436 SDValue OpHi = Op;
1437
1438 EVT InVT = Op.getValueType();
1439 if (InVT.isVector()) {
1440 // If the input also splits, handle it directly for a
1441 // compile time speedup. Otherwise split it by hand.
1442 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1443 GetSplitVector(Op, OpLo, OpHi);
1444 else
1445 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1446 }
1447
1448 OpsLo[i] = OpLo;
1449 OpsHi[i] = OpHi;
1450 }
1451
1452 EVT LoValueVTs[] = {LoVT, MVT::Other};
1453 EVT HiValueVTs[] = {HiVT, MVT::Other};
1454 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1455 N->getFlags());
1456 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1457 N->getFlags());
1458
1459 // Build a factor node to remember that this Op is independent of the
1460 // other one.
1461 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1462 Lo.getValue(1), Hi.getValue(1));
1463
1464 // Legalize the chain result - switch anything that used the old chain to
1465 // use the new one.
1466 ReplaceValueWith(SDValue(N, 1), Chain);
1467 }
1468
UnrollVectorOp_StrictFP(SDNode * N,unsigned ResNE)1469 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1470 SDValue Chain = N->getOperand(0);
1471 EVT VT = N->getValueType(0);
1472 unsigned NE = VT.getVectorNumElements();
1473 EVT EltVT = VT.getVectorElementType();
1474 SDLoc dl(N);
1475
1476 SmallVector<SDValue, 8> Scalars;
1477 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1478
1479 // If ResNE is 0, fully unroll the vector op.
1480 if (ResNE == 0)
1481 ResNE = NE;
1482 else if (NE > ResNE)
1483 NE = ResNE;
1484
1485 //The results of each unrolled operation, including the chain.
1486 EVT ChainVTs[] = {EltVT, MVT::Other};
1487 SmallVector<SDValue, 8> Chains;
1488
1489 unsigned i;
1490 for (i = 0; i != NE; ++i) {
1491 Operands[0] = Chain;
1492 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1493 SDValue Operand = N->getOperand(j);
1494 EVT OperandVT = Operand.getValueType();
1495 if (OperandVT.isVector()) {
1496 EVT OperandEltVT = OperandVT.getVectorElementType();
1497 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1498 Operand, DAG.getVectorIdxConstant(i, dl));
1499 } else {
1500 Operands[j] = Operand;
1501 }
1502 }
1503 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1504 Scalar.getNode()->setFlags(N->getFlags());
1505
1506 //Add in the scalar as well as its chain value to the
1507 //result vectors.
1508 Scalars.push_back(Scalar);
1509 Chains.push_back(Scalar.getValue(1));
1510 }
1511
1512 for (; i < ResNE; ++i)
1513 Scalars.push_back(DAG.getUNDEF(EltVT));
1514
1515 // Build a new factor node to connect the chain back together.
1516 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1517 ReplaceValueWith(SDValue(N, 1), Chain);
1518
1519 // Create a new BUILD_VECTOR node
1520 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1521 return DAG.getBuildVector(VecVT, dl, Scalars);
1522 }
1523
SplitVecRes_OverflowOp(SDNode * N,unsigned ResNo,SDValue & Lo,SDValue & Hi)1524 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1525 SDValue &Lo, SDValue &Hi) {
1526 SDLoc dl(N);
1527 EVT ResVT = N->getValueType(0);
1528 EVT OvVT = N->getValueType(1);
1529 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1530 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1531 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1532
1533 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1534 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1535 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1536 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1537 } else {
1538 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1539 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1540 }
1541
1542 unsigned Opcode = N->getOpcode();
1543 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1544 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1545 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1546 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1547 LoNode->setFlags(N->getFlags());
1548 HiNode->setFlags(N->getFlags());
1549
1550 Lo = SDValue(LoNode, ResNo);
1551 Hi = SDValue(HiNode, ResNo);
1552
1553 // Replace the other vector result not being explicitly split here.
1554 unsigned OtherNo = 1 - ResNo;
1555 EVT OtherVT = N->getValueType(OtherNo);
1556 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1557 SetSplitVector(SDValue(N, OtherNo),
1558 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1559 } else {
1560 SDValue OtherVal = DAG.getNode(
1561 ISD::CONCAT_VECTORS, dl, OtherVT,
1562 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1563 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1564 }
1565 }
1566
SplitVecRes_INSERT_VECTOR_ELT(SDNode * N,SDValue & Lo,SDValue & Hi)1567 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1568 SDValue &Hi) {
1569 SDValue Vec = N->getOperand(0);
1570 SDValue Elt = N->getOperand(1);
1571 SDValue Idx = N->getOperand(2);
1572 SDLoc dl(N);
1573 GetSplitVector(Vec, Lo, Hi);
1574
1575 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1576 unsigned IdxVal = CIdx->getZExtValue();
1577 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1578 if (IdxVal < LoNumElts) {
1579 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1580 Lo.getValueType(), Lo, Elt, Idx);
1581 return;
1582 } else if (!Vec.getValueType().isScalableVector()) {
1583 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1584 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1585 return;
1586 }
1587 }
1588
1589 // See if the target wants to custom expand this node.
1590 if (CustomLowerNode(N, N->getValueType(0), true))
1591 return;
1592
1593 // Make the vector elements byte-addressable if they aren't already.
1594 EVT VecVT = Vec.getValueType();
1595 EVT EltVT = VecVT.getVectorElementType();
1596 if (VecVT.getScalarSizeInBits() < 8) {
1597 EltVT = MVT::i8;
1598 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1599 VecVT.getVectorElementCount());
1600 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1601 // Extend the element type to match if needed.
1602 if (EltVT.bitsGT(Elt.getValueType()))
1603 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1604 }
1605
1606 // Spill the vector to the stack.
1607 // In cases where the vector is illegal it will be broken down into parts
1608 // and stored in parts - we should use the alignment for the smallest part.
1609 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1610 SDValue StackPtr =
1611 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1612 auto &MF = DAG.getMachineFunction();
1613 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1614 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1615
1616 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1617 SmallestAlign);
1618
1619 // Store the new element. This may be larger than the vector element type,
1620 // so use a truncating store.
1621 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1622 Store = DAG.getTruncStore(
1623 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1624 commonAlignment(SmallestAlign,
1625 EltVT.getFixedSizeInBits() / 8));
1626
1627 EVT LoVT, HiVT;
1628 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1629
1630 // Load the Lo part from the stack slot.
1631 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1632
1633 // Increment the pointer to the other part.
1634 auto Load = cast<LoadSDNode>(Lo);
1635 MachinePointerInfo MPI = Load->getPointerInfo();
1636 IncrementPointer(Load, LoVT, MPI, StackPtr);
1637
1638 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1639
1640 // If we adjusted the original type, we need to truncate the results.
1641 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1642 if (LoVT != Lo.getValueType())
1643 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1644 if (HiVT != Hi.getValueType())
1645 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1646 }
1647
SplitVecRes_STEP_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1648 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1649 SDValue &Hi) {
1650 EVT LoVT, HiVT;
1651 SDLoc dl(N);
1652 assert(N->getValueType(0).isScalableVector() &&
1653 "Only scalable vectors are supported for STEP_VECTOR");
1654 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1655 SDValue Step = N->getOperand(0);
1656
1657 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1658
1659 // Hi = Lo + (EltCnt * Step)
1660 EVT EltVT = Step.getValueType();
1661 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1662 SDValue StartOfHi =
1663 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1664 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1665 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1666
1667 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1668 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1669 }
1670
SplitVecRes_ScalarOp(SDNode * N,SDValue & Lo,SDValue & Hi)1671 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1672 SDValue &Hi) {
1673 EVT LoVT, HiVT;
1674 SDLoc dl(N);
1675 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1676 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1677 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1678 Hi = DAG.getUNDEF(HiVT);
1679 } else {
1680 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1681 Hi = Lo;
1682 }
1683 }
1684
SplitVecRes_LOAD(LoadSDNode * LD,SDValue & Lo,SDValue & Hi)1685 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1686 SDValue &Hi) {
1687 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1688 EVT LoVT, HiVT;
1689 SDLoc dl(LD);
1690 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1691
1692 ISD::LoadExtType ExtType = LD->getExtensionType();
1693 SDValue Ch = LD->getChain();
1694 SDValue Ptr = LD->getBasePtr();
1695 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1696 EVT MemoryVT = LD->getMemoryVT();
1697 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1698 AAMDNodes AAInfo = LD->getAAInfo();
1699
1700 EVT LoMemVT, HiMemVT;
1701 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1702
1703 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1704 SDValue Value, NewChain;
1705 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1706 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1707 ReplaceValueWith(SDValue(LD, 1), NewChain);
1708 return;
1709 }
1710
1711 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1712 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1713 MMOFlags, AAInfo);
1714
1715 MachinePointerInfo MPI;
1716 IncrementPointer(LD, LoMemVT, MPI, Ptr);
1717
1718 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1719 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1720
1721 // Build a factor node to remember that this load is independent of the
1722 // other one.
1723 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1724 Hi.getValue(1));
1725
1726 // Legalize the chain result - switch anything that used the old chain to
1727 // use the new one.
1728 ReplaceValueWith(SDValue(LD, 1), Ch);
1729 }
1730
SplitVecRes_MLOAD(MaskedLoadSDNode * MLD,SDValue & Lo,SDValue & Hi)1731 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1732 SDValue &Lo, SDValue &Hi) {
1733 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1734 EVT LoVT, HiVT;
1735 SDLoc dl(MLD);
1736 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1737
1738 SDValue Ch = MLD->getChain();
1739 SDValue Ptr = MLD->getBasePtr();
1740 SDValue Offset = MLD->getOffset();
1741 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1742 SDValue Mask = MLD->getMask();
1743 SDValue PassThru = MLD->getPassThru();
1744 Align Alignment = MLD->getOriginalAlign();
1745 ISD::LoadExtType ExtType = MLD->getExtensionType();
1746
1747 // Split Mask operand
1748 SDValue MaskLo, MaskHi;
1749 if (Mask.getOpcode() == ISD::SETCC) {
1750 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1751 } else {
1752 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1753 GetSplitVector(Mask, MaskLo, MaskHi);
1754 else
1755 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1756 }
1757
1758 EVT MemoryVT = MLD->getMemoryVT();
1759 EVT LoMemVT, HiMemVT;
1760 bool HiIsEmpty = false;
1761 std::tie(LoMemVT, HiMemVT) =
1762 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1763
1764 SDValue PassThruLo, PassThruHi;
1765 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1766 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1767 else
1768 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1769
1770 unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
1771 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1772 MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoSize, Alignment,
1773 MLD->getAAInfo(), MLD->getRanges());
1774
1775 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1776 MMO, MLD->getAddressingMode(), ExtType,
1777 MLD->isExpandingLoad());
1778
1779 if (HiIsEmpty) {
1780 // The hi masked load has zero storage size. We therefore simply set it to
1781 // the low masked load and rely on subsequent removal from the chain.
1782 Hi = Lo;
1783 } else {
1784 // Generate hi masked load.
1785 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1786 MLD->isExpandingLoad());
1787 unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
1788
1789 MachinePointerInfo MPI;
1790 if (LoMemVT.isScalableVector())
1791 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
1792 else
1793 MPI = MLD->getPointerInfo().getWithOffset(
1794 LoMemVT.getStoreSize().getFixedSize());
1795
1796 MMO = DAG.getMachineFunction().getMachineMemOperand(
1797 MPI, MachineMemOperand::MOLoad, HiSize, Alignment, MLD->getAAInfo(),
1798 MLD->getRanges());
1799
1800 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
1801 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
1802 MLD->isExpandingLoad());
1803 }
1804
1805 // Build a factor node to remember that this load is independent of the
1806 // other one.
1807 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1808 Hi.getValue(1));
1809
1810 // Legalize the chain result - switch anything that used the old chain to
1811 // use the new one.
1812 ReplaceValueWith(SDValue(MLD, 1), Ch);
1813
1814 }
1815
SplitVecRes_MGATHER(MaskedGatherSDNode * MGT,SDValue & Lo,SDValue & Hi)1816 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1817 SDValue &Lo, SDValue &Hi) {
1818 EVT LoVT, HiVT;
1819 SDLoc dl(MGT);
1820 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1821
1822 SDValue Ch = MGT->getChain();
1823 SDValue Ptr = MGT->getBasePtr();
1824 SDValue Mask = MGT->getMask();
1825 SDValue PassThru = MGT->getPassThru();
1826 SDValue Index = MGT->getIndex();
1827 SDValue Scale = MGT->getScale();
1828 EVT MemoryVT = MGT->getMemoryVT();
1829 Align Alignment = MGT->getOriginalAlign();
1830 ISD::LoadExtType ExtType = MGT->getExtensionType();
1831
1832 // Split Mask operand
1833 SDValue MaskLo, MaskHi;
1834 if (Mask.getOpcode() == ISD::SETCC) {
1835 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1836 } else {
1837 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1838 GetSplitVector(Mask, MaskLo, MaskHi);
1839 else
1840 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1841 }
1842
1843 EVT LoMemVT, HiMemVT;
1844 // Split MemoryVT
1845 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1846
1847 SDValue PassThruLo, PassThruHi;
1848 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1849 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1850 else
1851 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1852
1853 SDValue IndexHi, IndexLo;
1854 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1855 GetSplitVector(Index, IndexLo, IndexHi);
1856 else
1857 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1858
1859 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1860 MGT->getPointerInfo(), MachineMemOperand::MOLoad,
1861 MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
1862 MGT->getRanges());
1863
1864 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1865 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
1866 MMO, MGT->getIndexType(), ExtType);
1867
1868 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1869 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
1870 MMO, MGT->getIndexType(), ExtType);
1871
1872 // Build a factor node to remember that this load is independent of the
1873 // other one.
1874 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1875 Hi.getValue(1));
1876
1877 // Legalize the chain result - switch anything that used the old chain to
1878 // use the new one.
1879 ReplaceValueWith(SDValue(MGT, 1), Ch);
1880 }
1881
1882
SplitVecRes_SETCC(SDNode * N,SDValue & Lo,SDValue & Hi)1883 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1884 assert(N->getValueType(0).isVector() &&
1885 N->getOperand(0).getValueType().isVector() &&
1886 "Operand types must be vectors");
1887
1888 EVT LoVT, HiVT;
1889 SDLoc DL(N);
1890 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1891
1892 // If the input also splits, handle it directly. Otherwise split it by hand.
1893 SDValue LL, LH, RL, RH;
1894 if (getTypeAction(N->getOperand(0).getValueType()) ==
1895 TargetLowering::TypeSplitVector)
1896 GetSplitVector(N->getOperand(0), LL, LH);
1897 else
1898 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1899
1900 if (getTypeAction(N->getOperand(1).getValueType()) ==
1901 TargetLowering::TypeSplitVector)
1902 GetSplitVector(N->getOperand(1), RL, RH);
1903 else
1904 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1905
1906 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1907 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1908 }
1909
SplitVecRes_UnaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1910 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1911 SDValue &Hi) {
1912 // Get the dest types - they may not match the input types, e.g. int_to_fp.
1913 EVT LoVT, HiVT;
1914 SDLoc dl(N);
1915 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1916
1917 // If the input also splits, handle it directly for a compile time speedup.
1918 // Otherwise split it by hand.
1919 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1920 EVT InVT = N->getOperand(OpNo).getValueType();
1921 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1922 GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1923 else
1924 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1925
1926 if (N->getOpcode() == ISD::FP_ROUND) {
1927 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1),
1928 N->getFlags());
1929 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1),
1930 N->getFlags());
1931 } else {
1932 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags());
1933 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags());
1934 }
1935 }
1936
SplitVecRes_ExtendOp(SDNode * N,SDValue & Lo,SDValue & Hi)1937 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1938 SDValue &Hi) {
1939 SDLoc dl(N);
1940 EVT SrcVT = N->getOperand(0).getValueType();
1941 EVT DestVT = N->getValueType(0);
1942 EVT LoVT, HiVT;
1943 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1944
1945 // We can do better than a generic split operation if the extend is doing
1946 // more than just doubling the width of the elements and the following are
1947 // true:
1948 // - The number of vector elements is even,
1949 // - the source type is legal,
1950 // - the type of a split source is illegal,
1951 // - the type of an extended (by doubling element size) source is legal, and
1952 // - the type of that extended source when split is legal.
1953 //
1954 // This won't necessarily completely legalize the operation, but it will
1955 // more effectively move in the right direction and prevent falling down
1956 // to scalarization in many cases due to the input vector being split too
1957 // far.
1958 if (SrcVT.getVectorElementCount().isKnownEven() &&
1959 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
1960 LLVMContext &Ctx = *DAG.getContext();
1961 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1962 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1963
1964 EVT SplitLoVT, SplitHiVT;
1965 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1966 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1967 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1968 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1969 N->dump(&DAG); dbgs() << "\n");
1970 // Extend the source vector by one step.
1971 SDValue NewSrc =
1972 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1973 // Get the low and high halves of the new, extended one step, vector.
1974 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1975 // Extend those vector halves the rest of the way.
1976 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1977 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1978 return;
1979 }
1980 }
1981 // Fall back to the generic unary operator splitting otherwise.
1982 SplitVecRes_UnaryOp(N, Lo, Hi);
1983 }
1984
SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N,SDValue & Lo,SDValue & Hi)1985 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1986 SDValue &Lo, SDValue &Hi) {
1987 // The low and high parts of the original input give four input vectors.
1988 SDValue Inputs[4];
1989 SDLoc dl(N);
1990 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1991 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1992 EVT NewVT = Inputs[0].getValueType();
1993 unsigned NewElts = NewVT.getVectorNumElements();
1994
1995 // If Lo or Hi uses elements from at most two of the four input vectors, then
1996 // express it as a vector shuffle of those two inputs. Otherwise extract the
1997 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1998 SmallVector<int, 16> Ops;
1999 for (unsigned High = 0; High < 2; ++High) {
2000 SDValue &Output = High ? Hi : Lo;
2001
2002 // Build a shuffle mask for the output, discovering on the fly which
2003 // input vectors to use as shuffle operands (recorded in InputUsed).
2004 // If building a suitable shuffle vector proves too hard, then bail
2005 // out with useBuildVector set.
2006 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
2007 unsigned FirstMaskIdx = High * NewElts;
2008 bool useBuildVector = false;
2009 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2010 // The mask element. This indexes into the input.
2011 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2012
2013 // The input vector this mask element indexes into.
2014 unsigned Input = (unsigned)Idx / NewElts;
2015
2016 if (Input >= array_lengthof(Inputs)) {
2017 // The mask element does not index into any input vector.
2018 Ops.push_back(-1);
2019 continue;
2020 }
2021
2022 // Turn the index into an offset from the start of the input vector.
2023 Idx -= Input * NewElts;
2024
2025 // Find or create a shuffle vector operand to hold this input.
2026 unsigned OpNo;
2027 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
2028 if (InputUsed[OpNo] == Input) {
2029 // This input vector is already an operand.
2030 break;
2031 } else if (InputUsed[OpNo] == -1U) {
2032 // Create a new operand for this input vector.
2033 InputUsed[OpNo] = Input;
2034 break;
2035 }
2036 }
2037
2038 if (OpNo >= array_lengthof(InputUsed)) {
2039 // More than two input vectors used! Give up on trying to create a
2040 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
2041 useBuildVector = true;
2042 break;
2043 }
2044
2045 // Add the mask index for the new shuffle vector.
2046 Ops.push_back(Idx + OpNo * NewElts);
2047 }
2048
2049 if (useBuildVector) {
2050 EVT EltVT = NewVT.getVectorElementType();
2051 SmallVector<SDValue, 16> SVOps;
2052
2053 // Extract the input elements by hand.
2054 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2055 // The mask element. This indexes into the input.
2056 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2057
2058 // The input vector this mask element indexes into.
2059 unsigned Input = (unsigned)Idx / NewElts;
2060
2061 if (Input >= array_lengthof(Inputs)) {
2062 // The mask element is "undef" or indexes off the end of the input.
2063 SVOps.push_back(DAG.getUNDEF(EltVT));
2064 continue;
2065 }
2066
2067 // Turn the index into an offset from the start of the input vector.
2068 Idx -= Input * NewElts;
2069
2070 // Extract the vector element by hand.
2071 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2072 Inputs[Input],
2073 DAG.getVectorIdxConstant(Idx, dl)));
2074 }
2075
2076 // Construct the Lo/Hi output using a BUILD_VECTOR.
2077 Output = DAG.getBuildVector(NewVT, dl, SVOps);
2078 } else if (InputUsed[0] == -1U) {
2079 // No input vectors were used! The result is undefined.
2080 Output = DAG.getUNDEF(NewVT);
2081 } else {
2082 SDValue Op0 = Inputs[InputUsed[0]];
2083 // If only one input was used, use an undefined vector for the other.
2084 SDValue Op1 = InputUsed[1] == -1U ?
2085 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
2086 // At least one input vector was used. Create a new shuffle vector.
2087 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
2088 }
2089
2090 Ops.clear();
2091 }
2092 }
2093
SplitVecRes_VAARG(SDNode * N,SDValue & Lo,SDValue & Hi)2094 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2095 EVT OVT = N->getValueType(0);
2096 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2097 SDValue Chain = N->getOperand(0);
2098 SDValue Ptr = N->getOperand(1);
2099 SDValue SV = N->getOperand(2);
2100 SDLoc dl(N);
2101
2102 const Align Alignment =
2103 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2104
2105 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2106 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2107 Chain = Hi.getValue(1);
2108
2109 // Modified the chain - switch anything that used the old chain to use
2110 // the new one.
2111 ReplaceValueWith(SDValue(N, 1), Chain);
2112 }
2113
SplitVecRes_FP_TO_XINT_SAT(SDNode * N,SDValue & Lo,SDValue & Hi)2114 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2115 SDValue &Hi) {
2116 EVT DstVTLo, DstVTHi;
2117 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2118 SDLoc dl(N);
2119
2120 SDValue SrcLo, SrcHi;
2121 EVT SrcVT = N->getOperand(0).getValueType();
2122 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2123 GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2124 else
2125 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2126
2127 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2128 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2129 }
2130
2131 //===----------------------------------------------------------------------===//
2132 // Operand Vector Splitting
2133 //===----------------------------------------------------------------------===//
2134
2135 /// This method is called when the specified operand of the specified node is
2136 /// found to need vector splitting. At this point, all of the result types of
2137 /// the node are known to be legal, but other operands of the node may need
2138 /// legalization as well as the specified one.
SplitVectorOperand(SDNode * N,unsigned OpNo)2139 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2140 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2141 SDValue Res = SDValue();
2142
2143 // See if the target wants to custom split this node.
2144 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2145 return false;
2146
2147 switch (N->getOpcode()) {
2148 default:
2149 #ifndef NDEBUG
2150 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2151 N->dump(&DAG);
2152 dbgs() << "\n";
2153 #endif
2154 report_fatal_error("Do not know how to split this operator's "
2155 "operand!\n");
2156
2157 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2158 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2159 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2160 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2161 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2162 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2163 case ISD::TRUNCATE:
2164 Res = SplitVecOp_TruncateHelper(N);
2165 break;
2166 case ISD::STRICT_FP_ROUND:
2167 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2168 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2169 case ISD::STORE:
2170 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2171 break;
2172 case ISD::MSTORE:
2173 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2174 break;
2175 case ISD::MSCATTER:
2176 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
2177 break;
2178 case ISD::MGATHER:
2179 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
2180 break;
2181 case ISD::VSELECT:
2182 Res = SplitVecOp_VSELECT(N, OpNo);
2183 break;
2184 case ISD::STRICT_SINT_TO_FP:
2185 case ISD::STRICT_UINT_TO_FP:
2186 case ISD::SINT_TO_FP:
2187 case ISD::UINT_TO_FP:
2188 if (N->getValueType(0).bitsLT(
2189 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2190 Res = SplitVecOp_TruncateHelper(N);
2191 else
2192 Res = SplitVecOp_UnaryOp(N);
2193 break;
2194 case ISD::FP_TO_SINT_SAT:
2195 case ISD::FP_TO_UINT_SAT:
2196 Res = SplitVecOp_FP_TO_XINT_SAT(N);
2197 break;
2198 case ISD::FP_TO_SINT:
2199 case ISD::FP_TO_UINT:
2200 case ISD::STRICT_FP_TO_SINT:
2201 case ISD::STRICT_FP_TO_UINT:
2202 case ISD::STRICT_FP_EXTEND:
2203 case ISD::FP_EXTEND:
2204 case ISD::SIGN_EXTEND:
2205 case ISD::ZERO_EXTEND:
2206 case ISD::ANY_EXTEND:
2207 case ISD::FTRUNC:
2208 Res = SplitVecOp_UnaryOp(N);
2209 break;
2210
2211 case ISD::ANY_EXTEND_VECTOR_INREG:
2212 case ISD::SIGN_EXTEND_VECTOR_INREG:
2213 case ISD::ZERO_EXTEND_VECTOR_INREG:
2214 Res = SplitVecOp_ExtVecInRegOp(N);
2215 break;
2216
2217 case ISD::VECREDUCE_FADD:
2218 case ISD::VECREDUCE_FMUL:
2219 case ISD::VECREDUCE_ADD:
2220 case ISD::VECREDUCE_MUL:
2221 case ISD::VECREDUCE_AND:
2222 case ISD::VECREDUCE_OR:
2223 case ISD::VECREDUCE_XOR:
2224 case ISD::VECREDUCE_SMAX:
2225 case ISD::VECREDUCE_SMIN:
2226 case ISD::VECREDUCE_UMAX:
2227 case ISD::VECREDUCE_UMIN:
2228 case ISD::VECREDUCE_FMAX:
2229 case ISD::VECREDUCE_FMIN:
2230 Res = SplitVecOp_VECREDUCE(N, OpNo);
2231 break;
2232 case ISD::VECREDUCE_SEQ_FADD:
2233 case ISD::VECREDUCE_SEQ_FMUL:
2234 Res = SplitVecOp_VECREDUCE_SEQ(N);
2235 break;
2236 }
2237
2238 // If the result is null, the sub-method took care of registering results etc.
2239 if (!Res.getNode()) return false;
2240
2241 // If the result is N, the sub-method updated N in place. Tell the legalizer
2242 // core about this.
2243 if (Res.getNode() == N)
2244 return true;
2245
2246 if (N->isStrictFPOpcode())
2247 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2248 "Invalid operand expansion");
2249 else
2250 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2251 "Invalid operand expansion");
2252
2253 ReplaceValueWith(SDValue(N, 0), Res);
2254 return false;
2255 }
2256
SplitVecOp_VSELECT(SDNode * N,unsigned OpNo)2257 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2258 // The only possibility for an illegal operand is the mask, since result type
2259 // legalization would have handled this node already otherwise.
2260 assert(OpNo == 0 && "Illegal operand must be mask");
2261
2262 SDValue Mask = N->getOperand(0);
2263 SDValue Src0 = N->getOperand(1);
2264 SDValue Src1 = N->getOperand(2);
2265 EVT Src0VT = Src0.getValueType();
2266 SDLoc DL(N);
2267 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2268
2269 SDValue Lo, Hi;
2270 GetSplitVector(N->getOperand(0), Lo, Hi);
2271 assert(Lo.getValueType() == Hi.getValueType() &&
2272 "Lo and Hi have differing types");
2273
2274 EVT LoOpVT, HiOpVT;
2275 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2276 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2277
2278 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2279 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2280 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2281 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2282
2283 SDValue LoSelect =
2284 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2285 SDValue HiSelect =
2286 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2287
2288 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2289 }
2290
SplitVecOp_VECREDUCE(SDNode * N,unsigned OpNo)2291 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2292 EVT ResVT = N->getValueType(0);
2293 SDValue Lo, Hi;
2294 SDLoc dl(N);
2295
2296 SDValue VecOp = N->getOperand(OpNo);
2297 EVT VecVT = VecOp.getValueType();
2298 assert(VecVT.isVector() && "Can only split reduce vector operand");
2299 GetSplitVector(VecOp, Lo, Hi);
2300 EVT LoOpVT, HiOpVT;
2301 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2302
2303 // Use the appropriate scalar instruction on the split subvectors before
2304 // reducing the now partially reduced smaller vector.
2305 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2306 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2307 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2308 }
2309
SplitVecOp_VECREDUCE_SEQ(SDNode * N)2310 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2311 EVT ResVT = N->getValueType(0);
2312 SDValue Lo, Hi;
2313 SDLoc dl(N);
2314
2315 SDValue AccOp = N->getOperand(0);
2316 SDValue VecOp = N->getOperand(1);
2317 SDNodeFlags Flags = N->getFlags();
2318
2319 EVT VecVT = VecOp.getValueType();
2320 assert(VecVT.isVector() && "Can only split reduce vector operand");
2321 GetSplitVector(VecOp, Lo, Hi);
2322 EVT LoOpVT, HiOpVT;
2323 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2324
2325 // Reduce low half.
2326 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2327
2328 // Reduce high half, using low half result as initial value.
2329 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2330 }
2331
SplitVecOp_UnaryOp(SDNode * N)2332 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2333 // The result has a legal vector type, but the input needs splitting.
2334 EVT ResVT = N->getValueType(0);
2335 SDValue Lo, Hi;
2336 SDLoc dl(N);
2337 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2338 EVT InVT = Lo.getValueType();
2339
2340 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2341 InVT.getVectorElementCount());
2342
2343 if (N->isStrictFPOpcode()) {
2344 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2345 { N->getOperand(0), Lo });
2346 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2347 { N->getOperand(0), Hi });
2348
2349 // Build a factor node to remember that this operation is independent
2350 // of the other one.
2351 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2352 Hi.getValue(1));
2353
2354 // Legalize the chain result - switch anything that used the old chain to
2355 // use the new one.
2356 ReplaceValueWith(SDValue(N, 1), Ch);
2357 } else {
2358 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2359 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2360 }
2361
2362 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2363 }
2364
SplitVecOp_BITCAST(SDNode * N)2365 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2366 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2367 // end up being split all the way down to individual components. Convert the
2368 // split pieces into integers and reassemble.
2369 SDValue Lo, Hi;
2370 GetSplitVector(N->getOperand(0), Lo, Hi);
2371 Lo = BitConvertToInteger(Lo);
2372 Hi = BitConvertToInteger(Hi);
2373
2374 if (DAG.getDataLayout().isBigEndian())
2375 std::swap(Lo, Hi);
2376
2377 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2378 JoinIntegers(Lo, Hi));
2379 }
2380
SplitVecOp_INSERT_SUBVECTOR(SDNode * N,unsigned OpNo)2381 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
2382 unsigned OpNo) {
2383 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
2384 // We know that the result type is legal.
2385 EVT ResVT = N->getValueType(0);
2386
2387 SDValue Vec = N->getOperand(0);
2388 SDValue SubVec = N->getOperand(1);
2389 SDValue Idx = N->getOperand(2);
2390 SDLoc dl(N);
2391
2392 SDValue Lo, Hi;
2393 GetSplitVector(SubVec, Lo, Hi);
2394
2395 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2396 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2397
2398 SDValue FirstInsertion =
2399 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
2400 SDValue SecondInsertion =
2401 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
2402 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
2403
2404 return SecondInsertion;
2405 }
2406
SplitVecOp_EXTRACT_SUBVECTOR(SDNode * N)2407 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2408 // We know that the extracted result type is legal.
2409 EVT SubVT = N->getValueType(0);
2410
2411 SDValue Idx = N->getOperand(1);
2412 SDLoc dl(N);
2413 SDValue Lo, Hi;
2414
2415 if (SubVT.isScalableVector() !=
2416 N->getOperand(0).getValueType().isScalableVector())
2417 report_fatal_error("Extracting a fixed-length vector from an illegal "
2418 "scalable vector is not yet supported");
2419
2420 GetSplitVector(N->getOperand(0), Lo, Hi);
2421
2422 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2423 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2424
2425 if (IdxVal < LoElts) {
2426 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
2427 "Extracted subvector crosses vector split!");
2428 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2429 } else {
2430 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2431 DAG.getVectorIdxConstant(IdxVal - LoElts, dl));
2432 }
2433 }
2434
SplitVecOp_EXTRACT_VECTOR_ELT(SDNode * N)2435 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2436 SDValue Vec = N->getOperand(0);
2437 SDValue Idx = N->getOperand(1);
2438 EVT VecVT = Vec.getValueType();
2439
2440 if (isa<ConstantSDNode>(Idx)) {
2441 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2442
2443 SDValue Lo, Hi;
2444 GetSplitVector(Vec, Lo, Hi);
2445
2446 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2447
2448 if (IdxVal < LoElts)
2449 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2450 else if (!Vec.getValueType().isScalableVector())
2451 return SDValue(DAG.UpdateNodeOperands(N, Hi,
2452 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2453 Idx.getValueType())), 0);
2454 }
2455
2456 // See if the target wants to custom expand this node.
2457 if (CustomLowerNode(N, N->getValueType(0), true))
2458 return SDValue();
2459
2460 // Make the vector elements byte-addressable if they aren't already.
2461 SDLoc dl(N);
2462 EVT EltVT = VecVT.getVectorElementType();
2463 if (VecVT.getScalarSizeInBits() < 8) {
2464 EltVT = MVT::i8;
2465 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2466 VecVT.getVectorElementCount());
2467 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2468 }
2469
2470 // Store the vector to the stack.
2471 // In cases where the vector is illegal it will be broken down into parts
2472 // and stored in parts - we should use the alignment for the smallest part.
2473 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
2474 SDValue StackPtr =
2475 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
2476 auto &MF = DAG.getMachineFunction();
2477 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2478 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2479 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2480 SmallestAlign);
2481
2482 // Load back the required element.
2483 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2484
2485 // FIXME: This is to handle i1 vectors with elements promoted to i8.
2486 // i1 vector handling needs general improvement.
2487 if (N->getValueType(0).bitsLT(EltVT)) {
2488 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2489 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
2490 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2491 }
2492
2493 return DAG.getExtLoad(
2494 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2495 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
2496 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
2497 }
2498
SplitVecOp_ExtVecInRegOp(SDNode * N)2499 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2500 SDValue Lo, Hi;
2501
2502 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2503 // splitting the result has the same effect as splitting the input operand.
2504 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2505
2506 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2507 }
2508
SplitVecOp_MGATHER(MaskedGatherSDNode * MGT,unsigned OpNo)2509 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2510 unsigned OpNo) {
2511 EVT LoVT, HiVT;
2512 SDLoc dl(MGT);
2513 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2514
2515 SDValue Ch = MGT->getChain();
2516 SDValue Ptr = MGT->getBasePtr();
2517 SDValue Index = MGT->getIndex();
2518 SDValue Scale = MGT->getScale();
2519 SDValue Mask = MGT->getMask();
2520 SDValue PassThru = MGT->getPassThru();
2521 Align Alignment = MGT->getOriginalAlign();
2522 ISD::LoadExtType ExtType = MGT->getExtensionType();
2523
2524 SDValue MaskLo, MaskHi;
2525 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2526 // Split Mask operand
2527 GetSplitVector(Mask, MaskLo, MaskHi);
2528 else
2529 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2530
2531 EVT MemoryVT = MGT->getMemoryVT();
2532 EVT LoMemVT, HiMemVT;
2533 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2534
2535 SDValue PassThruLo, PassThruHi;
2536 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2537 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2538 else
2539 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2540
2541 SDValue IndexHi, IndexLo;
2542 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2543 GetSplitVector(Index, IndexLo, IndexHi);
2544 else
2545 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2546
2547 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2548 MGT->getPointerInfo(), MachineMemOperand::MOLoad,
2549 MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
2550 MGT->getRanges());
2551
2552 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2553 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2554 OpsLo, MMO, MGT->getIndexType(), ExtType);
2555
2556 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2557 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2558 OpsHi, MMO, MGT->getIndexType(), ExtType);
2559
2560 // Build a factor node to remember that this load is independent of the
2561 // other one.
2562 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2563 Hi.getValue(1));
2564
2565 // Legalize the chain result - switch anything that used the old chain to
2566 // use the new one.
2567 ReplaceValueWith(SDValue(MGT, 1), Ch);
2568
2569 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2570 Hi);
2571 ReplaceValueWith(SDValue(MGT, 0), Res);
2572 return SDValue();
2573 }
2574
SplitVecOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)2575 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2576 unsigned OpNo) {
2577 assert(N->isUnindexed() && "Indexed masked store of vector?");
2578 SDValue Ch = N->getChain();
2579 SDValue Ptr = N->getBasePtr();
2580 SDValue Offset = N->getOffset();
2581 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
2582 SDValue Mask = N->getMask();
2583 SDValue Data = N->getValue();
2584 Align Alignment = N->getOriginalAlign();
2585 SDLoc DL(N);
2586
2587 SDValue DataLo, DataHi;
2588 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2589 // Split Data operand
2590 GetSplitVector(Data, DataLo, DataHi);
2591 else
2592 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2593
2594 // Split Mask operand
2595 SDValue MaskLo, MaskHi;
2596 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2597 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2598 } else {
2599 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2600 GetSplitVector(Mask, MaskLo, MaskHi);
2601 else
2602 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2603 }
2604
2605 EVT MemoryVT = N->getMemoryVT();
2606 EVT LoMemVT, HiMemVT;
2607 bool HiIsEmpty = false;
2608 std::tie(LoMemVT, HiMemVT) =
2609 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
2610
2611 SDValue Lo, Hi, Res;
2612 unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
2613 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2614 N->getPointerInfo(), MachineMemOperand::MOStore, LoSize, Alignment,
2615 N->getAAInfo(), N->getRanges());
2616
2617 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
2618 N->getAddressingMode(), N->isTruncatingStore(),
2619 N->isCompressingStore());
2620
2621 if (HiIsEmpty) {
2622 // The hi masked store has zero storage size.
2623 // Only the lo masked store is needed.
2624 Res = Lo;
2625 } else {
2626
2627 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2628 N->isCompressingStore());
2629
2630 MachinePointerInfo MPI;
2631 if (LoMemVT.isScalableVector()) {
2632 Alignment = commonAlignment(
2633 Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
2634 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
2635 } else
2636 MPI = N->getPointerInfo().getWithOffset(
2637 LoMemVT.getStoreSize().getFixedSize());
2638
2639 unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
2640 MMO = DAG.getMachineFunction().getMachineMemOperand(
2641 MPI, MachineMemOperand::MOStore, HiSize, Alignment, N->getAAInfo(),
2642 N->getRanges());
2643
2644 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
2645 N->getAddressingMode(), N->isTruncatingStore(),
2646 N->isCompressingStore());
2647
2648 // Build a factor node to remember that this store is independent of the
2649 // other one.
2650 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2651 }
2652
2653 return Res;
2654 }
2655
SplitVecOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)2656 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2657 unsigned OpNo) {
2658 SDValue Ch = N->getChain();
2659 SDValue Ptr = N->getBasePtr();
2660 SDValue Mask = N->getMask();
2661 SDValue Index = N->getIndex();
2662 SDValue Scale = N->getScale();
2663 SDValue Data = N->getValue();
2664 EVT MemoryVT = N->getMemoryVT();
2665 Align Alignment = N->getOriginalAlign();
2666 SDLoc DL(N);
2667
2668 // Split all operands
2669
2670 EVT LoMemVT, HiMemVT;
2671 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2672
2673 SDValue DataLo, DataHi;
2674 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2675 // Split Data operand
2676 GetSplitVector(Data, DataLo, DataHi);
2677 else
2678 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2679
2680 // Split Mask operand
2681 SDValue MaskLo, MaskHi;
2682 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2683 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2684 } else {
2685 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2686 GetSplitVector(Mask, MaskLo, MaskHi);
2687 else
2688 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2689 }
2690
2691 SDValue IndexHi, IndexLo;
2692 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2693 GetSplitVector(Index, IndexLo, IndexHi);
2694 else
2695 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2696
2697 SDValue Lo;
2698 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2699 N->getPointerInfo(), MachineMemOperand::MOStore,
2700 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2701
2702 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2703 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT,
2704 DL, OpsLo, MMO, N->getIndexType(),
2705 N->isTruncatingStore());
2706
2707 // The order of the Scatter operation after split is well defined. The "Hi"
2708 // part comes after the "Lo". So these two operations should be chained one
2709 // after another.
2710 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2711 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT,
2712 DL, OpsHi, MMO, N->getIndexType(),
2713 N->isTruncatingStore());
2714 }
2715
SplitVecOp_STORE(StoreSDNode * N,unsigned OpNo)2716 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2717 assert(N->isUnindexed() && "Indexed store of vector?");
2718 assert(OpNo == 1 && "Can only split the stored value");
2719 SDLoc DL(N);
2720
2721 bool isTruncating = N->isTruncatingStore();
2722 SDValue Ch = N->getChain();
2723 SDValue Ptr = N->getBasePtr();
2724 EVT MemoryVT = N->getMemoryVT();
2725 Align Alignment = N->getOriginalAlign();
2726 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2727 AAMDNodes AAInfo = N->getAAInfo();
2728 SDValue Lo, Hi;
2729 GetSplitVector(N->getOperand(1), Lo, Hi);
2730
2731 EVT LoMemVT, HiMemVT;
2732 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2733
2734 // Scalarize if the split halves are not byte-sized.
2735 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2736 return TLI.scalarizeVectorStore(N, DAG);
2737
2738 if (isTruncating)
2739 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2740 Alignment, MMOFlags, AAInfo);
2741 else
2742 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2743 AAInfo);
2744
2745 MachinePointerInfo MPI;
2746 IncrementPointer(N, LoMemVT, MPI, Ptr);
2747
2748 if (isTruncating)
2749 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
2750 HiMemVT, Alignment, MMOFlags, AAInfo);
2751 else
2752 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
2753
2754 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2755 }
2756
SplitVecOp_CONCAT_VECTORS(SDNode * N)2757 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2758 SDLoc DL(N);
2759
2760 // The input operands all must have the same type, and we know the result
2761 // type is valid. Convert this to a buildvector which extracts all the
2762 // input elements.
2763 // TODO: If the input elements are power-two vectors, we could convert this to
2764 // a new CONCAT_VECTORS node with elements that are half-wide.
2765 SmallVector<SDValue, 32> Elts;
2766 EVT EltVT = N->getValueType(0).getVectorElementType();
2767 for (const SDValue &Op : N->op_values()) {
2768 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2769 i != e; ++i) {
2770 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2771 DAG.getVectorIdxConstant(i, DL)));
2772 }
2773 }
2774
2775 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2776 }
2777
SplitVecOp_TruncateHelper(SDNode * N)2778 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2779 // The result type is legal, but the input type is illegal. If splitting
2780 // ends up with the result type of each half still being legal, just
2781 // do that. If, however, that would result in an illegal result type,
2782 // we can try to get more clever with power-two vectors. Specifically,
2783 // split the input type, but also widen the result element size, then
2784 // concatenate the halves and truncate again. For example, consider a target
2785 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2786 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2787 // %inlo = v4i32 extract_subvector %in, 0
2788 // %inhi = v4i32 extract_subvector %in, 4
2789 // %lo16 = v4i16 trunc v4i32 %inlo
2790 // %hi16 = v4i16 trunc v4i32 %inhi
2791 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2792 // %res = v8i8 trunc v8i16 %in16
2793 //
2794 // Without this transform, the original truncate would end up being
2795 // scalarized, which is pretty much always a last resort.
2796 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
2797 SDValue InVec = N->getOperand(OpNo);
2798 EVT InVT = InVec->getValueType(0);
2799 EVT OutVT = N->getValueType(0);
2800 ElementCount NumElements = OutVT.getVectorElementCount();
2801 bool IsFloat = OutVT.isFloatingPoint();
2802
2803 unsigned InElementSize = InVT.getScalarSizeInBits();
2804 unsigned OutElementSize = OutVT.getScalarSizeInBits();
2805
2806 // Determine the split output VT. If its legal we can just split dirctly.
2807 EVT LoOutVT, HiOutVT;
2808 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2809 assert(LoOutVT == HiOutVT && "Unequal split?");
2810
2811 // If the input elements are only 1/2 the width of the result elements,
2812 // just use the normal splitting. Our trick only work if there's room
2813 // to split more than once.
2814 if (isTypeLegal(LoOutVT) ||
2815 InElementSize <= OutElementSize * 2)
2816 return SplitVecOp_UnaryOp(N);
2817 SDLoc DL(N);
2818
2819 // Don't touch if this will be scalarized.
2820 EVT FinalVT = InVT;
2821 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2822 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2823
2824 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2825 return SplitVecOp_UnaryOp(N);
2826
2827 // Get the split input vector.
2828 SDValue InLoVec, InHiVec;
2829 GetSplitVector(InVec, InLoVec, InHiVec);
2830
2831 // Truncate them to 1/2 the element size.
2832 //
2833 // This assumes the number of elements is a power of two; any vector that
2834 // isn't should be widened, not split.
2835 EVT HalfElementVT = IsFloat ?
2836 EVT::getFloatingPointVT(InElementSize/2) :
2837 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2838 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2839 NumElements.divideCoefficientBy(2));
2840
2841 SDValue HalfLo;
2842 SDValue HalfHi;
2843 SDValue Chain;
2844 if (N->isStrictFPOpcode()) {
2845 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2846 {N->getOperand(0), InLoVec});
2847 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2848 {N->getOperand(0), InHiVec});
2849 // Legalize the chain result - switch anything that used the old chain to
2850 // use the new one.
2851 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
2852 HalfHi.getValue(1));
2853 } else {
2854 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2855 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2856 }
2857
2858 // Concatenate them to get the full intermediate truncation result.
2859 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2860 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2861 HalfHi);
2862 // Now finish up by truncating all the way down to the original result
2863 // type. This should normally be something that ends up being legal directly,
2864 // but in theory if a target has very wide vectors and an annoyingly
2865 // restricted set of legal types, this split can chain to build things up.
2866
2867 if (N->isStrictFPOpcode()) {
2868 SDValue Res = DAG.getNode(
2869 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
2870 {Chain, InterVec,
2871 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
2872 // Relink the chain
2873 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2874 return Res;
2875 }
2876
2877 return IsFloat
2878 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2879 DAG.getTargetConstant(
2880 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2881 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2882 }
2883
SplitVecOp_VSETCC(SDNode * N)2884 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2885 assert(N->getValueType(0).isVector() &&
2886 N->getOperand(0).getValueType().isVector() &&
2887 "Operand types must be vectors");
2888 // The result has a legal vector type, but the input needs splitting.
2889 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2890 SDLoc DL(N);
2891 GetSplitVector(N->getOperand(0), Lo0, Hi0);
2892 GetSplitVector(N->getOperand(1), Lo1, Hi1);
2893 auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
2894
2895 LLVMContext &Context = *DAG.getContext();
2896 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
2897 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
2898
2899 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2900 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2901 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2902
2903 EVT OpVT = N->getOperand(0).getValueType();
2904 ISD::NodeType ExtendCode =
2905 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
2906 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
2907 }
2908
2909
SplitVecOp_FP_ROUND(SDNode * N)2910 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2911 // The result has a legal vector type, but the input needs splitting.
2912 EVT ResVT = N->getValueType(0);
2913 SDValue Lo, Hi;
2914 SDLoc DL(N);
2915 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2916 EVT InVT = Lo.getValueType();
2917
2918 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2919 InVT.getVectorElementCount());
2920
2921 if (N->isStrictFPOpcode()) {
2922 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2923 { N->getOperand(0), Lo, N->getOperand(2) });
2924 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2925 { N->getOperand(0), Hi, N->getOperand(2) });
2926 // Legalize the chain result - switch anything that used the old chain to
2927 // use the new one.
2928 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2929 Lo.getValue(1), Hi.getValue(1));
2930 ReplaceValueWith(SDValue(N, 1), NewChain);
2931 } else {
2932 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2933 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2934 }
2935
2936 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2937 }
2938
SplitVecOp_FCOPYSIGN(SDNode * N)2939 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2940 // The result (and the first input) has a legal vector type, but the second
2941 // input needs splitting.
2942 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2943 }
2944
SplitVecOp_FP_TO_XINT_SAT(SDNode * N)2945 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
2946 EVT ResVT = N->getValueType(0);
2947 SDValue Lo, Hi;
2948 SDLoc dl(N);
2949 GetSplitVector(N->getOperand(0), Lo, Hi);
2950 EVT InVT = Lo.getValueType();
2951
2952 EVT NewResVT =
2953 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2954 InVT.getVectorElementCount());
2955
2956 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
2957 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
2958
2959 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2960 }
2961
2962 //===----------------------------------------------------------------------===//
2963 // Result Vector Widening
2964 //===----------------------------------------------------------------------===//
2965
WidenVectorResult(SDNode * N,unsigned ResNo)2966 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2967 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2968 dbgs() << "\n");
2969
2970 // See if the target wants to custom widen this node.
2971 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2972 return;
2973
2974 SDValue Res = SDValue();
2975 switch (N->getOpcode()) {
2976 default:
2977 #ifndef NDEBUG
2978 dbgs() << "WidenVectorResult #" << ResNo << ": ";
2979 N->dump(&DAG);
2980 dbgs() << "\n";
2981 #endif
2982 llvm_unreachable("Do not know how to widen the result of this operator!");
2983
2984 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2985 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2986 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2987 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2988 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2989 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2990 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2991 case ISD::SPLAT_VECTOR:
2992 case ISD::SCALAR_TO_VECTOR:
2993 Res = WidenVecRes_ScalarOp(N);
2994 break;
2995 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2996 case ISD::VSELECT:
2997 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2998 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2999 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
3000 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
3001 case ISD::VECTOR_SHUFFLE:
3002 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
3003 break;
3004 case ISD::MLOAD:
3005 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3006 break;
3007 case ISD::MGATHER:
3008 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3009 break;
3010
3011 case ISD::ADD:
3012 case ISD::AND:
3013 case ISD::MUL:
3014 case ISD::MULHS:
3015 case ISD::MULHU:
3016 case ISD::OR:
3017 case ISD::SUB:
3018 case ISD::XOR:
3019 case ISD::SHL:
3020 case ISD::SRA:
3021 case ISD::SRL:
3022 case ISD::FMINNUM:
3023 case ISD::FMAXNUM:
3024 case ISD::FMINIMUM:
3025 case ISD::FMAXIMUM:
3026 case ISD::SMIN:
3027 case ISD::SMAX:
3028 case ISD::UMIN:
3029 case ISD::UMAX:
3030 case ISD::UADDSAT:
3031 case ISD::SADDSAT:
3032 case ISD::USUBSAT:
3033 case ISD::SSUBSAT:
3034 case ISD::SSHLSAT:
3035 case ISD::USHLSAT:
3036 case ISD::ROTL:
3037 case ISD::ROTR:
3038 Res = WidenVecRes_Binary(N);
3039 break;
3040
3041 case ISD::FADD:
3042 case ISD::FMUL:
3043 case ISD::FPOW:
3044 case ISD::FSUB:
3045 case ISD::FDIV:
3046 case ISD::FREM:
3047 case ISD::SDIV:
3048 case ISD::UDIV:
3049 case ISD::SREM:
3050 case ISD::UREM:
3051 Res = WidenVecRes_BinaryCanTrap(N);
3052 break;
3053
3054 case ISD::SMULFIX:
3055 case ISD::SMULFIXSAT:
3056 case ISD::UMULFIX:
3057 case ISD::UMULFIXSAT:
3058 // These are binary operations, but with an extra operand that shouldn't
3059 // be widened (the scale).
3060 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
3061 break;
3062
3063 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
3064 case ISD::STRICT_##DAGN:
3065 #include "llvm/IR/ConstrainedOps.def"
3066 Res = WidenVecRes_StrictFP(N);
3067 break;
3068
3069 case ISD::UADDO:
3070 case ISD::SADDO:
3071 case ISD::USUBO:
3072 case ISD::SSUBO:
3073 case ISD::UMULO:
3074 case ISD::SMULO:
3075 Res = WidenVecRes_OverflowOp(N, ResNo);
3076 break;
3077
3078 case ISD::FCOPYSIGN:
3079 Res = WidenVecRes_FCOPYSIGN(N);
3080 break;
3081
3082 case ISD::FPOWI:
3083 Res = WidenVecRes_POWI(N);
3084 break;
3085
3086 case ISD::ANY_EXTEND_VECTOR_INREG:
3087 case ISD::SIGN_EXTEND_VECTOR_INREG:
3088 case ISD::ZERO_EXTEND_VECTOR_INREG:
3089 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
3090 break;
3091
3092 case ISD::ANY_EXTEND:
3093 case ISD::FP_EXTEND:
3094 case ISD::FP_ROUND:
3095 case ISD::FP_TO_SINT:
3096 case ISD::FP_TO_UINT:
3097 case ISD::SIGN_EXTEND:
3098 case ISD::SINT_TO_FP:
3099 case ISD::TRUNCATE:
3100 case ISD::UINT_TO_FP:
3101 case ISD::ZERO_EXTEND:
3102 Res = WidenVecRes_Convert(N);
3103 break;
3104
3105 case ISD::FP_TO_SINT_SAT:
3106 case ISD::FP_TO_UINT_SAT:
3107 Res = WidenVecRes_FP_TO_XINT_SAT(N);
3108 break;
3109
3110 case ISD::FABS:
3111 case ISD::FCEIL:
3112 case ISD::FCOS:
3113 case ISD::FEXP:
3114 case ISD::FEXP2:
3115 case ISD::FFLOOR:
3116 case ISD::FLOG:
3117 case ISD::FLOG10:
3118 case ISD::FLOG2:
3119 case ISD::FNEARBYINT:
3120 case ISD::FRINT:
3121 case ISD::FROUND:
3122 case ISD::FROUNDEVEN:
3123 case ISD::FSIN:
3124 case ISD::FSQRT:
3125 case ISD::FTRUNC: {
3126 // We're going to widen this vector op to a legal type by padding with undef
3127 // elements. If the wide vector op is eventually going to be expanded to
3128 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3129 // libcalls on the undef elements.
3130 EVT VT = N->getValueType(0);
3131 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3132 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3133 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3134 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3135 break;
3136 }
3137 }
3138 // If the target has custom/legal support for the scalar FP intrinsic ops
3139 // (they are probably not destined to become libcalls), then widen those like
3140 // any other unary ops.
3141 LLVM_FALLTHROUGH;
3142
3143 case ISD::ABS:
3144 case ISD::BITREVERSE:
3145 case ISD::BSWAP:
3146 case ISD::CTLZ:
3147 case ISD::CTLZ_ZERO_UNDEF:
3148 case ISD::CTPOP:
3149 case ISD::CTTZ:
3150 case ISD::CTTZ_ZERO_UNDEF:
3151 case ISD::FNEG:
3152 case ISD::FREEZE:
3153 case ISD::ARITH_FENCE:
3154 case ISD::FCANONICALIZE:
3155 Res = WidenVecRes_Unary(N);
3156 break;
3157 case ISD::FMA:
3158 case ISD::FSHL:
3159 case ISD::FSHR:
3160 Res = WidenVecRes_Ternary(N);
3161 break;
3162 }
3163
3164 // If Res is null, the sub-method took care of registering the result.
3165 if (Res.getNode())
3166 SetWidenedVector(SDValue(N, ResNo), Res);
3167 }
3168
WidenVecRes_Ternary(SDNode * N)3169 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
3170 // Ternary op widening.
3171 SDLoc dl(N);
3172 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3173 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3174 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3175 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
3176 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
3177 }
3178
WidenVecRes_Binary(SDNode * N)3179 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
3180 // Binary op widening.
3181 SDLoc dl(N);
3182 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3183 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3184 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3185 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
3186 }
3187
WidenVecRes_BinaryWithExtraScalarOp(SDNode * N)3188 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
3189 // Binary op widening, but with an extra operand that shouldn't be widened.
3190 SDLoc dl(N);
3191 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3192 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3193 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3194 SDValue InOp3 = N->getOperand(2);
3195 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3196 N->getFlags());
3197 }
3198
3199 // Given a vector of operations that have been broken up to widen, see
3200 // if we can collect them together into the next widest legal VT. This
3201 // implementation is trap-safe.
CollectOpsToWiden(SelectionDAG & DAG,const TargetLowering & TLI,SmallVectorImpl<SDValue> & ConcatOps,unsigned ConcatEnd,EVT VT,EVT MaxVT,EVT WidenVT)3202 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
3203 SmallVectorImpl<SDValue> &ConcatOps,
3204 unsigned ConcatEnd, EVT VT, EVT MaxVT,
3205 EVT WidenVT) {
3206 // Check to see if we have a single operation with the widen type.
3207 if (ConcatEnd == 1) {
3208 VT = ConcatOps[0].getValueType();
3209 if (VT == WidenVT)
3210 return ConcatOps[0];
3211 }
3212
3213 SDLoc dl(ConcatOps[0]);
3214 EVT WidenEltVT = WidenVT.getVectorElementType();
3215
3216 // while (Some element of ConcatOps is not of type MaxVT) {
3217 // From the end of ConcatOps, collect elements of the same type and put
3218 // them into an op of the next larger supported type
3219 // }
3220 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
3221 int Idx = ConcatEnd - 1;
3222 VT = ConcatOps[Idx--].getValueType();
3223 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
3224 Idx--;
3225
3226 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
3227 EVT NextVT;
3228 do {
3229 NextSize *= 2;
3230 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
3231 } while (!TLI.isTypeLegal(NextVT));
3232
3233 if (!VT.isVector()) {
3234 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
3235 SDValue VecOp = DAG.getUNDEF(NextVT);
3236 unsigned NumToInsert = ConcatEnd - Idx - 1;
3237 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
3238 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
3239 ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
3240 }
3241 ConcatOps[Idx+1] = VecOp;
3242 ConcatEnd = Idx + 2;
3243 } else {
3244 // Vector type, create a CONCAT_VECTORS of type NextVT
3245 SDValue undefVec = DAG.getUNDEF(VT);
3246 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
3247 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3248 unsigned RealVals = ConcatEnd - Idx - 1;
3249 unsigned SubConcatEnd = 0;
3250 unsigned SubConcatIdx = Idx + 1;
3251 while (SubConcatEnd < RealVals)
3252 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
3253 while (SubConcatEnd < OpsToConcat)
3254 SubConcatOps[SubConcatEnd++] = undefVec;
3255 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
3256 NextVT, SubConcatOps);
3257 ConcatEnd = SubConcatIdx + 1;
3258 }
3259 }
3260
3261 // Check to see if we have a single operation with the widen type.
3262 if (ConcatEnd == 1) {
3263 VT = ConcatOps[0].getValueType();
3264 if (VT == WidenVT)
3265 return ConcatOps[0];
3266 }
3267
3268 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
3269 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
3270 if (NumOps != ConcatEnd ) {
3271 SDValue UndefVal = DAG.getUNDEF(MaxVT);
3272 for (unsigned j = ConcatEnd; j < NumOps; ++j)
3273 ConcatOps[j] = UndefVal;
3274 }
3275 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3276 makeArrayRef(ConcatOps.data(), NumOps));
3277 }
3278
WidenVecRes_BinaryCanTrap(SDNode * N)3279 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3280 // Binary op widening for operations that can trap.
3281 unsigned Opcode = N->getOpcode();
3282 SDLoc dl(N);
3283 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3284 EVT WidenEltVT = WidenVT.getVectorElementType();
3285 EVT VT = WidenVT;
3286 unsigned NumElts = VT.getVectorNumElements();
3287 const SDNodeFlags Flags = N->getFlags();
3288 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3289 NumElts = NumElts / 2;
3290 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3291 }
3292
3293 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
3294 // Operation doesn't trap so just widen as normal.
3295 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3296 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3297 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
3298 }
3299
3300 // No legal vector version so unroll the vector operation and then widen.
3301 if (NumElts == 1)
3302 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3303
3304 // Since the operation can trap, apply operation on the original vector.
3305 EVT MaxVT = VT;
3306 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3307 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3308 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3309
3310 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3311 unsigned ConcatEnd = 0; // Current ConcatOps index.
3312 int Idx = 0; // Current Idx into input vectors.
3313
3314 // NumElts := greatest legal vector size (at most WidenVT)
3315 // while (orig. vector has unhandled elements) {
3316 // take munches of size NumElts from the beginning and add to ConcatOps
3317 // NumElts := next smaller supported vector size or 1
3318 // }
3319 while (CurNumElts != 0) {
3320 while (CurNumElts >= NumElts) {
3321 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3322 DAG.getVectorIdxConstant(Idx, dl));
3323 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3324 DAG.getVectorIdxConstant(Idx, dl));
3325 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3326 Idx += NumElts;
3327 CurNumElts -= NumElts;
3328 }
3329 do {
3330 NumElts = NumElts / 2;
3331 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3332 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3333
3334 if (NumElts == 1) {
3335 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3336 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3337 InOp1, DAG.getVectorIdxConstant(Idx, dl));
3338 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3339 InOp2, DAG.getVectorIdxConstant(Idx, dl));
3340 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3341 EOp1, EOp2, Flags);
3342 }
3343 CurNumElts = 0;
3344 }
3345 }
3346
3347 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3348 }
3349
WidenVecRes_StrictFP(SDNode * N)3350 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3351 switch (N->getOpcode()) {
3352 case ISD::STRICT_FSETCC:
3353 case ISD::STRICT_FSETCCS:
3354 return WidenVecRes_STRICT_FSETCC(N);
3355 case ISD::STRICT_FP_EXTEND:
3356 case ISD::STRICT_FP_ROUND:
3357 case ISD::STRICT_FP_TO_SINT:
3358 case ISD::STRICT_FP_TO_UINT:
3359 case ISD::STRICT_SINT_TO_FP:
3360 case ISD::STRICT_UINT_TO_FP:
3361 return WidenVecRes_Convert_StrictFP(N);
3362 default:
3363 break;
3364 }
3365
3366 // StrictFP op widening for operations that can trap.
3367 unsigned NumOpers = N->getNumOperands();
3368 unsigned Opcode = N->getOpcode();
3369 SDLoc dl(N);
3370 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3371 EVT WidenEltVT = WidenVT.getVectorElementType();
3372 EVT VT = WidenVT;
3373 unsigned NumElts = VT.getVectorNumElements();
3374 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3375 NumElts = NumElts / 2;
3376 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3377 }
3378
3379 // No legal vector version so unroll the vector operation and then widen.
3380 if (NumElts == 1)
3381 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3382
3383 // Since the operation can trap, apply operation on the original vector.
3384 EVT MaxVT = VT;
3385 SmallVector<SDValue, 4> InOps;
3386 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3387
3388 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3389 SmallVector<SDValue, 16> Chains;
3390 unsigned ConcatEnd = 0; // Current ConcatOps index.
3391 int Idx = 0; // Current Idx into input vectors.
3392
3393 // The Chain is the first operand.
3394 InOps.push_back(N->getOperand(0));
3395
3396 // Now process the remaining operands.
3397 for (unsigned i = 1; i < NumOpers; ++i) {
3398 SDValue Oper = N->getOperand(i);
3399
3400 if (Oper.getValueType().isVector()) {
3401 assert(Oper.getValueType() == N->getValueType(0) &&
3402 "Invalid operand type to widen!");
3403 Oper = GetWidenedVector(Oper);
3404 }
3405
3406 InOps.push_back(Oper);
3407 }
3408
3409 // NumElts := greatest legal vector size (at most WidenVT)
3410 // while (orig. vector has unhandled elements) {
3411 // take munches of size NumElts from the beginning and add to ConcatOps
3412 // NumElts := next smaller supported vector size or 1
3413 // }
3414 while (CurNumElts != 0) {
3415 while (CurNumElts >= NumElts) {
3416 SmallVector<SDValue, 4> EOps;
3417
3418 for (unsigned i = 0; i < NumOpers; ++i) {
3419 SDValue Op = InOps[i];
3420
3421 if (Op.getValueType().isVector())
3422 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3423 DAG.getVectorIdxConstant(Idx, dl));
3424
3425 EOps.push_back(Op);
3426 }
3427
3428 EVT OperVT[] = {VT, MVT::Other};
3429 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3430 ConcatOps[ConcatEnd++] = Oper;
3431 Chains.push_back(Oper.getValue(1));
3432 Idx += NumElts;
3433 CurNumElts -= NumElts;
3434 }
3435 do {
3436 NumElts = NumElts / 2;
3437 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3438 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3439
3440 if (NumElts == 1) {
3441 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3442 SmallVector<SDValue, 4> EOps;
3443
3444 for (unsigned i = 0; i < NumOpers; ++i) {
3445 SDValue Op = InOps[i];
3446
3447 if (Op.getValueType().isVector())
3448 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3449 DAG.getVectorIdxConstant(Idx, dl));
3450
3451 EOps.push_back(Op);
3452 }
3453
3454 EVT WidenVT[] = {WidenEltVT, MVT::Other};
3455 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3456 ConcatOps[ConcatEnd++] = Oper;
3457 Chains.push_back(Oper.getValue(1));
3458 }
3459 CurNumElts = 0;
3460 }
3461 }
3462
3463 // Build a factor node to remember all the Ops that have been created.
3464 SDValue NewChain;
3465 if (Chains.size() == 1)
3466 NewChain = Chains[0];
3467 else
3468 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3469 ReplaceValueWith(SDValue(N, 1), NewChain);
3470
3471 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3472 }
3473
WidenVecRes_OverflowOp(SDNode * N,unsigned ResNo)3474 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3475 SDLoc DL(N);
3476 EVT ResVT = N->getValueType(0);
3477 EVT OvVT = N->getValueType(1);
3478 EVT WideResVT, WideOvVT;
3479 SDValue WideLHS, WideRHS;
3480
3481 // TODO: This might result in a widen/split loop.
3482 if (ResNo == 0) {
3483 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3484 WideOvVT = EVT::getVectorVT(
3485 *DAG.getContext(), OvVT.getVectorElementType(),
3486 WideResVT.getVectorNumElements());
3487
3488 WideLHS = GetWidenedVector(N->getOperand(0));
3489 WideRHS = GetWidenedVector(N->getOperand(1));
3490 } else {
3491 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3492 WideResVT = EVT::getVectorVT(
3493 *DAG.getContext(), ResVT.getVectorElementType(),
3494 WideOvVT.getVectorNumElements());
3495
3496 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3497 WideLHS = DAG.getNode(
3498 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3499 N->getOperand(0), Zero);
3500 WideRHS = DAG.getNode(
3501 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3502 N->getOperand(1), Zero);
3503 }
3504
3505 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3506 SDNode *WideNode = DAG.getNode(
3507 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3508
3509 // Replace the other vector result not being explicitly widened here.
3510 unsigned OtherNo = 1 - ResNo;
3511 EVT OtherVT = N->getValueType(OtherNo);
3512 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3513 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3514 } else {
3515 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3516 SDValue OtherVal = DAG.getNode(
3517 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3518 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3519 }
3520
3521 return SDValue(WideNode, ResNo);
3522 }
3523
WidenVecRes_Convert(SDNode * N)3524 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3525 LLVMContext &Ctx = *DAG.getContext();
3526 SDValue InOp = N->getOperand(0);
3527 SDLoc DL(N);
3528
3529 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
3530 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3531
3532 EVT InVT = InOp.getValueType();
3533
3534 unsigned Opcode = N->getOpcode();
3535 const SDNodeFlags Flags = N->getFlags();
3536
3537 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
3538 // not equal that of WidenVT.
3539 if (N->getOpcode() == ISD::ZERO_EXTEND &&
3540 getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
3541 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
3542 WidenVT.getScalarSizeInBits()) {
3543 InOp = ZExtPromotedInteger(InOp);
3544 InVT = InOp.getValueType();
3545 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
3546 Opcode = ISD::TRUNCATE;
3547 }
3548
3549 EVT InEltVT = InVT.getVectorElementType();
3550 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenNumElts);
3551 unsigned InVTNumElts = InVT.getVectorNumElements();
3552
3553 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3554 InOp = GetWidenedVector(N->getOperand(0));
3555 InVT = InOp.getValueType();
3556 InVTNumElts = InVT.getVectorNumElements();
3557 if (InVTNumElts == WidenNumElts) {
3558 if (N->getNumOperands() == 1)
3559 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3560 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3561 }
3562 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3563 // If both input and result vector types are of same width, extend
3564 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3565 // accepts fewer elements in the result than in the input.
3566 if (Opcode == ISD::ANY_EXTEND)
3567 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3568 if (Opcode == ISD::SIGN_EXTEND)
3569 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3570 if (Opcode == ISD::ZERO_EXTEND)
3571 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3572 }
3573 }
3574
3575 if (TLI.isTypeLegal(InWidenVT)) {
3576 // Because the result and the input are different vector types, widening
3577 // the result could create a legal type but widening the input might make
3578 // it an illegal type that might lead to repeatedly splitting the input
3579 // and then widening it. To avoid this, we widen the input only if
3580 // it results in a legal type.
3581 if (WidenNumElts % InVTNumElts == 0) {
3582 // Widen the input and call convert on the widened input vector.
3583 unsigned NumConcat = WidenNumElts/InVTNumElts;
3584 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3585 Ops[0] = InOp;
3586 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3587 if (N->getNumOperands() == 1)
3588 return DAG.getNode(Opcode, DL, WidenVT, InVec);
3589 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3590 }
3591
3592 if (InVTNumElts % WidenNumElts == 0) {
3593 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3594 DAG.getVectorIdxConstant(0, DL));
3595 // Extract the input and convert the shorten input vector.
3596 if (N->getNumOperands() == 1)
3597 return DAG.getNode(Opcode, DL, WidenVT, InVal);
3598 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3599 }
3600 }
3601
3602 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3603 EVT EltVT = WidenVT.getVectorElementType();
3604 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3605 // Use the original element count so we don't do more scalar opts than
3606 // necessary.
3607 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3608 for (unsigned i=0; i < MinElts; ++i) {
3609 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3610 DAG.getVectorIdxConstant(i, DL));
3611 if (N->getNumOperands() == 1)
3612 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3613 else
3614 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3615 }
3616
3617 return DAG.getBuildVector(WidenVT, DL, Ops);
3618 }
3619
WidenVecRes_FP_TO_XINT_SAT(SDNode * N)3620 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
3621 SDLoc dl(N);
3622 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3623 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
3624
3625 SDValue Src = N->getOperand(0);
3626 EVT SrcVT = Src.getValueType();
3627
3628 // Also widen the input.
3629 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
3630 Src = GetWidenedVector(Src);
3631 SrcVT = Src.getValueType();
3632 }
3633
3634 // Input and output not widened to the same size, give up.
3635 if (WidenNumElts != SrcVT.getVectorElementCount())
3636 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
3637
3638 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
3639 }
3640
WidenVecRes_Convert_StrictFP(SDNode * N)3641 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3642 SDValue InOp = N->getOperand(1);
3643 SDLoc DL(N);
3644 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3645
3646 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3647 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3648
3649 EVT InVT = InOp.getValueType();
3650 EVT InEltVT = InVT.getVectorElementType();
3651
3652 unsigned Opcode = N->getOpcode();
3653
3654 // FIXME: Optimizations need to be implemented here.
3655
3656 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3657 EVT EltVT = WidenVT.getVectorElementType();
3658 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
3659 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3660 SmallVector<SDValue, 32> OpChains;
3661 // Use the original element count so we don't do more scalar opts than
3662 // necessary.
3663 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3664 for (unsigned i=0; i < MinElts; ++i) {
3665 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3666 DAG.getVectorIdxConstant(i, DL));
3667 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3668 OpChains.push_back(Ops[i].getValue(1));
3669 }
3670 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3671 ReplaceValueWith(SDValue(N, 1), NewChain);
3672
3673 return DAG.getBuildVector(WidenVT, DL, Ops);
3674 }
3675
WidenVecRes_EXTEND_VECTOR_INREG(SDNode * N)3676 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3677 unsigned Opcode = N->getOpcode();
3678 SDValue InOp = N->getOperand(0);
3679 SDLoc DL(N);
3680
3681 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3682 EVT WidenSVT = WidenVT.getVectorElementType();
3683 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3684
3685 EVT InVT = InOp.getValueType();
3686 EVT InSVT = InVT.getVectorElementType();
3687 unsigned InVTNumElts = InVT.getVectorNumElements();
3688
3689 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3690 InOp = GetWidenedVector(InOp);
3691 InVT = InOp.getValueType();
3692 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3693 switch (Opcode) {
3694 case ISD::ANY_EXTEND_VECTOR_INREG:
3695 case ISD::SIGN_EXTEND_VECTOR_INREG:
3696 case ISD::ZERO_EXTEND_VECTOR_INREG:
3697 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3698 }
3699 }
3700 }
3701
3702 // Unroll, extend the scalars and rebuild the vector.
3703 SmallVector<SDValue, 16> Ops;
3704 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3705 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3706 DAG.getVectorIdxConstant(i, DL));
3707 switch (Opcode) {
3708 case ISD::ANY_EXTEND_VECTOR_INREG:
3709 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3710 break;
3711 case ISD::SIGN_EXTEND_VECTOR_INREG:
3712 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3713 break;
3714 case ISD::ZERO_EXTEND_VECTOR_INREG:
3715 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3716 break;
3717 default:
3718 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3719 }
3720 Ops.push_back(Val);
3721 }
3722
3723 while (Ops.size() != WidenNumElts)
3724 Ops.push_back(DAG.getUNDEF(WidenSVT));
3725
3726 return DAG.getBuildVector(WidenVT, DL, Ops);
3727 }
3728
WidenVecRes_FCOPYSIGN(SDNode * N)3729 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3730 // If this is an FCOPYSIGN with same input types, we can treat it as a
3731 // normal (can trap) binary op.
3732 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3733 return WidenVecRes_BinaryCanTrap(N);
3734
3735 // If the types are different, fall back to unrolling.
3736 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3737 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3738 }
3739
WidenVecRes_POWI(SDNode * N)3740 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3741 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3742 SDValue InOp = GetWidenedVector(N->getOperand(0));
3743 SDValue ShOp = N->getOperand(1);
3744 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3745 }
3746
WidenVecRes_Unary(SDNode * N)3747 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3748 // Unary op widening.
3749 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3750 SDValue InOp = GetWidenedVector(N->getOperand(0));
3751 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3752 }
3753
WidenVecRes_InregOp(SDNode * N)3754 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3755 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3756 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3757 cast<VTSDNode>(N->getOperand(1))->getVT()
3758 .getVectorElementType(),
3759 WidenVT.getVectorNumElements());
3760 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3761 return DAG.getNode(N->getOpcode(), SDLoc(N),
3762 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3763 }
3764
WidenVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)3765 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3766 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3767 return GetWidenedVector(WidenVec);
3768 }
3769
WidenVecRes_BITCAST(SDNode * N)3770 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3771 SDValue InOp = N->getOperand(0);
3772 EVT InVT = InOp.getValueType();
3773 EVT VT = N->getValueType(0);
3774 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3775 SDLoc dl(N);
3776
3777 switch (getTypeAction(InVT)) {
3778 case TargetLowering::TypeLegal:
3779 break;
3780 case TargetLowering::TypeScalarizeScalableVector:
3781 report_fatal_error("Scalarization of scalable vectors is not supported.");
3782 case TargetLowering::TypePromoteInteger: {
3783 // If the incoming type is a vector that is being promoted, then
3784 // we know that the elements are arranged differently and that we
3785 // must perform the conversion using a stack slot.
3786 if (InVT.isVector())
3787 break;
3788
3789 // If the InOp is promoted to the same size, convert it. Otherwise,
3790 // fall out of the switch and widen the promoted input.
3791 SDValue NInOp = GetPromotedInteger(InOp);
3792 EVT NInVT = NInOp.getValueType();
3793 if (WidenVT.bitsEq(NInVT)) {
3794 // For big endian targets we need to shift the input integer or the
3795 // interesting bits will end up at the wrong place.
3796 if (DAG.getDataLayout().isBigEndian()) {
3797 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
3798 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
3799 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
3800 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
3801 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
3802 }
3803 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
3804 }
3805 InOp = NInOp;
3806 InVT = NInVT;
3807 break;
3808 }
3809 case TargetLowering::TypeSoftenFloat:
3810 case TargetLowering::TypePromoteFloat:
3811 case TargetLowering::TypeSoftPromoteHalf:
3812 case TargetLowering::TypeExpandInteger:
3813 case TargetLowering::TypeExpandFloat:
3814 case TargetLowering::TypeScalarizeVector:
3815 case TargetLowering::TypeSplitVector:
3816 break;
3817 case TargetLowering::TypeWidenVector:
3818 // If the InOp is widened to the same size, convert it. Otherwise, fall
3819 // out of the switch and widen the widened input.
3820 InOp = GetWidenedVector(InOp);
3821 InVT = InOp.getValueType();
3822 if (WidenVT.bitsEq(InVT))
3823 // The input widens to the same size. Convert to the widen value.
3824 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3825 break;
3826 }
3827
3828 unsigned WidenSize = WidenVT.getSizeInBits();
3829 unsigned InSize = InVT.getSizeInBits();
3830 // x86mmx is not an acceptable vector element type, so don't try.
3831 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3832 // Determine new input vector type. The new input vector type will use
3833 // the same element type (if its a vector) or use the input type as a
3834 // vector. It is the same size as the type to widen to.
3835 EVT NewInVT;
3836 unsigned NewNumElts = WidenSize / InSize;
3837 if (InVT.isVector()) {
3838 EVT InEltVT = InVT.getVectorElementType();
3839 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3840 WidenSize / InEltVT.getSizeInBits());
3841 } else {
3842 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3843 }
3844
3845 if (TLI.isTypeLegal(NewInVT)) {
3846 SDValue NewVec;
3847 if (InVT.isVector()) {
3848 // Because the result and the input are different vector types, widening
3849 // the result could create a legal type but widening the input might make
3850 // it an illegal type that might lead to repeatedly splitting the input
3851 // and then widening it. To avoid this, we widen the input only if
3852 // it results in a legal type.
3853 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3854 Ops[0] = InOp;
3855
3856 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3857 } else {
3858 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3859 }
3860 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3861 }
3862 }
3863
3864 return CreateStackStoreLoad(InOp, WidenVT);
3865 }
3866
WidenVecRes_BUILD_VECTOR(SDNode * N)3867 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3868 SDLoc dl(N);
3869 // Build a vector with undefined for the new nodes.
3870 EVT VT = N->getValueType(0);
3871
3872 // Integer BUILD_VECTOR operands may be larger than the node's vector element
3873 // type. The UNDEFs need to have the same type as the existing operands.
3874 EVT EltVT = N->getOperand(0).getValueType();
3875 unsigned NumElts = VT.getVectorNumElements();
3876
3877 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3878 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3879
3880 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3881 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3882 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3883
3884 return DAG.getBuildVector(WidenVT, dl, NewOps);
3885 }
3886
WidenVecRes_CONCAT_VECTORS(SDNode * N)3887 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3888 EVT InVT = N->getOperand(0).getValueType();
3889 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3890 SDLoc dl(N);
3891 unsigned NumOperands = N->getNumOperands();
3892
3893 bool InputWidened = false; // Indicates we need to widen the input.
3894 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3895 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
3896 unsigned NumInElts = InVT.getVectorMinNumElements();
3897 if (WidenNumElts % NumInElts == 0) {
3898 // Add undef vectors to widen to correct length.
3899 unsigned NumConcat = WidenNumElts / NumInElts;
3900 SDValue UndefVal = DAG.getUNDEF(InVT);
3901 SmallVector<SDValue, 16> Ops(NumConcat);
3902 for (unsigned i=0; i < NumOperands; ++i)
3903 Ops[i] = N->getOperand(i);
3904 for (unsigned i = NumOperands; i != NumConcat; ++i)
3905 Ops[i] = UndefVal;
3906 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3907 }
3908 } else {
3909 InputWidened = true;
3910 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3911 // The inputs and the result are widen to the same value.
3912 unsigned i;
3913 for (i=1; i < NumOperands; ++i)
3914 if (!N->getOperand(i).isUndef())
3915 break;
3916
3917 if (i == NumOperands)
3918 // Everything but the first operand is an UNDEF so just return the
3919 // widened first operand.
3920 return GetWidenedVector(N->getOperand(0));
3921
3922 if (NumOperands == 2) {
3923 assert(!WidenVT.isScalableVector() &&
3924 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
3925 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3926 unsigned NumInElts = InVT.getVectorNumElements();
3927
3928 // Replace concat of two operands with a shuffle.
3929 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3930 for (unsigned i = 0; i < NumInElts; ++i) {
3931 MaskOps[i] = i;
3932 MaskOps[i + NumInElts] = i + WidenNumElts;
3933 }
3934 return DAG.getVectorShuffle(WidenVT, dl,
3935 GetWidenedVector(N->getOperand(0)),
3936 GetWidenedVector(N->getOperand(1)),
3937 MaskOps);
3938 }
3939 }
3940 }
3941
3942 assert(!WidenVT.isScalableVector() &&
3943 "Cannot use build vectors to widen CONCAT_VECTOR result");
3944 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3945 unsigned NumInElts = InVT.getVectorNumElements();
3946
3947 // Fall back to use extracts and build vector.
3948 EVT EltVT = WidenVT.getVectorElementType();
3949 SmallVector<SDValue, 16> Ops(WidenNumElts);
3950 unsigned Idx = 0;
3951 for (unsigned i=0; i < NumOperands; ++i) {
3952 SDValue InOp = N->getOperand(i);
3953 if (InputWidened)
3954 InOp = GetWidenedVector(InOp);
3955 for (unsigned j = 0; j < NumInElts; ++j)
3956 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3957 DAG.getVectorIdxConstant(j, dl));
3958 }
3959 SDValue UndefVal = DAG.getUNDEF(EltVT);
3960 for (; Idx < WidenNumElts; ++Idx)
3961 Ops[Idx] = UndefVal;
3962 return DAG.getBuildVector(WidenVT, dl, Ops);
3963 }
3964
WidenVecRes_EXTRACT_SUBVECTOR(SDNode * N)3965 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3966 EVT VT = N->getValueType(0);
3967 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3968 SDValue InOp = N->getOperand(0);
3969 SDValue Idx = N->getOperand(1);
3970 SDLoc dl(N);
3971
3972 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3973 InOp = GetWidenedVector(InOp);
3974
3975 EVT InVT = InOp.getValueType();
3976
3977 // Check if we can just return the input vector after widening.
3978 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3979 if (IdxVal == 0 && InVT == WidenVT)
3980 return InOp;
3981
3982 if (VT.isScalableVector())
3983 report_fatal_error("Don't know how to widen the result of "
3984 "EXTRACT_SUBVECTOR for scalable vectors");
3985
3986 // Check if we can extract from the vector.
3987 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3988 unsigned InNumElts = InVT.getVectorNumElements();
3989 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3990 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3991
3992 // We could try widening the input to the right length but for now, extract
3993 // the original elements, fill the rest with undefs and build a vector.
3994 SmallVector<SDValue, 16> Ops(WidenNumElts);
3995 EVT EltVT = VT.getVectorElementType();
3996 unsigned NumElts = VT.getVectorNumElements();
3997 unsigned i;
3998 for (i = 0; i < NumElts; ++i)
3999 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4000 DAG.getVectorIdxConstant(IdxVal + i, dl));
4001
4002 SDValue UndefVal = DAG.getUNDEF(EltVT);
4003 for (; i < WidenNumElts; ++i)
4004 Ops[i] = UndefVal;
4005 return DAG.getBuildVector(WidenVT, dl, Ops);
4006 }
4007
WidenVecRes_INSERT_VECTOR_ELT(SDNode * N)4008 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
4009 SDValue InOp = GetWidenedVector(N->getOperand(0));
4010 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
4011 InOp.getValueType(), InOp,
4012 N->getOperand(1), N->getOperand(2));
4013 }
4014
WidenVecRes_LOAD(SDNode * N)4015 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
4016 LoadSDNode *LD = cast<LoadSDNode>(N);
4017 ISD::LoadExtType ExtType = LD->getExtensionType();
4018
4019 // A vector must always be stored in memory as-is, i.e. without any padding
4020 // between the elements, since various code depend on it, e.g. in the
4021 // handling of a bitcast of a vector type to int, which may be done with a
4022 // vector store followed by an integer load. A vector that does not have
4023 // elements that are byte-sized must therefore be stored as an integer
4024 // built out of the extracted vector elements.
4025 if (!LD->getMemoryVT().isByteSized()) {
4026 SDValue Value, NewChain;
4027 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
4028 ReplaceValueWith(SDValue(LD, 0), Value);
4029 ReplaceValueWith(SDValue(LD, 1), NewChain);
4030 return SDValue();
4031 }
4032
4033 SDValue Result;
4034 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
4035 if (ExtType != ISD::NON_EXTLOAD)
4036 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
4037 else
4038 Result = GenWidenVectorLoads(LdChain, LD);
4039
4040 // If we generate a single load, we can use that for the chain. Otherwise,
4041 // build a factor node to remember the multiple loads are independent and
4042 // chain to that.
4043 SDValue NewChain;
4044 if (LdChain.size() == 1)
4045 NewChain = LdChain[0];
4046 else
4047 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
4048
4049 // Modified the chain - switch anything that used the old chain to use
4050 // the new one.
4051 ReplaceValueWith(SDValue(N, 1), NewChain);
4052
4053 return Result;
4054 }
4055
WidenVecRes_MLOAD(MaskedLoadSDNode * N)4056 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
4057
4058 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
4059 SDValue Mask = N->getMask();
4060 EVT MaskVT = Mask.getValueType();
4061 SDValue PassThru = GetWidenedVector(N->getPassThru());
4062 ISD::LoadExtType ExtType = N->getExtensionType();
4063 SDLoc dl(N);
4064
4065 // The mask should be widened as well
4066 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4067 MaskVT.getVectorElementType(),
4068 WidenVT.getVectorNumElements());
4069 Mask = ModifyToType(Mask, WideMaskVT, true);
4070
4071 SDValue Res = DAG.getMaskedLoad(
4072 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
4073 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
4074 ExtType, N->isExpandingLoad());
4075 // Legalize the chain result - switch anything that used the old chain to
4076 // use the new one.
4077 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4078 return Res;
4079 }
4080
WidenVecRes_MGATHER(MaskedGatherSDNode * N)4081 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
4082
4083 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4084 SDValue Mask = N->getMask();
4085 EVT MaskVT = Mask.getValueType();
4086 SDValue PassThru = GetWidenedVector(N->getPassThru());
4087 SDValue Scale = N->getScale();
4088 unsigned NumElts = WideVT.getVectorNumElements();
4089 SDLoc dl(N);
4090
4091 // The mask should be widened as well
4092 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4093 MaskVT.getVectorElementType(),
4094 WideVT.getVectorNumElements());
4095 Mask = ModifyToType(Mask, WideMaskVT, true);
4096
4097 // Widen the Index operand
4098 SDValue Index = N->getIndex();
4099 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4100 Index.getValueType().getScalarType(),
4101 NumElts);
4102 Index = ModifyToType(Index, WideIndexVT);
4103 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
4104 Scale };
4105
4106 // Widen the MemoryType
4107 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4108 N->getMemoryVT().getScalarType(), NumElts);
4109 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
4110 WideMemVT, dl, Ops, N->getMemOperand(),
4111 N->getIndexType(), N->getExtensionType());
4112
4113 // Legalize the chain result - switch anything that used the old chain to
4114 // use the new one.
4115 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4116 return Res;
4117 }
4118
WidenVecRes_ScalarOp(SDNode * N)4119 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
4120 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4121 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
4122 }
4123
4124 // Return true is this is a SETCC node or a strict version of it.
isSETCCOp(unsigned Opcode)4125 static inline bool isSETCCOp(unsigned Opcode) {
4126 switch (Opcode) {
4127 case ISD::SETCC:
4128 case ISD::STRICT_FSETCC:
4129 case ISD::STRICT_FSETCCS:
4130 return true;
4131 }
4132 return false;
4133 }
4134
4135 // Return true if this is a node that could have two SETCCs as operands.
isLogicalMaskOp(unsigned Opcode)4136 static inline bool isLogicalMaskOp(unsigned Opcode) {
4137 switch (Opcode) {
4138 case ISD::AND:
4139 case ISD::OR:
4140 case ISD::XOR:
4141 return true;
4142 }
4143 return false;
4144 }
4145
4146 // If N is a SETCC or a strict variant of it, return the type
4147 // of the compare operands.
getSETCCOperandType(SDValue N)4148 static inline EVT getSETCCOperandType(SDValue N) {
4149 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4150 return N->getOperand(OpNo).getValueType();
4151 }
4152
4153 // This is used just for the assert in convertMask(). Check that this either
4154 // a SETCC or a previously handled SETCC by convertMask().
4155 #ifndef NDEBUG
isSETCCorConvertedSETCC(SDValue N)4156 static inline bool isSETCCorConvertedSETCC(SDValue N) {
4157 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
4158 N = N.getOperand(0);
4159 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
4160 for (unsigned i = 1; i < N->getNumOperands(); ++i)
4161 if (!N->getOperand(i)->isUndef())
4162 return false;
4163 N = N.getOperand(0);
4164 }
4165
4166 if (N.getOpcode() == ISD::TRUNCATE)
4167 N = N.getOperand(0);
4168 else if (N.getOpcode() == ISD::SIGN_EXTEND)
4169 N = N.getOperand(0);
4170
4171 if (isLogicalMaskOp(N.getOpcode()))
4172 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
4173 isSETCCorConvertedSETCC(N.getOperand(1));
4174
4175 return (isSETCCOp(N.getOpcode()) ||
4176 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
4177 }
4178 #endif
4179
4180 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
4181 // to ToMaskVT if needed with vector extension or truncation.
convertMask(SDValue InMask,EVT MaskVT,EVT ToMaskVT)4182 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
4183 EVT ToMaskVT) {
4184 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
4185 // FIXME: This code seems to be too restrictive, we might consider
4186 // generalizing it or dropping it.
4187 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
4188
4189 // Make a new Mask node, with a legal result VT.
4190 SDValue Mask;
4191 SmallVector<SDValue, 4> Ops;
4192 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
4193 Ops.push_back(InMask->getOperand(i));
4194 if (InMask->isStrictFPOpcode()) {
4195 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
4196 { MaskVT, MVT::Other }, Ops);
4197 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
4198 }
4199 else
4200 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
4201
4202 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
4203 // extend or truncate is needed.
4204 LLVMContext &Ctx = *DAG.getContext();
4205 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
4206 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
4207 if (MaskScalarBits < ToMaskScalBits) {
4208 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4209 MaskVT.getVectorNumElements());
4210 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
4211 } else if (MaskScalarBits > ToMaskScalBits) {
4212 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4213 MaskVT.getVectorNumElements());
4214 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
4215 }
4216
4217 assert(Mask->getValueType(0).getScalarSizeInBits() ==
4218 ToMaskVT.getScalarSizeInBits() &&
4219 "Mask should have the right element size by now.");
4220
4221 // Adjust Mask to the right number of elements.
4222 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
4223 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
4224 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
4225 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
4226 ZeroIdx);
4227 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
4228 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
4229 EVT SubVT = Mask->getValueType(0);
4230 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
4231 SubOps[0] = Mask;
4232 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
4233 }
4234
4235 assert((Mask->getValueType(0) == ToMaskVT) &&
4236 "A mask of ToMaskVT should have been produced by now.");
4237
4238 return Mask;
4239 }
4240
4241 // This method tries to handle some special cases for the vselect mask
4242 // and if needed adjusting the mask vector type to match that of the VSELECT.
4243 // Without it, many cases end up with scalarization of the SETCC, with many
4244 // unnecessary instructions.
WidenVSELECTMask(SDNode * N)4245 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
4246 LLVMContext &Ctx = *DAG.getContext();
4247 SDValue Cond = N->getOperand(0);
4248
4249 if (N->getOpcode() != ISD::VSELECT)
4250 return SDValue();
4251
4252 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
4253 return SDValue();
4254
4255 // If this is a splitted VSELECT that was previously already handled, do
4256 // nothing.
4257 EVT CondVT = Cond->getValueType(0);
4258 if (CondVT.getScalarSizeInBits() != 1)
4259 return SDValue();
4260
4261 EVT VSelVT = N->getValueType(0);
4262
4263 // This method can't handle scalable vector types.
4264 // FIXME: This support could be added in the future.
4265 if (VSelVT.isScalableVector())
4266 return SDValue();
4267
4268 // Only handle vector types which are a power of 2.
4269 if (!isPowerOf2_64(VSelVT.getSizeInBits()))
4270 return SDValue();
4271
4272 // Don't touch if this will be scalarized.
4273 EVT FinalVT = VSelVT;
4274 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
4275 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
4276
4277 if (FinalVT.getVectorNumElements() == 1)
4278 return SDValue();
4279
4280 // If there is support for an i1 vector mask, don't touch.
4281 if (isSETCCOp(Cond.getOpcode())) {
4282 EVT SetCCOpVT = getSETCCOperandType(Cond);
4283 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
4284 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
4285 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
4286 if (SetCCResVT.getScalarSizeInBits() == 1)
4287 return SDValue();
4288 } else if (CondVT.getScalarType() == MVT::i1) {
4289 // If there is support for an i1 vector mask (or only scalar i1 conditions),
4290 // don't touch.
4291 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
4292 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
4293
4294 if (CondVT.getScalarType() == MVT::i1)
4295 return SDValue();
4296 }
4297
4298 // Widen the vselect result type if needed.
4299 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector)
4300 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
4301
4302 // The mask of the VSELECT should have integer elements.
4303 EVT ToMaskVT = VSelVT;
4304 if (!ToMaskVT.getScalarType().isInteger())
4305 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
4306
4307 SDValue Mask;
4308 if (isSETCCOp(Cond->getOpcode())) {
4309 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
4310 Mask = convertMask(Cond, MaskVT, ToMaskVT);
4311 } else if (isLogicalMaskOp(Cond->getOpcode()) &&
4312 isSETCCOp(Cond->getOperand(0).getOpcode()) &&
4313 isSETCCOp(Cond->getOperand(1).getOpcode())) {
4314 // Cond is (AND/OR/XOR (SETCC, SETCC))
4315 SDValue SETCC0 = Cond->getOperand(0);
4316 SDValue SETCC1 = Cond->getOperand(1);
4317 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
4318 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
4319 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
4320 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
4321 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
4322 EVT MaskVT;
4323 // If the two SETCCs have different VTs, either extend/truncate one of
4324 // them to the other "towards" ToMaskVT, or truncate one and extend the
4325 // other to ToMaskVT.
4326 if (ScalarBits0 != ScalarBits1) {
4327 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
4328 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
4329 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
4330 MaskVT = WideVT;
4331 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
4332 MaskVT = NarrowVT;
4333 else
4334 MaskVT = ToMaskVT;
4335 } else
4336 // If the two SETCCs have the same VT, don't change it.
4337 MaskVT = VT0;
4338
4339 // Make new SETCCs and logical nodes.
4340 SETCC0 = convertMask(SETCC0, VT0, MaskVT);
4341 SETCC1 = convertMask(SETCC1, VT1, MaskVT);
4342 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
4343
4344 // Convert the logical op for VSELECT if needed.
4345 Mask = convertMask(Cond, MaskVT, ToMaskVT);
4346 } else
4347 return SDValue();
4348
4349 return Mask;
4350 }
4351
WidenVecRes_SELECT(SDNode * N)4352 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
4353 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4354 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4355
4356 SDValue Cond1 = N->getOperand(0);
4357 EVT CondVT = Cond1.getValueType();
4358 if (CondVT.isVector()) {
4359 if (SDValue WideCond = WidenVSELECTMask(N)) {
4360 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4361 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4362 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4363 return DAG.getNode(N->getOpcode(), SDLoc(N),
4364 WidenVT, WideCond, InOp1, InOp2);
4365 }
4366
4367 EVT CondEltVT = CondVT.getVectorElementType();
4368 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
4369 CondEltVT, WidenNumElts);
4370 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
4371 Cond1 = GetWidenedVector(Cond1);
4372
4373 // If we have to split the condition there is no point in widening the
4374 // select. This would result in an cycle of widening the select ->
4375 // widening the condition operand -> splitting the condition operand ->
4376 // splitting the select -> widening the select. Instead split this select
4377 // further and widen the resulting type.
4378 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
4379 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
4380 SDValue Res = ModifyToType(SplitSelect, WidenVT);
4381 return Res;
4382 }
4383
4384 if (Cond1.getValueType() != CondWidenVT)
4385 Cond1 = ModifyToType(Cond1, CondWidenVT);
4386 }
4387
4388 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4389 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4390 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4391 return DAG.getNode(N->getOpcode(), SDLoc(N),
4392 WidenVT, Cond1, InOp1, InOp2);
4393 }
4394
WidenVecRes_SELECT_CC(SDNode * N)4395 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
4396 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
4397 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
4398 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
4399 InOp1.getValueType(), N->getOperand(0),
4400 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
4401 }
4402
WidenVecRes_UNDEF(SDNode * N)4403 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
4404 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4405 return DAG.getUNDEF(WidenVT);
4406 }
4407
WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N)4408 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
4409 EVT VT = N->getValueType(0);
4410 SDLoc dl(N);
4411
4412 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4413 unsigned NumElts = VT.getVectorNumElements();
4414 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4415
4416 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4417 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4418
4419 // Adjust mask based on new input vector length.
4420 SmallVector<int, 16> NewMask;
4421 for (unsigned i = 0; i != NumElts; ++i) {
4422 int Idx = N->getMaskElt(i);
4423 if (Idx < (int)NumElts)
4424 NewMask.push_back(Idx);
4425 else
4426 NewMask.push_back(Idx - NumElts + WidenNumElts);
4427 }
4428 for (unsigned i = NumElts; i != WidenNumElts; ++i)
4429 NewMask.push_back(-1);
4430 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
4431 }
4432
WidenVecRes_SETCC(SDNode * N)4433 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4434 assert(N->getValueType(0).isVector() &&
4435 N->getOperand(0).getValueType().isVector() &&
4436 "Operands must be vectors");
4437 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4438 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4439
4440 SDValue InOp1 = N->getOperand(0);
4441 EVT InVT = InOp1.getValueType();
4442 assert(InVT.isVector() && "can not widen non-vector type");
4443 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
4444 InVT.getVectorElementType(), WidenNumElts);
4445
4446 // The input and output types often differ here, and it could be that while
4447 // we'd prefer to widen the result type, the input operands have been split.
4448 // In this case, we also need to split the result of this node as well.
4449 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
4450 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4451 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4452 return Res;
4453 }
4454
4455 // If the inputs also widen, handle them directly. Otherwise widen by hand.
4456 SDValue InOp2 = N->getOperand(1);
4457 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4458 InOp1 = GetWidenedVector(InOp1);
4459 InOp2 = GetWidenedVector(InOp2);
4460 } else {
4461 InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
4462 InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
4463 }
4464
4465 // Assume that the input and output will be widen appropriately. If not,
4466 // we will have to unroll it at some point.
4467 assert(InOp1.getValueType() == WidenInVT &&
4468 InOp2.getValueType() == WidenInVT &&
4469 "Input not widened to expected type!");
4470 (void)WidenInVT;
4471 return DAG.getNode(ISD::SETCC, SDLoc(N),
4472 WidenVT, InOp1, InOp2, N->getOperand(2));
4473 }
4474
WidenVecRes_STRICT_FSETCC(SDNode * N)4475 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
4476 assert(N->getValueType(0).isVector() &&
4477 N->getOperand(1).getValueType().isVector() &&
4478 "Operands must be vectors");
4479 EVT VT = N->getValueType(0);
4480 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4481 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4482 unsigned NumElts = VT.getVectorNumElements();
4483 EVT EltVT = VT.getVectorElementType();
4484
4485 SDLoc dl(N);
4486 SDValue Chain = N->getOperand(0);
4487 SDValue LHS = N->getOperand(1);
4488 SDValue RHS = N->getOperand(2);
4489 SDValue CC = N->getOperand(3);
4490 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
4491
4492 // Fully unroll and reassemble.
4493 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT));
4494 SmallVector<SDValue, 8> Chains(NumElts);
4495 for (unsigned i = 0; i != NumElts; ++i) {
4496 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
4497 DAG.getVectorIdxConstant(i, dl));
4498 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
4499 DAG.getVectorIdxConstant(i, dl));
4500
4501 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
4502 {Chain, LHSElem, RHSElem, CC});
4503 Chains[i] = Scalars[i].getValue(1);
4504 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
4505 DAG.getBoolConstant(true, dl, EltVT, VT),
4506 DAG.getBoolConstant(false, dl, EltVT, VT));
4507 }
4508
4509 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4510 ReplaceValueWith(SDValue(N, 1), NewChain);
4511
4512 return DAG.getBuildVector(WidenVT, dl, Scalars);
4513 }
4514
4515 //===----------------------------------------------------------------------===//
4516 // Widen Vector Operand
4517 //===----------------------------------------------------------------------===//
WidenVectorOperand(SDNode * N,unsigned OpNo)4518 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
4519 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
4520 dbgs() << "\n");
4521 SDValue Res = SDValue();
4522
4523 // See if the target wants to custom widen this node.
4524 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4525 return false;
4526
4527 switch (N->getOpcode()) {
4528 default:
4529 #ifndef NDEBUG
4530 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
4531 N->dump(&DAG);
4532 dbgs() << "\n";
4533 #endif
4534 llvm_unreachable("Do not know how to widen this operator's operand!");
4535
4536 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
4537 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
4538 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
4539 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
4540 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
4541 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
4542 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
4543 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
4544 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
4545 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
4546 case ISD::STRICT_FSETCC:
4547 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
4548 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
4549 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
4550
4551 case ISD::ANY_EXTEND:
4552 case ISD::SIGN_EXTEND:
4553 case ISD::ZERO_EXTEND:
4554 Res = WidenVecOp_EXTEND(N);
4555 break;
4556
4557 case ISD::FP_EXTEND:
4558 case ISD::STRICT_FP_EXTEND:
4559 case ISD::FP_ROUND:
4560 case ISD::STRICT_FP_ROUND:
4561 case ISD::FP_TO_SINT:
4562 case ISD::STRICT_FP_TO_SINT:
4563 case ISD::FP_TO_UINT:
4564 case ISD::STRICT_FP_TO_UINT:
4565 case ISD::SINT_TO_FP:
4566 case ISD::STRICT_SINT_TO_FP:
4567 case ISD::UINT_TO_FP:
4568 case ISD::STRICT_UINT_TO_FP:
4569 case ISD::TRUNCATE:
4570 Res = WidenVecOp_Convert(N);
4571 break;
4572
4573 case ISD::FP_TO_SINT_SAT:
4574 case ISD::FP_TO_UINT_SAT:
4575 Res = WidenVecOp_FP_TO_XINT_SAT(N);
4576 break;
4577
4578 case ISD::VECREDUCE_FADD:
4579 case ISD::VECREDUCE_FMUL:
4580 case ISD::VECREDUCE_ADD:
4581 case ISD::VECREDUCE_MUL:
4582 case ISD::VECREDUCE_AND:
4583 case ISD::VECREDUCE_OR:
4584 case ISD::VECREDUCE_XOR:
4585 case ISD::VECREDUCE_SMAX:
4586 case ISD::VECREDUCE_SMIN:
4587 case ISD::VECREDUCE_UMAX:
4588 case ISD::VECREDUCE_UMIN:
4589 case ISD::VECREDUCE_FMAX:
4590 case ISD::VECREDUCE_FMIN:
4591 Res = WidenVecOp_VECREDUCE(N);
4592 break;
4593 case ISD::VECREDUCE_SEQ_FADD:
4594 case ISD::VECREDUCE_SEQ_FMUL:
4595 Res = WidenVecOp_VECREDUCE_SEQ(N);
4596 break;
4597 }
4598
4599 // If Res is null, the sub-method took care of registering the result.
4600 if (!Res.getNode()) return false;
4601
4602 // If the result is N, the sub-method updated N in place. Tell the legalizer
4603 // core about this.
4604 if (Res.getNode() == N)
4605 return true;
4606
4607
4608 if (N->isStrictFPOpcode())
4609 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4610 "Invalid operand expansion");
4611 else
4612 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4613 "Invalid operand expansion");
4614
4615 ReplaceValueWith(SDValue(N, 0), Res);
4616 return false;
4617 }
4618
WidenVecOp_EXTEND(SDNode * N)4619 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4620 SDLoc DL(N);
4621 EVT VT = N->getValueType(0);
4622
4623 SDValue InOp = N->getOperand(0);
4624 assert(getTypeAction(InOp.getValueType()) ==
4625 TargetLowering::TypeWidenVector &&
4626 "Unexpected type action");
4627 InOp = GetWidenedVector(InOp);
4628 assert(VT.getVectorNumElements() <
4629 InOp.getValueType().getVectorNumElements() &&
4630 "Input wasn't widened!");
4631
4632 // We may need to further widen the operand until it has the same total
4633 // vector size as the result.
4634 EVT InVT = InOp.getValueType();
4635 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
4636 EVT InEltVT = InVT.getVectorElementType();
4637 for (EVT FixedVT : MVT::vector_valuetypes()) {
4638 EVT FixedEltVT = FixedVT.getVectorElementType();
4639 if (TLI.isTypeLegal(FixedVT) &&
4640 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
4641 FixedEltVT == InEltVT) {
4642 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
4643 "Not enough elements in the fixed type for the operand!");
4644 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
4645 "We can't have the same type as we started with!");
4646 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
4647 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT,
4648 DAG.getUNDEF(FixedVT), InOp,
4649 DAG.getVectorIdxConstant(0, DL));
4650 else
4651 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
4652 DAG.getVectorIdxConstant(0, DL));
4653 break;
4654 }
4655 }
4656 InVT = InOp.getValueType();
4657 if (InVT.getSizeInBits() != VT.getSizeInBits())
4658 // We couldn't find a legal vector type that was a widening of the input
4659 // and could be extended in-register to the result type, so we have to
4660 // scalarize.
4661 return WidenVecOp_Convert(N);
4662 }
4663
4664 // Use special DAG nodes to represent the operation of extending the
4665 // low lanes.
4666 switch (N->getOpcode()) {
4667 default:
4668 llvm_unreachable("Extend legalization on extend operation!");
4669 case ISD::ANY_EXTEND:
4670 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
4671 case ISD::SIGN_EXTEND:
4672 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
4673 case ISD::ZERO_EXTEND:
4674 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
4675 }
4676 }
4677
WidenVecOp_FCOPYSIGN(SDNode * N)4678 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4679 // The result (and first input) is legal, but the second input is illegal.
4680 // We can't do much to fix that, so just unroll and let the extracts off of
4681 // the second input be widened as needed later.
4682 return DAG.UnrollVectorOp(N);
4683 }
4684
WidenVecOp_Convert(SDNode * N)4685 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4686 // Since the result is legal and the input is illegal.
4687 EVT VT = N->getValueType(0);
4688 EVT EltVT = VT.getVectorElementType();
4689 SDLoc dl(N);
4690 unsigned NumElts = VT.getVectorNumElements();
4691 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
4692 assert(getTypeAction(InOp.getValueType()) ==
4693 TargetLowering::TypeWidenVector &&
4694 "Unexpected type action");
4695 InOp = GetWidenedVector(InOp);
4696 EVT InVT = InOp.getValueType();
4697 unsigned Opcode = N->getOpcode();
4698
4699 // See if a widened result type would be legal, if so widen the node.
4700 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
4701 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4702 InVT.getVectorNumElements());
4703 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
4704 SDValue Res;
4705 if (N->isStrictFPOpcode()) {
4706 if (Opcode == ISD::STRICT_FP_ROUND)
4707 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4708 { N->getOperand(0), InOp, N->getOperand(2) });
4709 else
4710 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4711 { N->getOperand(0), InOp });
4712 // Legalize the chain result - switch anything that used the old chain to
4713 // use the new one.
4714 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4715 } else {
4716 if (Opcode == ISD::FP_ROUND)
4717 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1));
4718 else
4719 Res = DAG.getNode(Opcode, dl, WideVT, InOp);
4720 }
4721 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
4722 DAG.getVectorIdxConstant(0, dl));
4723 }
4724
4725 EVT InEltVT = InVT.getVectorElementType();
4726
4727 // Unroll the convert into some scalar code and create a nasty build vector.
4728 SmallVector<SDValue, 16> Ops(NumElts);
4729 if (N->isStrictFPOpcode()) {
4730 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4731 SmallVector<SDValue, 32> OpChains;
4732 for (unsigned i=0; i < NumElts; ++i) {
4733 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4734 DAG.getVectorIdxConstant(i, dl));
4735 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
4736 OpChains.push_back(Ops[i].getValue(1));
4737 }
4738 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4739 ReplaceValueWith(SDValue(N, 1), NewChain);
4740 } else {
4741 for (unsigned i = 0; i < NumElts; ++i)
4742 Ops[i] = DAG.getNode(Opcode, dl, EltVT,
4743 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT,
4744 InOp, DAG.getVectorIdxConstant(i, dl)));
4745 }
4746
4747 return DAG.getBuildVector(VT, dl, Ops);
4748 }
4749
WidenVecOp_FP_TO_XINT_SAT(SDNode * N)4750 SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
4751 EVT DstVT = N->getValueType(0);
4752 SDValue Src = GetWidenedVector(N->getOperand(0));
4753 EVT SrcVT = Src.getValueType();
4754 ElementCount WideNumElts = SrcVT.getVectorElementCount();
4755 SDLoc dl(N);
4756
4757 // See if a widened result type would be legal, if so widen the node.
4758 EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(),
4759 DstVT.getVectorElementType(), WideNumElts);
4760 if (TLI.isTypeLegal(WideDstVT)) {
4761 SDValue Res =
4762 DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1));
4763 return DAG.getNode(
4764 ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res,
4765 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4766 }
4767
4768 // Give up and unroll.
4769 return DAG.UnrollVectorOp(N);
4770 }
4771
WidenVecOp_BITCAST(SDNode * N)4772 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4773 EVT VT = N->getValueType(0);
4774 SDValue InOp = GetWidenedVector(N->getOperand(0));
4775 EVT InWidenVT = InOp.getValueType();
4776 SDLoc dl(N);
4777
4778 // Check if we can convert between two legal vector types and extract.
4779 unsigned InWidenSize = InWidenVT.getSizeInBits();
4780 unsigned Size = VT.getSizeInBits();
4781 // x86mmx is not an acceptable vector element type, so don't try.
4782 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
4783 unsigned NewNumElts = InWidenSize / Size;
4784 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4785 if (TLI.isTypeLegal(NewVT)) {
4786 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4787 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
4788 DAG.getVectorIdxConstant(0, dl));
4789 }
4790 }
4791
4792 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
4793 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
4794 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
4795 // having to copy via memory.
4796 if (VT.isVector()) {
4797 EVT EltVT = VT.getVectorElementType();
4798 unsigned EltSize = EltVT.getSizeInBits();
4799 if (InWidenSize % EltSize == 0) {
4800 unsigned NewNumElts = InWidenSize / EltSize;
4801 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
4802 if (TLI.isTypeLegal(NewVT)) {
4803 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4804 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
4805 DAG.getVectorIdxConstant(0, dl));
4806 }
4807 }
4808 }
4809
4810 return CreateStackStoreLoad(InOp, VT);
4811 }
4812
WidenVecOp_CONCAT_VECTORS(SDNode * N)4813 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4814 EVT VT = N->getValueType(0);
4815 EVT EltVT = VT.getVectorElementType();
4816 EVT InVT = N->getOperand(0).getValueType();
4817 SDLoc dl(N);
4818
4819 // If the widen width for this operand is the same as the width of the concat
4820 // and all but the first operand is undef, just use the widened operand.
4821 unsigned NumOperands = N->getNumOperands();
4822 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4823 unsigned i;
4824 for (i = 1; i < NumOperands; ++i)
4825 if (!N->getOperand(i).isUndef())
4826 break;
4827
4828 if (i == NumOperands)
4829 return GetWidenedVector(N->getOperand(0));
4830 }
4831
4832 // Otherwise, fall back to a nasty build vector.
4833 unsigned NumElts = VT.getVectorNumElements();
4834 SmallVector<SDValue, 16> Ops(NumElts);
4835
4836 unsigned NumInElts = InVT.getVectorNumElements();
4837
4838 unsigned Idx = 0;
4839 for (unsigned i=0; i < NumOperands; ++i) {
4840 SDValue InOp = N->getOperand(i);
4841 assert(getTypeAction(InOp.getValueType()) ==
4842 TargetLowering::TypeWidenVector &&
4843 "Unexpected type action");
4844 InOp = GetWidenedVector(InOp);
4845 for (unsigned j = 0; j < NumInElts; ++j)
4846 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4847 DAG.getVectorIdxConstant(j, dl));
4848 }
4849 return DAG.getBuildVector(VT, dl, Ops);
4850 }
4851
WidenVecOp_INSERT_SUBVECTOR(SDNode * N)4852 SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
4853 SDValue SubVec = N->getOperand(1);
4854 SDValue InVec = N->getOperand(0);
4855
4856 if (getTypeAction(InVec.getValueType()) == TargetLowering::TypeWidenVector)
4857 InVec = GetWidenedVector(InVec);
4858
4859 if (getTypeAction(SubVec.getValueType()) == TargetLowering::TypeWidenVector)
4860 SubVec = GetWidenedVector(SubVec);
4861
4862 if (SubVec.getValueType() == InVec.getValueType() && InVec.isUndef() &&
4863 N->getConstantOperandVal(2) == 0)
4864 return SubVec;
4865
4866 report_fatal_error("Don't know how to widen the operands for "
4867 "INSERT_SUBVECTOR");
4868 }
4869
WidenVecOp_EXTRACT_SUBVECTOR(SDNode * N)4870 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4871 SDValue InOp = GetWidenedVector(N->getOperand(0));
4872 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4873 N->getValueType(0), InOp, N->getOperand(1));
4874 }
4875
WidenVecOp_EXTRACT_VECTOR_ELT(SDNode * N)4876 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4877 SDValue InOp = GetWidenedVector(N->getOperand(0));
4878 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4879 N->getValueType(0), InOp, N->getOperand(1));
4880 }
4881
WidenVecOp_STORE(SDNode * N)4882 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4883 // We have to widen the value, but we want only to store the original
4884 // vector type.
4885 StoreSDNode *ST = cast<StoreSDNode>(N);
4886
4887 if (!ST->getMemoryVT().getScalarType().isByteSized())
4888 return TLI.scalarizeVectorStore(ST, DAG);
4889
4890 if (ST->isTruncatingStore())
4891 return TLI.scalarizeVectorStore(ST, DAG);
4892
4893 SmallVector<SDValue, 16> StChain;
4894 GenWidenVectorStores(StChain, ST);
4895
4896 if (StChain.size() == 1)
4897 return StChain[0];
4898 else
4899 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4900 }
4901
WidenVecOp_MSTORE(SDNode * N,unsigned OpNo)4902 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4903 assert((OpNo == 1 || OpNo == 3) &&
4904 "Can widen only data or mask operand of mstore");
4905 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
4906 SDValue Mask = MST->getMask();
4907 EVT MaskVT = Mask.getValueType();
4908 SDValue StVal = MST->getValue();
4909 SDLoc dl(N);
4910
4911 if (OpNo == 1) {
4912 // Widen the value.
4913 StVal = GetWidenedVector(StVal);
4914
4915 // The mask should be widened as well.
4916 EVT WideVT = StVal.getValueType();
4917 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4918 MaskVT.getVectorElementType(),
4919 WideVT.getVectorNumElements());
4920 Mask = ModifyToType(Mask, WideMaskVT, true);
4921 } else {
4922 // Widen the mask.
4923 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4924 Mask = ModifyToType(Mask, WideMaskVT, true);
4925
4926 EVT ValueVT = StVal.getValueType();
4927 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4928 ValueVT.getVectorElementType(),
4929 WideMaskVT.getVectorNumElements());
4930 StVal = ModifyToType(StVal, WideVT);
4931 }
4932
4933 assert(Mask.getValueType().getVectorNumElements() ==
4934 StVal.getValueType().getVectorNumElements() &&
4935 "Mask and data vectors should have the same number of elements");
4936 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4937 MST->getOffset(), Mask, MST->getMemoryVT(),
4938 MST->getMemOperand(), MST->getAddressingMode(),
4939 false, MST->isCompressingStore());
4940 }
4941
WidenVecOp_MGATHER(SDNode * N,unsigned OpNo)4942 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4943 assert(OpNo == 4 && "Can widen only the index of mgather");
4944 auto *MG = cast<MaskedGatherSDNode>(N);
4945 SDValue DataOp = MG->getPassThru();
4946 SDValue Mask = MG->getMask();
4947 SDValue Scale = MG->getScale();
4948
4949 // Just widen the index. It's allowed to have extra elements.
4950 SDValue Index = GetWidenedVector(MG->getIndex());
4951
4952 SDLoc dl(N);
4953 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4954 Scale};
4955 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4956 MG->getMemOperand(), MG->getIndexType(),
4957 MG->getExtensionType());
4958 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4959 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4960 return SDValue();
4961 }
4962
WidenVecOp_MSCATTER(SDNode * N,unsigned OpNo)4963 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4964 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
4965 SDValue DataOp = MSC->getValue();
4966 SDValue Mask = MSC->getMask();
4967 SDValue Index = MSC->getIndex();
4968 SDValue Scale = MSC->getScale();
4969 EVT WideMemVT = MSC->getMemoryVT();
4970
4971 if (OpNo == 1) {
4972 DataOp = GetWidenedVector(DataOp);
4973 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
4974
4975 // Widen index.
4976 EVT IndexVT = Index.getValueType();
4977 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4978 IndexVT.getVectorElementType(), NumElts);
4979 Index = ModifyToType(Index, WideIndexVT);
4980
4981 // The mask should be widened as well.
4982 EVT MaskVT = Mask.getValueType();
4983 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4984 MaskVT.getVectorElementType(), NumElts);
4985 Mask = ModifyToType(Mask, WideMaskVT, true);
4986
4987 // Widen the MemoryType
4988 WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4989 MSC->getMemoryVT().getScalarType(), NumElts);
4990 } else if (OpNo == 4) {
4991 // Just widen the index. It's allowed to have extra elements.
4992 Index = GetWidenedVector(Index);
4993 } else
4994 llvm_unreachable("Can't widen this operand of mscatter");
4995
4996 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4997 Scale};
4998 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N),
4999 Ops, MSC->getMemOperand(), MSC->getIndexType(),
5000 MSC->isTruncatingStore());
5001 }
5002
WidenVecOp_SETCC(SDNode * N)5003 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
5004 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
5005 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
5006 SDLoc dl(N);
5007 EVT VT = N->getValueType(0);
5008
5009 // WARNING: In this code we widen the compare instruction with garbage.
5010 // This garbage may contain denormal floats which may be slow. Is this a real
5011 // concern ? Should we zero the unused lanes if this is a float compare ?
5012
5013 // Get a new SETCC node to compare the newly widened operands.
5014 // Only some of the compared elements are legal.
5015 EVT SVT = getSetCCResultType(InOp0.getValueType());
5016 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
5017 if (VT.getScalarType() == MVT::i1)
5018 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
5019 SVT.getVectorNumElements());
5020
5021 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
5022 SVT, InOp0, InOp1, N->getOperand(2));
5023
5024 // Extract the needed results from the result vector.
5025 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
5026 SVT.getVectorElementType(),
5027 VT.getVectorNumElements());
5028 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
5029 DAG.getVectorIdxConstant(0, dl));
5030
5031 EVT OpVT = N->getOperand(0).getValueType();
5032 ISD::NodeType ExtendCode =
5033 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
5034 return DAG.getNode(ExtendCode, dl, VT, CC);
5035 }
5036
WidenVecOp_STRICT_FSETCC(SDNode * N)5037 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
5038 SDValue Chain = N->getOperand(0);
5039 SDValue LHS = GetWidenedVector(N->getOperand(1));
5040 SDValue RHS = GetWidenedVector(N->getOperand(2));
5041 SDValue CC = N->getOperand(3);
5042 SDLoc dl(N);
5043
5044 EVT VT = N->getValueType(0);
5045 EVT EltVT = VT.getVectorElementType();
5046 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
5047 unsigned NumElts = VT.getVectorNumElements();
5048
5049 // Unroll into a build vector.
5050 SmallVector<SDValue, 8> Scalars(NumElts);
5051 SmallVector<SDValue, 8> Chains(NumElts);
5052
5053 for (unsigned i = 0; i != NumElts; ++i) {
5054 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
5055 DAG.getVectorIdxConstant(i, dl));
5056 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
5057 DAG.getVectorIdxConstant(i, dl));
5058
5059 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
5060 {Chain, LHSElem, RHSElem, CC});
5061 Chains[i] = Scalars[i].getValue(1);
5062 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
5063 DAG.getBoolConstant(true, dl, EltVT, VT),
5064 DAG.getBoolConstant(false, dl, EltVT, VT));
5065 }
5066
5067 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
5068 ReplaceValueWith(SDValue(N, 1), NewChain);
5069
5070 return DAG.getBuildVector(VT, dl, Scalars);
5071 }
5072
WidenVecOp_VECREDUCE(SDNode * N)5073 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
5074 SDLoc dl(N);
5075 SDValue Op = GetWidenedVector(N->getOperand(0));
5076 EVT OrigVT = N->getOperand(0).getValueType();
5077 EVT WideVT = Op.getValueType();
5078 EVT ElemVT = OrigVT.getVectorElementType();
5079 SDNodeFlags Flags = N->getFlags();
5080
5081 unsigned Opc = N->getOpcode();
5082 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
5083 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
5084 assert(NeutralElem && "Neutral element must exist");
5085
5086 // Pad the vector with the neutral element.
5087 unsigned OrigElts = OrigVT.getVectorNumElements();
5088 unsigned WideElts = WideVT.getVectorNumElements();
5089 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
5090 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
5091 DAG.getVectorIdxConstant(Idx, dl));
5092
5093 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
5094 }
5095
WidenVecOp_VECREDUCE_SEQ(SDNode * N)5096 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
5097 SDLoc dl(N);
5098 SDValue AccOp = N->getOperand(0);
5099 SDValue VecOp = N->getOperand(1);
5100 SDValue Op = GetWidenedVector(VecOp);
5101
5102 EVT OrigVT = VecOp.getValueType();
5103 EVT WideVT = Op.getValueType();
5104 EVT ElemVT = OrigVT.getVectorElementType();
5105 SDNodeFlags Flags = N->getFlags();
5106
5107 unsigned Opc = N->getOpcode();
5108 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
5109 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
5110
5111 // Pad the vector with the neutral element.
5112 unsigned OrigElts = OrigVT.getVectorNumElements();
5113 unsigned WideElts = WideVT.getVectorNumElements();
5114 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
5115 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
5116 DAG.getVectorIdxConstant(Idx, dl));
5117
5118 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags);
5119 }
5120
WidenVecOp_VSELECT(SDNode * N)5121 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
5122 // This only gets called in the case that the left and right inputs and
5123 // result are of a legal odd vector type, and the condition is illegal i1 of
5124 // the same odd width that needs widening.
5125 EVT VT = N->getValueType(0);
5126 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
5127
5128 SDValue Cond = GetWidenedVector(N->getOperand(0));
5129 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
5130 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
5131 SDLoc DL(N);
5132
5133 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
5134 LeftIn, RightIn);
5135 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
5136 DAG.getVectorIdxConstant(0, DL));
5137 }
5138
5139 //===----------------------------------------------------------------------===//
5140 // Vector Widening Utilities
5141 //===----------------------------------------------------------------------===//
5142
5143 // Utility function to find the type to chop up a widen vector for load/store
5144 // TLI: Target lowering used to determine legal types.
5145 // Width: Width left need to load/store.
5146 // WidenVT: The widen vector type to load to/store from
5147 // Align: If 0, don't allow use of a wider type
5148 // WidenEx: If Align is not 0, the amount additional we can load/store from.
5149
FindMemType(SelectionDAG & DAG,const TargetLowering & TLI,unsigned Width,EVT WidenVT,unsigned Align=0,unsigned WidenEx=0)5150 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
5151 unsigned Width, EVT WidenVT,
5152 unsigned Align = 0, unsigned WidenEx = 0) {
5153 EVT WidenEltVT = WidenVT.getVectorElementType();
5154 const bool Scalable = WidenVT.isScalableVector();
5155 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize();
5156 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
5157 unsigned AlignInBits = Align*8;
5158
5159 // If we have one element to load/store, return it.
5160 EVT RetVT = WidenEltVT;
5161 if (!Scalable && Width == WidenEltWidth)
5162 return RetVT;
5163
5164 // Don't bother looking for an integer type if the vector is scalable, skip
5165 // to vector types.
5166 if (!Scalable) {
5167 // See if there is larger legal integer than the element type to load/store.
5168 for (EVT MemVT : reverse(MVT::integer_valuetypes())) {
5169 unsigned MemVTWidth = MemVT.getSizeInBits();
5170 if (MemVT.getSizeInBits() <= WidenEltWidth)
5171 break;
5172 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
5173 if ((Action == TargetLowering::TypeLegal ||
5174 Action == TargetLowering::TypePromoteInteger) &&
5175 (WidenWidth % MemVTWidth) == 0 &&
5176 isPowerOf2_32(WidenWidth / MemVTWidth) &&
5177 (MemVTWidth <= Width ||
5178 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
5179 if (MemVTWidth == WidenWidth)
5180 return MemVT;
5181 RetVT = MemVT;
5182 break;
5183 }
5184 }
5185 }
5186
5187 // See if there is a larger vector type to load/store that has the same vector
5188 // element type and is evenly divisible with the WidenVT.
5189 for (EVT MemVT : reverse(MVT::vector_valuetypes())) {
5190 // Skip vector MVTs which don't match the scalable property of WidenVT.
5191 if (Scalable != MemVT.isScalableVector())
5192 continue;
5193 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize();
5194 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
5195 if ((Action == TargetLowering::TypeLegal ||
5196 Action == TargetLowering::TypePromoteInteger) &&
5197 WidenEltVT == MemVT.getVectorElementType() &&
5198 (WidenWidth % MemVTWidth) == 0 &&
5199 isPowerOf2_32(WidenWidth / MemVTWidth) &&
5200 (MemVTWidth <= Width ||
5201 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
5202 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
5203 return MemVT;
5204 }
5205 }
5206
5207 if (Scalable)
5208 report_fatal_error("Using element-wise loads and stores for widening "
5209 "operations is not supported for scalable vectors");
5210 return RetVT;
5211 }
5212
5213 // Builds a vector type from scalar loads
5214 // VecTy: Resulting Vector type
5215 // LDOps: Load operators to build a vector type
5216 // [Start,End) the list of loads to use.
BuildVectorFromScalar(SelectionDAG & DAG,EVT VecTy,SmallVectorImpl<SDValue> & LdOps,unsigned Start,unsigned End)5217 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
5218 SmallVectorImpl<SDValue> &LdOps,
5219 unsigned Start, unsigned End) {
5220 SDLoc dl(LdOps[Start]);
5221 EVT LdTy = LdOps[Start].getValueType();
5222 unsigned Width = VecTy.getSizeInBits();
5223 unsigned NumElts = Width / LdTy.getSizeInBits();
5224 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
5225
5226 unsigned Idx = 1;
5227 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
5228
5229 for (unsigned i = Start + 1; i != End; ++i) {
5230 EVT NewLdTy = LdOps[i].getValueType();
5231 if (NewLdTy != LdTy) {
5232 NumElts = Width / NewLdTy.getSizeInBits();
5233 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
5234 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
5235 // Readjust position and vector position based on new load type.
5236 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
5237 LdTy = NewLdTy;
5238 }
5239 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
5240 DAG.getVectorIdxConstant(Idx++, dl));
5241 }
5242 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
5243 }
5244
GenWidenVectorLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD)5245 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
5246 LoadSDNode *LD) {
5247 // The strategy assumes that we can efficiently load power-of-two widths.
5248 // The routine chops the vector into the largest vector loads with the same
5249 // element type or scalar loads and then recombines it to the widen vector
5250 // type.
5251 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
5252 EVT LdVT = LD->getMemoryVT();
5253 SDLoc dl(LD);
5254 assert(LdVT.isVector() && WidenVT.isVector());
5255 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
5256 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
5257
5258 // Load information
5259 SDValue Chain = LD->getChain();
5260 SDValue BasePtr = LD->getBasePtr();
5261 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
5262 AAMDNodes AAInfo = LD->getAAInfo();
5263
5264 TypeSize LdWidth = LdVT.getSizeInBits();
5265 TypeSize WidenWidth = WidenVT.getSizeInBits();
5266 TypeSize WidthDiff = WidenWidth - LdWidth;
5267 // Allow wider loads if they are sufficiently aligned to avoid memory faults
5268 // and if the original load is simple.
5269 unsigned LdAlign = (!LD->isSimple()) ? 0 : LD->getAlignment();
5270
5271 // Find the vector type that can load from.
5272 EVT NewVT = FindMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign,
5273 WidthDiff.getKnownMinSize());
5274 TypeSize NewVTWidth = NewVT.getSizeInBits();
5275 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
5276 LD->getOriginalAlign(), MMOFlags, AAInfo);
5277 LdChain.push_back(LdOp.getValue(1));
5278
5279 // Check if we can load the element with one instruction.
5280 if (TypeSize::isKnownLE(LdWidth, NewVTWidth)) {
5281 if (!NewVT.isVector()) {
5282 unsigned NumElts = WidenWidth.getFixedSize() / NewVTWidth.getFixedSize();
5283 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
5284 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
5285 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
5286 }
5287 if (NewVT == WidenVT)
5288 return LdOp;
5289
5290 // TODO: We don't currently have any tests that exercise this code path.
5291 assert(WidenWidth.getFixedSize() % NewVTWidth.getFixedSize() == 0);
5292 unsigned NumConcat = WidenWidth.getFixedSize() / NewVTWidth.getFixedSize();
5293 SmallVector<SDValue, 16> ConcatOps(NumConcat);
5294 SDValue UndefVal = DAG.getUNDEF(NewVT);
5295 ConcatOps[0] = LdOp;
5296 for (unsigned i = 1; i != NumConcat; ++i)
5297 ConcatOps[i] = UndefVal;
5298 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
5299 }
5300
5301 // Load vector by using multiple loads from largest vector to scalar.
5302 SmallVector<SDValue, 16> LdOps;
5303 LdOps.push_back(LdOp);
5304
5305 uint64_t ScaledOffset = 0;
5306 MachinePointerInfo MPI = LD->getPointerInfo();
5307 do {
5308 LdWidth -= NewVTWidth;
5309 IncrementPointer(cast<LoadSDNode>(LdOp), NewVT, MPI, BasePtr,
5310 &ScaledOffset);
5311
5312 if (TypeSize::isKnownLT(LdWidth, NewVTWidth)) {
5313 // The current type we are using is too large. Find a better size.
5314 NewVT = FindMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign,
5315 WidthDiff.getKnownMinSize());
5316 NewVTWidth = NewVT.getSizeInBits();
5317 }
5318
5319 Align NewAlign = ScaledOffset == 0
5320 ? LD->getOriginalAlign()
5321 : commonAlignment(LD->getAlign(), ScaledOffset);
5322 SDValue L =
5323 DAG.getLoad(NewVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags, AAInfo);
5324 LdChain.push_back(L.getValue(1));
5325
5326 LdOps.push_back(L);
5327 LdOp = L;
5328 } while (TypeSize::isKnownGT(LdWidth, NewVTWidth));
5329
5330 // Build the vector from the load operations.
5331 unsigned End = LdOps.size();
5332 if (!LdOps[0].getValueType().isVector())
5333 // All the loads are scalar loads.
5334 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
5335
5336 // If the load contains vectors, build the vector using concat vector.
5337 // All of the vectors used to load are power-of-2, and the scalar loads can be
5338 // combined to make a power-of-2 vector.
5339 SmallVector<SDValue, 16> ConcatOps(End);
5340 int i = End - 1;
5341 int Idx = End;
5342 EVT LdTy = LdOps[i].getValueType();
5343 // First, combine the scalar loads to a vector.
5344 if (!LdTy.isVector()) {
5345 for (--i; i >= 0; --i) {
5346 LdTy = LdOps[i].getValueType();
5347 if (LdTy.isVector())
5348 break;
5349 }
5350 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
5351 }
5352
5353 ConcatOps[--Idx] = LdOps[i];
5354 for (--i; i >= 0; --i) {
5355 EVT NewLdTy = LdOps[i].getValueType();
5356 if (NewLdTy != LdTy) {
5357 // Create a larger vector.
5358 TypeSize LdTySize = LdTy.getSizeInBits();
5359 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
5360 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
5361 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinSize()));
5362 unsigned NumOps =
5363 NewLdTySize.getKnownMinSize() / LdTySize.getKnownMinSize();
5364 SmallVector<SDValue, 16> WidenOps(NumOps);
5365 unsigned j = 0;
5366 for (; j != End-Idx; ++j)
5367 WidenOps[j] = ConcatOps[Idx+j];
5368 for (; j != NumOps; ++j)
5369 WidenOps[j] = DAG.getUNDEF(LdTy);
5370
5371 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
5372 WidenOps);
5373 Idx = End - 1;
5374 LdTy = NewLdTy;
5375 }
5376 ConcatOps[--Idx] = LdOps[i];
5377 }
5378
5379 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
5380 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
5381 makeArrayRef(&ConcatOps[Idx], End - Idx));
5382
5383 // We need to fill the rest with undefs to build the vector.
5384 unsigned NumOps =
5385 WidenWidth.getKnownMinSize() / LdTy.getSizeInBits().getKnownMinSize();
5386 SmallVector<SDValue, 16> WidenOps(NumOps);
5387 SDValue UndefVal = DAG.getUNDEF(LdTy);
5388 {
5389 unsigned i = 0;
5390 for (; i != End-Idx; ++i)
5391 WidenOps[i] = ConcatOps[Idx+i];
5392 for (; i != NumOps; ++i)
5393 WidenOps[i] = UndefVal;
5394 }
5395 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
5396 }
5397
5398 SDValue
GenWidenVectorExtLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD,ISD::LoadExtType ExtType)5399 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
5400 LoadSDNode *LD,
5401 ISD::LoadExtType ExtType) {
5402 // For extension loads, it may not be more efficient to chop up the vector
5403 // and then extend it. Instead, we unroll the load and build a new vector.
5404 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
5405 EVT LdVT = LD->getMemoryVT();
5406 SDLoc dl(LD);
5407 assert(LdVT.isVector() && WidenVT.isVector());
5408 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
5409
5410 // Load information
5411 SDValue Chain = LD->getChain();
5412 SDValue BasePtr = LD->getBasePtr();
5413 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
5414 AAMDNodes AAInfo = LD->getAAInfo();
5415
5416 if (LdVT.isScalableVector())
5417 report_fatal_error("Generating widen scalable extending vector loads is "
5418 "not yet supported");
5419
5420 EVT EltVT = WidenVT.getVectorElementType();
5421 EVT LdEltVT = LdVT.getVectorElementType();
5422 unsigned NumElts = LdVT.getVectorNumElements();
5423
5424 // Load each element and widen.
5425 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5426 SmallVector<SDValue, 16> Ops(WidenNumElts);
5427 unsigned Increment = LdEltVT.getSizeInBits() / 8;
5428 Ops[0] =
5429 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
5430 LdEltVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
5431 LdChain.push_back(Ops[0].getValue(1));
5432 unsigned i = 0, Offset = Increment;
5433 for (i=1; i < NumElts; ++i, Offset += Increment) {
5434 SDValue NewBasePtr =
5435 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::Fixed(Offset));
5436 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
5437 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
5438 LD->getOriginalAlign(), MMOFlags, AAInfo);
5439 LdChain.push_back(Ops[i].getValue(1));
5440 }
5441
5442 // Fill the rest with undefs.
5443 SDValue UndefVal = DAG.getUNDEF(EltVT);
5444 for (; i != WidenNumElts; ++i)
5445 Ops[i] = UndefVal;
5446
5447 return DAG.getBuildVector(WidenVT, dl, Ops);
5448 }
5449
GenWidenVectorStores(SmallVectorImpl<SDValue> & StChain,StoreSDNode * ST)5450 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
5451 StoreSDNode *ST) {
5452 // The strategy assumes that we can efficiently store power-of-two widths.
5453 // The routine chops the vector into the largest vector stores with the same
5454 // element type or scalar stores.
5455 SDValue Chain = ST->getChain();
5456 SDValue BasePtr = ST->getBasePtr();
5457 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
5458 AAMDNodes AAInfo = ST->getAAInfo();
5459 SDValue ValOp = GetWidenedVector(ST->getValue());
5460 SDLoc dl(ST);
5461
5462 EVT StVT = ST->getMemoryVT();
5463 TypeSize StWidth = StVT.getSizeInBits();
5464 EVT ValVT = ValOp.getValueType();
5465 TypeSize ValWidth = ValVT.getSizeInBits();
5466 EVT ValEltVT = ValVT.getVectorElementType();
5467 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
5468 assert(StVT.getVectorElementType() == ValEltVT);
5469 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
5470 "Mismatch between store and value types");
5471
5472 int Idx = 0; // current index to store
5473
5474 MachinePointerInfo MPI = ST->getPointerInfo();
5475 uint64_t ScaledOffset = 0;
5476 while (StWidth.isNonZero()) {
5477 // Find the largest vector type we can store with.
5478 EVT NewVT = FindMemType(DAG, TLI, StWidth.getKnownMinSize(), ValVT);
5479 TypeSize NewVTWidth = NewVT.getSizeInBits();
5480
5481 if (NewVT.isVector()) {
5482 unsigned NumVTElts = NewVT.getVectorMinNumElements();
5483 do {
5484 Align NewAlign = ScaledOffset == 0
5485 ? ST->getOriginalAlign()
5486 : commonAlignment(ST->getAlign(), ScaledOffset);
5487 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
5488 DAG.getVectorIdxConstant(Idx, dl));
5489 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign,
5490 MMOFlags, AAInfo);
5491 StChain.push_back(PartStore);
5492
5493 StWidth -= NewVTWidth;
5494 Idx += NumVTElts;
5495
5496 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr,
5497 &ScaledOffset);
5498 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
5499 } else {
5500 // Cast the vector to the scalar type we can store.
5501 unsigned NumElts = ValWidth.getFixedSize() / NewVTWidth.getFixedSize();
5502 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
5503 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
5504 // Readjust index position based on new vector type.
5505 Idx = Idx * ValEltWidth / NewVTWidth.getFixedSize();
5506 do {
5507 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
5508 DAG.getVectorIdxConstant(Idx++, dl));
5509 SDValue PartStore =
5510 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getOriginalAlign(),
5511 MMOFlags, AAInfo);
5512 StChain.push_back(PartStore);
5513
5514 StWidth -= NewVTWidth;
5515 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr);
5516 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
5517 // Restore index back to be relative to the original widen element type.
5518 Idx = Idx * NewVTWidth.getFixedSize() / ValEltWidth;
5519 }
5520 }
5521 }
5522
5523 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
5524 /// input vector must have the same element type as NVT.
5525 /// FillWithZeroes specifies that the vector should be widened with zeroes.
ModifyToType(SDValue InOp,EVT NVT,bool FillWithZeroes)5526 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5527 bool FillWithZeroes) {
5528 // Note that InOp might have been widened so it might already have
5529 // the right width or it might need be narrowed.
5530 EVT InVT = InOp.getValueType();
5531 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
5532 "input and widen element type must match");
5533 SDLoc dl(InOp);
5534
5535 // Check if InOp already has the right width.
5536 if (InVT == NVT)
5537 return InOp;
5538
5539 unsigned InNumElts = InVT.getVectorNumElements();
5540 unsigned WidenNumElts = NVT.getVectorNumElements();
5541 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
5542 unsigned NumConcat = WidenNumElts / InNumElts;
5543 SmallVector<SDValue, 16> Ops(NumConcat);
5544 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
5545 DAG.getUNDEF(InVT);
5546 Ops[0] = InOp;
5547 for (unsigned i = 1; i != NumConcat; ++i)
5548 Ops[i] = FillVal;
5549
5550 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
5551 }
5552
5553 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
5554 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
5555 DAG.getVectorIdxConstant(0, dl));
5556
5557 // Fall back to extract and build.
5558 SmallVector<SDValue, 16> Ops(WidenNumElts);
5559 EVT EltVT = NVT.getVectorElementType();
5560 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
5561 unsigned Idx;
5562 for (Idx = 0; Idx < MinNumElts; ++Idx)
5563 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5564 DAG.getVectorIdxConstant(Idx, dl));
5565
5566 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
5567 DAG.getUNDEF(EltVT);
5568 for ( ; Idx < WidenNumElts; ++Idx)
5569 Ops[Idx] = FillVal;
5570 return DAG.getBuildVector(NVT, dl, Ops);
5571 }
5572
SplitVecRes_VECTOR_REVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)5573 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
5574 SDValue &Hi) {
5575 SDValue InLo, InHi;
5576 GetSplitVector(N->getOperand(0), InLo, InHi);
5577 SDLoc DL(N);
5578
5579 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
5580 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
5581 }
5582
SplitVecRes_VECTOR_SPLICE(SDNode * N,SDValue & Lo,SDValue & Hi)5583 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
5584 SDValue &Hi) {
5585 EVT VT = N->getValueType(0);
5586 SDLoc DL(N);
5587
5588 EVT LoVT, HiVT;
5589 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
5590
5591 SDValue Expanded = TLI.expandVectorSplice(N, DAG);
5592 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
5593 DAG.getVectorIdxConstant(0, DL));
5594 Hi =
5595 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
5596 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
5597 }
5598