1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "LegalizeTypes.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/Support/ErrorHandling.h"
26 #include "llvm/Support/TypeSize.h"
27 #include "llvm/Support/raw_ostream.h"
28 using namespace llvm;
29
30 #define DEBUG_TYPE "legalize-types"
31
32 //===----------------------------------------------------------------------===//
33 // Result Vector Scalarization: <1 x ty> -> ty.
34 //===----------------------------------------------------------------------===//
35
ScalarizeVectorResult(SDNode * N,unsigned ResNo)36 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
37 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
38 dbgs() << "\n");
39 SDValue R = SDValue();
40
41 switch (N->getOpcode()) {
42 default:
43 #ifndef NDEBUG
44 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45 N->dump(&DAG);
46 dbgs() << "\n";
47 #endif
48 report_fatal_error("Do not know how to scalarize the result of this "
49 "operator!\n");
50
51 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
67 case ISD::ANY_EXTEND_VECTOR_INREG:
68 case ISD::SIGN_EXTEND_VECTOR_INREG:
69 case ISD::ZERO_EXTEND_VECTOR_INREG:
70 R = ScalarizeVecRes_VecInregOp(N);
71 break;
72 case ISD::ABS:
73 case ISD::ANY_EXTEND:
74 case ISD::BITREVERSE:
75 case ISD::BSWAP:
76 case ISD::CTLZ:
77 case ISD::CTLZ_ZERO_UNDEF:
78 case ISD::CTPOP:
79 case ISD::CTTZ:
80 case ISD::CTTZ_ZERO_UNDEF:
81 case ISD::FABS:
82 case ISD::FCEIL:
83 case ISD::FCOS:
84 case ISD::FEXP:
85 case ISD::FEXP2:
86 case ISD::FFLOOR:
87 case ISD::FLOG:
88 case ISD::FLOG10:
89 case ISD::FLOG2:
90 case ISD::FNEARBYINT:
91 case ISD::FNEG:
92 case ISD::FREEZE:
93 case ISD::FP_EXTEND:
94 case ISD::FP_TO_SINT:
95 case ISD::FP_TO_UINT:
96 case ISD::FRINT:
97 case ISD::FROUND:
98 case ISD::FROUNDEVEN:
99 case ISD::FSIN:
100 case ISD::FSQRT:
101 case ISD::FTRUNC:
102 case ISD::SIGN_EXTEND:
103 case ISD::SINT_TO_FP:
104 case ISD::TRUNCATE:
105 case ISD::UINT_TO_FP:
106 case ISD::ZERO_EXTEND:
107 case ISD::FCANONICALIZE:
108 R = ScalarizeVecRes_UnaryOp(N);
109 break;
110
111 case ISD::ADD:
112 case ISD::AND:
113 case ISD::FADD:
114 case ISD::FCOPYSIGN:
115 case ISD::FDIV:
116 case ISD::FMUL:
117 case ISD::FMINNUM:
118 case ISD::FMAXNUM:
119 case ISD::FMINNUM_IEEE:
120 case ISD::FMAXNUM_IEEE:
121 case ISD::FMINIMUM:
122 case ISD::FMAXIMUM:
123 case ISD::SMIN:
124 case ISD::SMAX:
125 case ISD::UMIN:
126 case ISD::UMAX:
127
128 case ISD::SADDSAT:
129 case ISD::UADDSAT:
130 case ISD::SSUBSAT:
131 case ISD::USUBSAT:
132 case ISD::SSHLSAT:
133 case ISD::USHLSAT:
134
135 case ISD::FPOW:
136 case ISD::FREM:
137 case ISD::FSUB:
138 case ISD::MUL:
139 case ISD::OR:
140 case ISD::SDIV:
141 case ISD::SREM:
142 case ISD::SUB:
143 case ISD::UDIV:
144 case ISD::UREM:
145 case ISD::XOR:
146 case ISD::SHL:
147 case ISD::SRA:
148 case ISD::SRL:
149 case ISD::ROTL:
150 case ISD::ROTR:
151 R = ScalarizeVecRes_BinOp(N);
152 break;
153 case ISD::FMA:
154 case ISD::FSHL:
155 case ISD::FSHR:
156 R = ScalarizeVecRes_TernaryOp(N);
157 break;
158
159 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
160 case ISD::STRICT_##DAGN:
161 #include "llvm/IR/ConstrainedOps.def"
162 R = ScalarizeVecRes_StrictFPOp(N);
163 break;
164
165 case ISD::UADDO:
166 case ISD::SADDO:
167 case ISD::USUBO:
168 case ISD::SSUBO:
169 case ISD::UMULO:
170 case ISD::SMULO:
171 R = ScalarizeVecRes_OverflowOp(N, ResNo);
172 break;
173 case ISD::SMULFIX:
174 case ISD::SMULFIXSAT:
175 case ISD::UMULFIX:
176 case ISD::UMULFIXSAT:
177 case ISD::SDIVFIX:
178 case ISD::SDIVFIXSAT:
179 case ISD::UDIVFIX:
180 case ISD::UDIVFIXSAT:
181 R = ScalarizeVecRes_FIX(N);
182 break;
183 }
184
185 // If R is null, the sub-method took care of registering the result.
186 if (R.getNode())
187 SetScalarizedVector(SDValue(N, ResNo), R);
188 }
189
ScalarizeVecRes_BinOp(SDNode * N)190 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
191 SDValue LHS = GetScalarizedVector(N->getOperand(0));
192 SDValue RHS = GetScalarizedVector(N->getOperand(1));
193 return DAG.getNode(N->getOpcode(), SDLoc(N),
194 LHS.getValueType(), LHS, RHS, N->getFlags());
195 }
196
ScalarizeVecRes_TernaryOp(SDNode * N)197 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
198 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
199 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
200 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
201 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
202 Op2, N->getFlags());
203 }
204
ScalarizeVecRes_FIX(SDNode * N)205 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
206 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
207 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
208 SDValue Op2 = N->getOperand(2);
209 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
210 Op2, N->getFlags());
211 }
212
ScalarizeVecRes_StrictFPOp(SDNode * N)213 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
214 EVT VT = N->getValueType(0).getVectorElementType();
215 unsigned NumOpers = N->getNumOperands();
216 SDValue Chain = N->getOperand(0);
217 EVT ValueVTs[] = {VT, MVT::Other};
218 SDLoc dl(N);
219
220 SmallVector<SDValue, 4> Opers(NumOpers);
221
222 // The Chain is the first operand.
223 Opers[0] = Chain;
224
225 // Now process the remaining operands.
226 for (unsigned i = 1; i < NumOpers; ++i) {
227 SDValue Oper = N->getOperand(i);
228
229 if (Oper.getValueType().isVector())
230 Oper = GetScalarizedVector(Oper);
231
232 Opers[i] = Oper;
233 }
234
235 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
236 Opers, N->getFlags());
237
238 // Legalize the chain result - switch anything that used the old chain to
239 // use the new one.
240 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
241 return Result;
242 }
243
ScalarizeVecRes_OverflowOp(SDNode * N,unsigned ResNo)244 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
245 unsigned ResNo) {
246 SDLoc DL(N);
247 EVT ResVT = N->getValueType(0);
248 EVT OvVT = N->getValueType(1);
249
250 SDValue ScalarLHS, ScalarRHS;
251 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
252 ScalarLHS = GetScalarizedVector(N->getOperand(0));
253 ScalarRHS = GetScalarizedVector(N->getOperand(1));
254 } else {
255 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
256 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
257 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
258 ScalarLHS = ElemsLHS[0];
259 ScalarRHS = ElemsRHS[0];
260 }
261
262 SDVTList ScalarVTs = DAG.getVTList(
263 ResVT.getVectorElementType(), OvVT.getVectorElementType());
264 SDNode *ScalarNode = DAG.getNode(
265 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
266 ScalarNode->setFlags(N->getFlags());
267
268 // Replace the other vector result not being explicitly scalarized here.
269 unsigned OtherNo = 1 - ResNo;
270 EVT OtherVT = N->getValueType(OtherNo);
271 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
272 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
273 } else {
274 SDValue OtherVal = DAG.getNode(
275 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
276 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
277 }
278
279 return SDValue(ScalarNode, ResNo);
280 }
281
ScalarizeVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)282 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
283 unsigned ResNo) {
284 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
285 return GetScalarizedVector(Op);
286 }
287
ScalarizeVecRes_BITCAST(SDNode * N)288 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
289 SDValue Op = N->getOperand(0);
290 if (Op.getValueType().isVector()
291 && Op.getValueType().getVectorNumElements() == 1
292 && !isSimpleLegalType(Op.getValueType()))
293 Op = GetScalarizedVector(Op);
294 EVT NewVT = N->getValueType(0).getVectorElementType();
295 return DAG.getNode(ISD::BITCAST, SDLoc(N),
296 NewVT, Op);
297 }
298
ScalarizeVecRes_BUILD_VECTOR(SDNode * N)299 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
300 EVT EltVT = N->getValueType(0).getVectorElementType();
301 SDValue InOp = N->getOperand(0);
302 // The BUILD_VECTOR operands may be of wider element types and
303 // we may need to truncate them back to the requested return type.
304 if (EltVT.isInteger())
305 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
306 return InOp;
307 }
308
ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode * N)309 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
310 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
311 N->getValueType(0).getVectorElementType(),
312 N->getOperand(0), N->getOperand(1));
313 }
314
ScalarizeVecRes_FP_ROUND(SDNode * N)315 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
316 EVT NewVT = N->getValueType(0).getVectorElementType();
317 SDValue Op = GetScalarizedVector(N->getOperand(0));
318 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
319 NewVT, Op, N->getOperand(1));
320 }
321
ScalarizeVecRes_FPOWI(SDNode * N)322 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
323 SDValue Op = GetScalarizedVector(N->getOperand(0));
324 return DAG.getNode(ISD::FPOWI, SDLoc(N),
325 Op.getValueType(), Op, N->getOperand(1));
326 }
327
ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode * N)328 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
329 // The value to insert may have a wider type than the vector element type,
330 // so be sure to truncate it to the element type if necessary.
331 SDValue Op = N->getOperand(1);
332 EVT EltVT = N->getValueType(0).getVectorElementType();
333 if (Op.getValueType() != EltVT)
334 // FIXME: Can this happen for floating point types?
335 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
336 return Op;
337 }
338
ScalarizeVecRes_LOAD(LoadSDNode * N)339 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
340 assert(N->isUnindexed() && "Indexed vector load?");
341
342 SDValue Result = DAG.getLoad(
343 ISD::UNINDEXED, N->getExtensionType(),
344 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
345 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
346 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
347 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
348
349 // Legalize the chain result - switch anything that used the old chain to
350 // use the new one.
351 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
352 return Result;
353 }
354
ScalarizeVecRes_UnaryOp(SDNode * N)355 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
356 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
357 EVT DestVT = N->getValueType(0).getVectorElementType();
358 SDValue Op = N->getOperand(0);
359 EVT OpVT = Op.getValueType();
360 SDLoc DL(N);
361 // The result needs scalarizing, but it's not a given that the source does.
362 // This is a workaround for targets where it's impossible to scalarize the
363 // result of a conversion, because the source type is legal.
364 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
365 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
366 // legal and was not scalarized.
367 // See the similar logic in ScalarizeVecRes_SETCC
368 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
369 Op = GetScalarizedVector(Op);
370 } else {
371 EVT VT = OpVT.getVectorElementType();
372 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
373 DAG.getVectorIdxConstant(0, DL));
374 }
375 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
376 }
377
ScalarizeVecRes_InregOp(SDNode * N)378 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
379 EVT EltVT = N->getValueType(0).getVectorElementType();
380 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
381 SDValue LHS = GetScalarizedVector(N->getOperand(0));
382 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
383 LHS, DAG.getValueType(ExtVT));
384 }
385
ScalarizeVecRes_VecInregOp(SDNode * N)386 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
387 SDLoc DL(N);
388 SDValue Op = N->getOperand(0);
389
390 EVT OpVT = Op.getValueType();
391 EVT OpEltVT = OpVT.getVectorElementType();
392 EVT EltVT = N->getValueType(0).getVectorElementType();
393
394 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
395 Op = GetScalarizedVector(Op);
396 } else {
397 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
398 DAG.getVectorIdxConstant(0, DL));
399 }
400
401 switch (N->getOpcode()) {
402 case ISD::ANY_EXTEND_VECTOR_INREG:
403 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
404 case ISD::SIGN_EXTEND_VECTOR_INREG:
405 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
406 case ISD::ZERO_EXTEND_VECTOR_INREG:
407 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
408 }
409
410 llvm_unreachable("Illegal extend_vector_inreg opcode");
411 }
412
ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode * N)413 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
414 // If the operand is wider than the vector element type then it is implicitly
415 // truncated. Make that explicit here.
416 EVT EltVT = N->getValueType(0).getVectorElementType();
417 SDValue InOp = N->getOperand(0);
418 if (InOp.getValueType() != EltVT)
419 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
420 return InOp;
421 }
422
ScalarizeVecRes_VSELECT(SDNode * N)423 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
424 SDValue Cond = N->getOperand(0);
425 EVT OpVT = Cond.getValueType();
426 SDLoc DL(N);
427 // The vselect result and true/value operands needs scalarizing, but it's
428 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
429 // See the similar logic in ScalarizeVecRes_SETCC
430 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
431 Cond = GetScalarizedVector(Cond);
432 } else {
433 EVT VT = OpVT.getVectorElementType();
434 Cond = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
435 DAG.getVectorIdxConstant(0, DL));
436 }
437
438 SDValue LHS = GetScalarizedVector(N->getOperand(1));
439 TargetLowering::BooleanContent ScalarBool =
440 TLI.getBooleanContents(false, false);
441 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
442
443 // If integer and float booleans have different contents then we can't
444 // reliably optimize in all cases. There is a full explanation for this in
445 // DAGCombiner::visitSELECT() where the same issue affects folding
446 // (select C, 0, 1) to (xor C, 1).
447 if (TLI.getBooleanContents(false, false) !=
448 TLI.getBooleanContents(false, true)) {
449 // At least try the common case where the boolean is generated by a
450 // comparison.
451 if (Cond->getOpcode() == ISD::SETCC) {
452 EVT OpVT = Cond->getOperand(0).getValueType();
453 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
454 VecBool = TLI.getBooleanContents(OpVT);
455 } else
456 ScalarBool = TargetLowering::UndefinedBooleanContent;
457 }
458
459 EVT CondVT = Cond.getValueType();
460 if (ScalarBool != VecBool) {
461 switch (ScalarBool) {
462 case TargetLowering::UndefinedBooleanContent:
463 break;
464 case TargetLowering::ZeroOrOneBooleanContent:
465 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
466 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
467 // Vector read from all ones, scalar expects a single 1 so mask.
468 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
469 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
470 break;
471 case TargetLowering::ZeroOrNegativeOneBooleanContent:
472 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
473 VecBool == TargetLowering::ZeroOrOneBooleanContent);
474 // Vector reads from a one, scalar from all ones so sign extend.
475 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
476 Cond, DAG.getValueType(MVT::i1));
477 break;
478 }
479 }
480
481 // Truncate the condition if needed
482 auto BoolVT = getSetCCResultType(CondVT);
483 if (BoolVT.bitsLT(CondVT))
484 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
485
486 return DAG.getSelect(SDLoc(N),
487 LHS.getValueType(), Cond, LHS,
488 GetScalarizedVector(N->getOperand(2)));
489 }
490
ScalarizeVecRes_SELECT(SDNode * N)491 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
492 SDValue LHS = GetScalarizedVector(N->getOperand(1));
493 return DAG.getSelect(SDLoc(N),
494 LHS.getValueType(), N->getOperand(0), LHS,
495 GetScalarizedVector(N->getOperand(2)));
496 }
497
ScalarizeVecRes_SELECT_CC(SDNode * N)498 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
499 SDValue LHS = GetScalarizedVector(N->getOperand(2));
500 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
501 N->getOperand(0), N->getOperand(1),
502 LHS, GetScalarizedVector(N->getOperand(3)),
503 N->getOperand(4));
504 }
505
ScalarizeVecRes_UNDEF(SDNode * N)506 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
507 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
508 }
509
ScalarizeVecRes_VECTOR_SHUFFLE(SDNode * N)510 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
511 // Figure out if the scalar is the LHS or RHS and return it.
512 SDValue Arg = N->getOperand(2).getOperand(0);
513 if (Arg.isUndef())
514 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
515 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
516 return GetScalarizedVector(N->getOperand(Op));
517 }
518
ScalarizeVecRes_SETCC(SDNode * N)519 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
520 assert(N->getValueType(0).isVector() &&
521 N->getOperand(0).getValueType().isVector() &&
522 "Operand types must be vectors");
523 SDValue LHS = N->getOperand(0);
524 SDValue RHS = N->getOperand(1);
525 EVT OpVT = LHS.getValueType();
526 EVT NVT = N->getValueType(0).getVectorElementType();
527 SDLoc DL(N);
528
529 // The result needs scalarizing, but it's not a given that the source does.
530 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
531 LHS = GetScalarizedVector(LHS);
532 RHS = GetScalarizedVector(RHS);
533 } else {
534 EVT VT = OpVT.getVectorElementType();
535 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
536 DAG.getVectorIdxConstant(0, DL));
537 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
538 DAG.getVectorIdxConstant(0, DL));
539 }
540
541 // Turn it into a scalar SETCC.
542 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
543 N->getOperand(2));
544 // Vectors may have a different boolean contents to scalars. Promote the
545 // value appropriately.
546 ISD::NodeType ExtendCode =
547 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
548 return DAG.getNode(ExtendCode, DL, NVT, Res);
549 }
550
551
552 //===----------------------------------------------------------------------===//
553 // Operand Vector Scalarization <1 x ty> -> ty.
554 //===----------------------------------------------------------------------===//
555
ScalarizeVectorOperand(SDNode * N,unsigned OpNo)556 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
557 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
558 dbgs() << "\n");
559 SDValue Res = SDValue();
560
561 switch (N->getOpcode()) {
562 default:
563 #ifndef NDEBUG
564 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
565 N->dump(&DAG);
566 dbgs() << "\n";
567 #endif
568 report_fatal_error("Do not know how to scalarize this operator's "
569 "operand!\n");
570 case ISD::BITCAST:
571 Res = ScalarizeVecOp_BITCAST(N);
572 break;
573 case ISD::ANY_EXTEND:
574 case ISD::ZERO_EXTEND:
575 case ISD::SIGN_EXTEND:
576 case ISD::TRUNCATE:
577 case ISD::FP_TO_SINT:
578 case ISD::FP_TO_UINT:
579 case ISD::SINT_TO_FP:
580 case ISD::UINT_TO_FP:
581 Res = ScalarizeVecOp_UnaryOp(N);
582 break;
583 case ISD::STRICT_SINT_TO_FP:
584 case ISD::STRICT_UINT_TO_FP:
585 case ISD::STRICT_FP_TO_SINT:
586 case ISD::STRICT_FP_TO_UINT:
587 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
588 break;
589 case ISD::CONCAT_VECTORS:
590 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
591 break;
592 case ISD::EXTRACT_VECTOR_ELT:
593 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
594 break;
595 case ISD::VSELECT:
596 Res = ScalarizeVecOp_VSELECT(N);
597 break;
598 case ISD::SETCC:
599 Res = ScalarizeVecOp_VSETCC(N);
600 break;
601 case ISD::STORE:
602 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
603 break;
604 case ISD::STRICT_FP_ROUND:
605 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
606 break;
607 case ISD::FP_ROUND:
608 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
609 break;
610 case ISD::VECREDUCE_FADD:
611 case ISD::VECREDUCE_FMUL:
612 case ISD::VECREDUCE_ADD:
613 case ISD::VECREDUCE_MUL:
614 case ISD::VECREDUCE_AND:
615 case ISD::VECREDUCE_OR:
616 case ISD::VECREDUCE_XOR:
617 case ISD::VECREDUCE_SMAX:
618 case ISD::VECREDUCE_SMIN:
619 case ISD::VECREDUCE_UMAX:
620 case ISD::VECREDUCE_UMIN:
621 case ISD::VECREDUCE_FMAX:
622 case ISD::VECREDUCE_FMIN:
623 Res = ScalarizeVecOp_VECREDUCE(N);
624 break;
625 case ISD::VECREDUCE_SEQ_FADD:
626 case ISD::VECREDUCE_SEQ_FMUL:
627 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
628 break;
629 }
630
631 // If the result is null, the sub-method took care of registering results etc.
632 if (!Res.getNode()) return false;
633
634 // If the result is N, the sub-method updated N in place. Tell the legalizer
635 // core about this.
636 if (Res.getNode() == N)
637 return true;
638
639 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
640 "Invalid operand expansion");
641
642 ReplaceValueWith(SDValue(N, 0), Res);
643 return false;
644 }
645
646 /// If the value to convert is a vector that needs to be scalarized, it must be
647 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_BITCAST(SDNode * N)648 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
649 SDValue Elt = GetScalarizedVector(N->getOperand(0));
650 return DAG.getNode(ISD::BITCAST, SDLoc(N),
651 N->getValueType(0), Elt);
652 }
653
654 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
655 /// Do the operation on the element instead.
ScalarizeVecOp_UnaryOp(SDNode * N)656 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
657 assert(N->getValueType(0).getVectorNumElements() == 1 &&
658 "Unexpected vector type!");
659 SDValue Elt = GetScalarizedVector(N->getOperand(0));
660 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
661 N->getValueType(0).getScalarType(), Elt);
662 // Revectorize the result so the types line up with what the uses of this
663 // expression expect.
664 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
665 }
666
667 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
668 /// Do the strict FP operation on the element instead.
ScalarizeVecOp_UnaryOp_StrictFP(SDNode * N)669 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
670 assert(N->getValueType(0).getVectorNumElements() == 1 &&
671 "Unexpected vector type!");
672 SDValue Elt = GetScalarizedVector(N->getOperand(1));
673 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
674 { N->getValueType(0).getScalarType(), MVT::Other },
675 { N->getOperand(0), Elt });
676 // Legalize the chain result - switch anything that used the old chain to
677 // use the new one.
678 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
679 // Revectorize the result so the types line up with what the uses of this
680 // expression expect.
681 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
682
683 // Do our own replacement and return SDValue() to tell the caller that we
684 // handled all replacements since caller can only handle a single result.
685 ReplaceValueWith(SDValue(N, 0), Res);
686 return SDValue();
687 }
688
689 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
ScalarizeVecOp_CONCAT_VECTORS(SDNode * N)690 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
691 SmallVector<SDValue, 8> Ops(N->getNumOperands());
692 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
693 Ops[i] = GetScalarizedVector(N->getOperand(i));
694 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
695 }
696
697 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
698 /// so just return the element, ignoring the index.
ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode * N)699 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
700 EVT VT = N->getValueType(0);
701 SDValue Res = GetScalarizedVector(N->getOperand(0));
702 if (Res.getValueType() != VT)
703 Res = VT.isFloatingPoint()
704 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
705 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
706 return Res;
707 }
708
709 /// If the input condition is a vector that needs to be scalarized, it must be
710 /// <1 x i1>, so just convert to a normal ISD::SELECT
711 /// (still with vector output type since that was acceptable if we got here).
ScalarizeVecOp_VSELECT(SDNode * N)712 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
713 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
714 EVT VT = N->getValueType(0);
715
716 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
717 N->getOperand(2));
718 }
719
720 /// If the operand is a vector that needs to be scalarized then the
721 /// result must be v1i1, so just convert to a scalar SETCC and wrap
722 /// with a scalar_to_vector since the res type is legal if we got here
ScalarizeVecOp_VSETCC(SDNode * N)723 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
724 assert(N->getValueType(0).isVector() &&
725 N->getOperand(0).getValueType().isVector() &&
726 "Operand types must be vectors");
727 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
728
729 EVT VT = N->getValueType(0);
730 SDValue LHS = GetScalarizedVector(N->getOperand(0));
731 SDValue RHS = GetScalarizedVector(N->getOperand(1));
732
733 EVT OpVT = N->getOperand(0).getValueType();
734 EVT NVT = VT.getVectorElementType();
735 SDLoc DL(N);
736 // Turn it into a scalar SETCC.
737 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
738 N->getOperand(2));
739
740 // Vectors may have a different boolean contents to scalars. Promote the
741 // value appropriately.
742 ISD::NodeType ExtendCode =
743 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
744
745 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
746
747 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
748 }
749
750 /// If the value to store is a vector that needs to be scalarized, it must be
751 /// <1 x ty>. Just store the element.
ScalarizeVecOp_STORE(StoreSDNode * N,unsigned OpNo)752 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
753 assert(N->isUnindexed() && "Indexed store of one-element vector?");
754 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
755 SDLoc dl(N);
756
757 if (N->isTruncatingStore())
758 return DAG.getTruncStore(
759 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
760 N->getBasePtr(), N->getPointerInfo(),
761 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
762 N->getMemOperand()->getFlags(), N->getAAInfo());
763
764 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
765 N->getBasePtr(), N->getPointerInfo(),
766 N->getOriginalAlign(), N->getMemOperand()->getFlags(),
767 N->getAAInfo());
768 }
769
770 /// If the value to round is a vector that needs to be scalarized, it must be
771 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_ROUND(SDNode * N,unsigned OpNo)772 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
773 SDValue Elt = GetScalarizedVector(N->getOperand(0));
774 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
775 N->getValueType(0).getVectorElementType(), Elt,
776 N->getOperand(1));
777 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
778 }
779
ScalarizeVecOp_STRICT_FP_ROUND(SDNode * N,unsigned OpNo)780 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
781 unsigned OpNo) {
782 assert(OpNo == 1 && "Wrong operand for scalarization!");
783 SDValue Elt = GetScalarizedVector(N->getOperand(1));
784 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
785 { N->getValueType(0).getVectorElementType(),
786 MVT::Other },
787 { N->getOperand(0), Elt, N->getOperand(2) });
788 // Legalize the chain result - switch anything that used the old chain to
789 // use the new one.
790 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
791
792 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
793
794 // Do our own replacement and return SDValue() to tell the caller that we
795 // handled all replacements since caller can only handle a single result.
796 ReplaceValueWith(SDValue(N, 0), Res);
797 return SDValue();
798 }
799
ScalarizeVecOp_VECREDUCE(SDNode * N)800 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
801 SDValue Res = GetScalarizedVector(N->getOperand(0));
802 // Result type may be wider than element type.
803 if (Res.getValueType() != N->getValueType(0))
804 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
805 return Res;
806 }
807
ScalarizeVecOp_VECREDUCE_SEQ(SDNode * N)808 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
809 SDValue AccOp = N->getOperand(0);
810 SDValue VecOp = N->getOperand(1);
811
812 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
813
814 SDValue Op = GetScalarizedVector(VecOp);
815 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
816 AccOp, Op, N->getFlags());
817 }
818
819 //===----------------------------------------------------------------------===//
820 // Result Vector Splitting
821 //===----------------------------------------------------------------------===//
822
823 /// This method is called when the specified result of the specified node is
824 /// found to need vector splitting. At this point, the node may also have
825 /// invalid operands or may have other results that need legalization, we just
826 /// know that (at least) one result needs vector splitting.
SplitVectorResult(SDNode * N,unsigned ResNo)827 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
828 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
829 SDValue Lo, Hi;
830
831 // See if the target wants to custom expand this node.
832 if (CustomLowerNode(N, N->getValueType(ResNo), true))
833 return;
834
835 switch (N->getOpcode()) {
836 default:
837 #ifndef NDEBUG
838 dbgs() << "SplitVectorResult #" << ResNo << ": ";
839 N->dump(&DAG);
840 dbgs() << "\n";
841 #endif
842 report_fatal_error("Do not know how to split the result of this "
843 "operator!\n");
844
845 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
846 case ISD::VSELECT:
847 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
848 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
849 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
850 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
851 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
852 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
853 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
854 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
855 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
856 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
857 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
858 case ISD::SPLAT_VECTOR:
859 case ISD::SCALAR_TO_VECTOR:
860 SplitVecRes_ScalarOp(N, Lo, Hi);
861 break;
862 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
863 case ISD::LOAD:
864 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
865 break;
866 case ISD::MLOAD:
867 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
868 break;
869 case ISD::MGATHER:
870 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
871 break;
872 case ISD::SETCC:
873 SplitVecRes_SETCC(N, Lo, Hi);
874 break;
875 case ISD::VECTOR_SHUFFLE:
876 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
877 break;
878 case ISD::VAARG:
879 SplitVecRes_VAARG(N, Lo, Hi);
880 break;
881
882 case ISD::ANY_EXTEND_VECTOR_INREG:
883 case ISD::SIGN_EXTEND_VECTOR_INREG:
884 case ISD::ZERO_EXTEND_VECTOR_INREG:
885 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
886 break;
887
888 case ISD::ABS:
889 case ISD::BITREVERSE:
890 case ISD::BSWAP:
891 case ISD::CTLZ:
892 case ISD::CTTZ:
893 case ISD::CTLZ_ZERO_UNDEF:
894 case ISD::CTTZ_ZERO_UNDEF:
895 case ISD::CTPOP:
896 case ISD::FABS:
897 case ISD::FCEIL:
898 case ISD::FCOS:
899 case ISD::FEXP:
900 case ISD::FEXP2:
901 case ISD::FFLOOR:
902 case ISD::FLOG:
903 case ISD::FLOG10:
904 case ISD::FLOG2:
905 case ISD::FNEARBYINT:
906 case ISD::FNEG:
907 case ISD::FREEZE:
908 case ISD::FP_EXTEND:
909 case ISD::FP_ROUND:
910 case ISD::FP_TO_SINT:
911 case ISD::FP_TO_UINT:
912 case ISD::FRINT:
913 case ISD::FROUND:
914 case ISD::FROUNDEVEN:
915 case ISD::FSIN:
916 case ISD::FSQRT:
917 case ISD::FTRUNC:
918 case ISD::SINT_TO_FP:
919 case ISD::TRUNCATE:
920 case ISD::UINT_TO_FP:
921 case ISD::FCANONICALIZE:
922 SplitVecRes_UnaryOp(N, Lo, Hi);
923 break;
924
925 case ISD::ANY_EXTEND:
926 case ISD::SIGN_EXTEND:
927 case ISD::ZERO_EXTEND:
928 SplitVecRes_ExtendOp(N, Lo, Hi);
929 break;
930
931 case ISD::ADD:
932 case ISD::SUB:
933 case ISD::MUL:
934 case ISD::MULHS:
935 case ISD::MULHU:
936 case ISD::FADD:
937 case ISD::FSUB:
938 case ISD::FMUL:
939 case ISD::FMINNUM:
940 case ISD::FMAXNUM:
941 case ISD::FMINIMUM:
942 case ISD::FMAXIMUM:
943 case ISD::SDIV:
944 case ISD::UDIV:
945 case ISD::FDIV:
946 case ISD::FPOW:
947 case ISD::AND:
948 case ISD::OR:
949 case ISD::XOR:
950 case ISD::SHL:
951 case ISD::SRA:
952 case ISD::SRL:
953 case ISD::UREM:
954 case ISD::SREM:
955 case ISD::FREM:
956 case ISD::SMIN:
957 case ISD::SMAX:
958 case ISD::UMIN:
959 case ISD::UMAX:
960 case ISD::SADDSAT:
961 case ISD::UADDSAT:
962 case ISD::SSUBSAT:
963 case ISD::USUBSAT:
964 case ISD::SSHLSAT:
965 case ISD::USHLSAT:
966 case ISD::ROTL:
967 case ISD::ROTR:
968 SplitVecRes_BinOp(N, Lo, Hi);
969 break;
970 case ISD::FMA:
971 case ISD::FSHL:
972 case ISD::FSHR:
973 SplitVecRes_TernaryOp(N, Lo, Hi);
974 break;
975
976 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
977 case ISD::STRICT_##DAGN:
978 #include "llvm/IR/ConstrainedOps.def"
979 SplitVecRes_StrictFPOp(N, Lo, Hi);
980 break;
981
982 case ISD::UADDO:
983 case ISD::SADDO:
984 case ISD::USUBO:
985 case ISD::SSUBO:
986 case ISD::UMULO:
987 case ISD::SMULO:
988 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
989 break;
990 case ISD::SMULFIX:
991 case ISD::SMULFIXSAT:
992 case ISD::UMULFIX:
993 case ISD::UMULFIXSAT:
994 case ISD::SDIVFIX:
995 case ISD::SDIVFIXSAT:
996 case ISD::UDIVFIX:
997 case ISD::UDIVFIXSAT:
998 SplitVecRes_FIX(N, Lo, Hi);
999 break;
1000 }
1001
1002 // If Lo/Hi is null, the sub-method took care of registering results etc.
1003 if (Lo.getNode())
1004 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1005 }
1006
IncrementPointer(MemSDNode * N,EVT MemVT,MachinePointerInfo & MPI,SDValue & Ptr,uint64_t * ScaledOffset)1007 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1008 MachinePointerInfo &MPI, SDValue &Ptr,
1009 uint64_t *ScaledOffset) {
1010 SDLoc DL(N);
1011 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1012
1013 if (MemVT.isScalableVector()) {
1014 SDNodeFlags Flags;
1015 SDValue BytesIncrement = DAG.getVScale(
1016 DL, Ptr.getValueType(),
1017 APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1018 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1019 Flags.setNoUnsignedWrap(true);
1020 if (ScaledOffset)
1021 *ScaledOffset += IncrementSize;
1022 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1023 Flags);
1024 } else {
1025 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1026 // Increment the pointer to the other half.
1027 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1028 }
1029 }
1030
SplitVecRes_BinOp(SDNode * N,SDValue & Lo,SDValue & Hi)1031 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
1032 SDValue &Hi) {
1033 SDValue LHSLo, LHSHi;
1034 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1035 SDValue RHSLo, RHSHi;
1036 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1037 SDLoc dl(N);
1038
1039 const SDNodeFlags Flags = N->getFlags();
1040 unsigned Opcode = N->getOpcode();
1041 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1042 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1043 }
1044
SplitVecRes_TernaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1045 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1046 SDValue &Hi) {
1047 SDValue Op0Lo, Op0Hi;
1048 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1049 SDValue Op1Lo, Op1Hi;
1050 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1051 SDValue Op2Lo, Op2Hi;
1052 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1053 SDLoc dl(N);
1054
1055 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo,
1056 Op2Lo, N->getFlags());
1057 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi,
1058 Op2Hi, N->getFlags());
1059 }
1060
SplitVecRes_FIX(SDNode * N,SDValue & Lo,SDValue & Hi)1061 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1062 SDValue LHSLo, LHSHi;
1063 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1064 SDValue RHSLo, RHSHi;
1065 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1066 SDLoc dl(N);
1067 SDValue Op2 = N->getOperand(2);
1068
1069 unsigned Opcode = N->getOpcode();
1070 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1071 N->getFlags());
1072 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1073 N->getFlags());
1074 }
1075
SplitVecRes_BITCAST(SDNode * N,SDValue & Lo,SDValue & Hi)1076 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1077 SDValue &Hi) {
1078 // We know the result is a vector. The input may be either a vector or a
1079 // scalar value.
1080 EVT LoVT, HiVT;
1081 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1082 SDLoc dl(N);
1083
1084 SDValue InOp = N->getOperand(0);
1085 EVT InVT = InOp.getValueType();
1086
1087 // Handle some special cases efficiently.
1088 switch (getTypeAction(InVT)) {
1089 case TargetLowering::TypeLegal:
1090 case TargetLowering::TypePromoteInteger:
1091 case TargetLowering::TypePromoteFloat:
1092 case TargetLowering::TypeSoftPromoteHalf:
1093 case TargetLowering::TypeSoftenFloat:
1094 case TargetLowering::TypeScalarizeVector:
1095 case TargetLowering::TypeWidenVector:
1096 break;
1097 case TargetLowering::TypeExpandInteger:
1098 case TargetLowering::TypeExpandFloat:
1099 // A scalar to vector conversion, where the scalar needs expansion.
1100 // If the vector is being split in two then we can just convert the
1101 // expanded pieces.
1102 if (LoVT == HiVT) {
1103 GetExpandedOp(InOp, Lo, Hi);
1104 if (DAG.getDataLayout().isBigEndian())
1105 std::swap(Lo, Hi);
1106 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1107 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1108 return;
1109 }
1110 break;
1111 case TargetLowering::TypeSplitVector:
1112 // If the input is a vector that needs to be split, convert each split
1113 // piece of the input now.
1114 GetSplitVector(InOp, Lo, Hi);
1115 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1116 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1117 return;
1118 case TargetLowering::TypeScalarizeScalableVector:
1119 report_fatal_error("Scalarization of scalable vectors is not supported.");
1120 }
1121
1122 // In the general case, convert the input to an integer and split it by hand.
1123 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1124 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1125 if (DAG.getDataLayout().isBigEndian())
1126 std::swap(LoIntVT, HiIntVT);
1127
1128 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1129
1130 if (DAG.getDataLayout().isBigEndian())
1131 std::swap(Lo, Hi);
1132 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1133 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1134 }
1135
SplitVecRes_BUILD_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1136 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1137 SDValue &Hi) {
1138 EVT LoVT, HiVT;
1139 SDLoc dl(N);
1140 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1141 unsigned LoNumElts = LoVT.getVectorNumElements();
1142 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1143 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1144
1145 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1146 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1147 }
1148
SplitVecRes_CONCAT_VECTORS(SDNode * N,SDValue & Lo,SDValue & Hi)1149 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1150 SDValue &Hi) {
1151 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1152 SDLoc dl(N);
1153 unsigned NumSubvectors = N->getNumOperands() / 2;
1154 if (NumSubvectors == 1) {
1155 Lo = N->getOperand(0);
1156 Hi = N->getOperand(1);
1157 return;
1158 }
1159
1160 EVT LoVT, HiVT;
1161 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1162
1163 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1164 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1165
1166 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1167 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1168 }
1169
SplitVecRes_EXTRACT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1170 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1171 SDValue &Hi) {
1172 SDValue Vec = N->getOperand(0);
1173 SDValue Idx = N->getOperand(1);
1174 SDLoc dl(N);
1175
1176 EVT LoVT, HiVT;
1177 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1178
1179 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1180 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1181 Hi = DAG.getNode(
1182 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1183 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorNumElements(), dl));
1184 }
1185
SplitVecRes_INSERT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1186 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1187 SDValue &Hi) {
1188 SDValue Vec = N->getOperand(0);
1189 SDValue SubVec = N->getOperand(1);
1190 SDValue Idx = N->getOperand(2);
1191 SDLoc dl(N);
1192 GetSplitVector(Vec, Lo, Hi);
1193
1194 EVT VecVT = Vec.getValueType();
1195 unsigned VecElems = VecVT.getVectorNumElements();
1196 unsigned SubElems = SubVec.getValueType().getVectorNumElements();
1197
1198 // If we know the index is 0, and we know the subvector doesn't cross the
1199 // boundary between the halves, we can avoid spilling the vector, and insert
1200 // into the lower half of the split vector directly.
1201 // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
1202 // there is no boundary crossing. But those cases don't seem to get hit in
1203 // practice.
1204 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1205 if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
1206 EVT LoVT, HiVT;
1207 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1208 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1209 return;
1210 }
1211
1212 // Spill the vector to the stack.
1213 // In cases where the vector is illegal it will be broken down into parts
1214 // and stored in parts - we should use the alignment for the smallest part.
1215 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1216 SDValue StackPtr =
1217 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1218 auto &MF = DAG.getMachineFunction();
1219 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1220 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1221
1222 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1223 SmallestAlign);
1224
1225 // Store the new subvector into the specified index.
1226 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1227 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1228 MachinePointerInfo::getUnknownStack(MF));
1229
1230 // Load the Lo part from the stack slot.
1231 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1232 SmallestAlign);
1233
1234 // Increment the pointer to the other part.
1235 unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1236 StackPtr =
1237 DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(IncrementSize), dl);
1238
1239 // Load the Hi part from the stack slot.
1240 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr,
1241 PtrInfo.getWithOffset(IncrementSize), SmallestAlign);
1242 }
1243
SplitVecRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)1244 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1245 SDValue &Hi) {
1246 SDLoc dl(N);
1247 GetSplitVector(N->getOperand(0), Lo, Hi);
1248 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1249 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1250 }
1251
SplitVecRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1252 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1253 SDValue &Hi) {
1254 SDValue LHSLo, LHSHi;
1255 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1256 SDLoc DL(N);
1257
1258 SDValue RHSLo, RHSHi;
1259 SDValue RHS = N->getOperand(1);
1260 EVT RHSVT = RHS.getValueType();
1261 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1262 GetSplitVector(RHS, RHSLo, RHSHi);
1263 else
1264 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1265
1266
1267 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1268 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1269 }
1270
SplitVecRes_InregOp(SDNode * N,SDValue & Lo,SDValue & Hi)1271 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1272 SDValue &Hi) {
1273 SDValue LHSLo, LHSHi;
1274 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1275 SDLoc dl(N);
1276
1277 EVT LoVT, HiVT;
1278 std::tie(LoVT, HiVT) =
1279 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1280
1281 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1282 DAG.getValueType(LoVT));
1283 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1284 DAG.getValueType(HiVT));
1285 }
1286
SplitVecRes_ExtVecInRegOp(SDNode * N,SDValue & Lo,SDValue & Hi)1287 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1288 SDValue &Hi) {
1289 unsigned Opcode = N->getOpcode();
1290 SDValue N0 = N->getOperand(0);
1291
1292 SDLoc dl(N);
1293 SDValue InLo, InHi;
1294
1295 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1296 GetSplitVector(N0, InLo, InHi);
1297 else
1298 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1299
1300 EVT InLoVT = InLo.getValueType();
1301 unsigned InNumElements = InLoVT.getVectorNumElements();
1302
1303 EVT OutLoVT, OutHiVT;
1304 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1305 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1306 assert((2 * OutNumElements) <= InNumElements &&
1307 "Illegal extend vector in reg split");
1308
1309 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1310 // input vector (i.e. we only use InLo):
1311 // OutLo will extend the first OutNumElements from InLo.
1312 // OutHi will extend the next OutNumElements from InLo.
1313
1314 // Shuffle the elements from InLo for OutHi into the bottom elements to
1315 // create a 'fake' InHi.
1316 SmallVector<int, 8> SplitHi(InNumElements, -1);
1317 for (unsigned i = 0; i != OutNumElements; ++i)
1318 SplitHi[i] = i + OutNumElements;
1319 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1320
1321 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1322 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1323 }
1324
SplitVecRes_StrictFPOp(SDNode * N,SDValue & Lo,SDValue & Hi)1325 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1326 SDValue &Hi) {
1327 unsigned NumOps = N->getNumOperands();
1328 SDValue Chain = N->getOperand(0);
1329 EVT LoVT, HiVT;
1330 SDLoc dl(N);
1331 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1332
1333 SmallVector<SDValue, 4> OpsLo(NumOps);
1334 SmallVector<SDValue, 4> OpsHi(NumOps);
1335
1336 // The Chain is the first operand.
1337 OpsLo[0] = Chain;
1338 OpsHi[0] = Chain;
1339
1340 // Now process the remaining operands.
1341 for (unsigned i = 1; i < NumOps; ++i) {
1342 SDValue Op = N->getOperand(i);
1343 SDValue OpLo = Op;
1344 SDValue OpHi = Op;
1345
1346 EVT InVT = Op.getValueType();
1347 if (InVT.isVector()) {
1348 // If the input also splits, handle it directly for a
1349 // compile time speedup. Otherwise split it by hand.
1350 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1351 GetSplitVector(Op, OpLo, OpHi);
1352 else
1353 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1354 }
1355
1356 OpsLo[i] = OpLo;
1357 OpsHi[i] = OpHi;
1358 }
1359
1360 EVT LoValueVTs[] = {LoVT, MVT::Other};
1361 EVT HiValueVTs[] = {HiVT, MVT::Other};
1362 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1363 N->getFlags());
1364 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1365 N->getFlags());
1366
1367 // Build a factor node to remember that this Op is independent of the
1368 // other one.
1369 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1370 Lo.getValue(1), Hi.getValue(1));
1371
1372 // Legalize the chain result - switch anything that used the old chain to
1373 // use the new one.
1374 ReplaceValueWith(SDValue(N, 1), Chain);
1375 }
1376
UnrollVectorOp_StrictFP(SDNode * N,unsigned ResNE)1377 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1378 SDValue Chain = N->getOperand(0);
1379 EVT VT = N->getValueType(0);
1380 unsigned NE = VT.getVectorNumElements();
1381 EVT EltVT = VT.getVectorElementType();
1382 SDLoc dl(N);
1383
1384 SmallVector<SDValue, 8> Scalars;
1385 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1386
1387 // If ResNE is 0, fully unroll the vector op.
1388 if (ResNE == 0)
1389 ResNE = NE;
1390 else if (NE > ResNE)
1391 NE = ResNE;
1392
1393 //The results of each unrolled operation, including the chain.
1394 EVT ChainVTs[] = {EltVT, MVT::Other};
1395 SmallVector<SDValue, 8> Chains;
1396
1397 unsigned i;
1398 for (i = 0; i != NE; ++i) {
1399 Operands[0] = Chain;
1400 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1401 SDValue Operand = N->getOperand(j);
1402 EVT OperandVT = Operand.getValueType();
1403 if (OperandVT.isVector()) {
1404 EVT OperandEltVT = OperandVT.getVectorElementType();
1405 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1406 Operand, DAG.getVectorIdxConstant(i, dl));
1407 } else {
1408 Operands[j] = Operand;
1409 }
1410 }
1411 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1412 Scalar.getNode()->setFlags(N->getFlags());
1413
1414 //Add in the scalar as well as its chain value to the
1415 //result vectors.
1416 Scalars.push_back(Scalar);
1417 Chains.push_back(Scalar.getValue(1));
1418 }
1419
1420 for (; i < ResNE; ++i)
1421 Scalars.push_back(DAG.getUNDEF(EltVT));
1422
1423 // Build a new factor node to connect the chain back together.
1424 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1425 ReplaceValueWith(SDValue(N, 1), Chain);
1426
1427 // Create a new BUILD_VECTOR node
1428 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1429 return DAG.getBuildVector(VecVT, dl, Scalars);
1430 }
1431
SplitVecRes_OverflowOp(SDNode * N,unsigned ResNo,SDValue & Lo,SDValue & Hi)1432 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1433 SDValue &Lo, SDValue &Hi) {
1434 SDLoc dl(N);
1435 EVT ResVT = N->getValueType(0);
1436 EVT OvVT = N->getValueType(1);
1437 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1438 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1439 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1440
1441 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1442 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1443 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1444 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1445 } else {
1446 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1447 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1448 }
1449
1450 unsigned Opcode = N->getOpcode();
1451 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1452 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1453 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1454 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1455 LoNode->setFlags(N->getFlags());
1456 HiNode->setFlags(N->getFlags());
1457
1458 Lo = SDValue(LoNode, ResNo);
1459 Hi = SDValue(HiNode, ResNo);
1460
1461 // Replace the other vector result not being explicitly split here.
1462 unsigned OtherNo = 1 - ResNo;
1463 EVT OtherVT = N->getValueType(OtherNo);
1464 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1465 SetSplitVector(SDValue(N, OtherNo),
1466 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1467 } else {
1468 SDValue OtherVal = DAG.getNode(
1469 ISD::CONCAT_VECTORS, dl, OtherVT,
1470 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1471 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1472 }
1473 }
1474
SplitVecRes_INSERT_VECTOR_ELT(SDNode * N,SDValue & Lo,SDValue & Hi)1475 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1476 SDValue &Hi) {
1477 SDValue Vec = N->getOperand(0);
1478 SDValue Elt = N->getOperand(1);
1479 SDValue Idx = N->getOperand(2);
1480 SDLoc dl(N);
1481 GetSplitVector(Vec, Lo, Hi);
1482
1483 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1484 unsigned IdxVal = CIdx->getZExtValue();
1485 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1486 if (IdxVal < LoNumElts) {
1487 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1488 Lo.getValueType(), Lo, Elt, Idx);
1489 return;
1490 } else if (!Vec.getValueType().isScalableVector()) {
1491 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1492 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1493 return;
1494 }
1495 }
1496
1497 // See if the target wants to custom expand this node.
1498 if (CustomLowerNode(N, N->getValueType(0), true))
1499 return;
1500
1501 // Make the vector elements byte-addressable if they aren't already.
1502 EVT VecVT = Vec.getValueType();
1503 EVT EltVT = VecVT.getVectorElementType();
1504 if (VecVT.getScalarSizeInBits() < 8) {
1505 EltVT = MVT::i8;
1506 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1507 VecVT.getVectorElementCount());
1508 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1509 // Extend the element type to match if needed.
1510 if (EltVT.bitsGT(Elt.getValueType()))
1511 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1512 }
1513
1514 // Spill the vector to the stack.
1515 // In cases where the vector is illegal it will be broken down into parts
1516 // and stored in parts - we should use the alignment for the smallest part.
1517 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1518 SDValue StackPtr =
1519 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1520 auto &MF = DAG.getMachineFunction();
1521 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1522 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1523
1524 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1525 SmallestAlign);
1526
1527 // Store the new element. This may be larger than the vector element type,
1528 // so use a truncating store.
1529 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1530 Store = DAG.getTruncStore(
1531 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1532 commonAlignment(SmallestAlign,
1533 EltVT.getFixedSizeInBits() / 8));
1534
1535 EVT LoVT, HiVT;
1536 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1537
1538 // Load the Lo part from the stack slot.
1539 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1540
1541 // Increment the pointer to the other part.
1542 auto Load = cast<LoadSDNode>(Lo);
1543 MachinePointerInfo MPI = Load->getPointerInfo();
1544 IncrementPointer(Load, LoVT, MPI, StackPtr);
1545
1546 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1547
1548 // If we adjusted the original type, we need to truncate the results.
1549 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1550 if (LoVT != Lo.getValueType())
1551 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1552 if (HiVT != Hi.getValueType())
1553 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1554 }
1555
SplitVecRes_ScalarOp(SDNode * N,SDValue & Lo,SDValue & Hi)1556 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1557 SDValue &Hi) {
1558 EVT LoVT, HiVT;
1559 SDLoc dl(N);
1560 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1561 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1562 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1563 Hi = DAG.getUNDEF(HiVT);
1564 } else {
1565 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1566 Hi = Lo;
1567 }
1568 }
1569
SplitVecRes_LOAD(LoadSDNode * LD,SDValue & Lo,SDValue & Hi)1570 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1571 SDValue &Hi) {
1572 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1573 EVT LoVT, HiVT;
1574 SDLoc dl(LD);
1575 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1576
1577 ISD::LoadExtType ExtType = LD->getExtensionType();
1578 SDValue Ch = LD->getChain();
1579 SDValue Ptr = LD->getBasePtr();
1580 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1581 EVT MemoryVT = LD->getMemoryVT();
1582 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1583 AAMDNodes AAInfo = LD->getAAInfo();
1584
1585 EVT LoMemVT, HiMemVT;
1586 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1587
1588 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1589 SDValue Value, NewChain;
1590 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1591 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1592 ReplaceValueWith(SDValue(LD, 1), NewChain);
1593 return;
1594 }
1595
1596 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1597 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1598 MMOFlags, AAInfo);
1599
1600 MachinePointerInfo MPI;
1601 IncrementPointer(LD, LoMemVT, MPI, Ptr);
1602
1603 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1604 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1605
1606 // Build a factor node to remember that this load is independent of the
1607 // other one.
1608 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1609 Hi.getValue(1));
1610
1611 // Legalize the chain result - switch anything that used the old chain to
1612 // use the new one.
1613 ReplaceValueWith(SDValue(LD, 1), Ch);
1614 }
1615
SplitVecRes_MLOAD(MaskedLoadSDNode * MLD,SDValue & Lo,SDValue & Hi)1616 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1617 SDValue &Lo, SDValue &Hi) {
1618 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1619 EVT LoVT, HiVT;
1620 SDLoc dl(MLD);
1621 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1622
1623 SDValue Ch = MLD->getChain();
1624 SDValue Ptr = MLD->getBasePtr();
1625 SDValue Offset = MLD->getOffset();
1626 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1627 SDValue Mask = MLD->getMask();
1628 SDValue PassThru = MLD->getPassThru();
1629 Align Alignment = MLD->getOriginalAlign();
1630 ISD::LoadExtType ExtType = MLD->getExtensionType();
1631
1632 // Split Mask operand
1633 SDValue MaskLo, MaskHi;
1634 if (Mask.getOpcode() == ISD::SETCC) {
1635 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1636 } else {
1637 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1638 GetSplitVector(Mask, MaskLo, MaskHi);
1639 else
1640 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1641 }
1642
1643 EVT MemoryVT = MLD->getMemoryVT();
1644 EVT LoMemVT, HiMemVT;
1645 bool HiIsEmpty = false;
1646 std::tie(LoMemVT, HiMemVT) =
1647 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1648
1649 SDValue PassThruLo, PassThruHi;
1650 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1651 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1652 else
1653 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1654
1655 unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
1656 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1657 MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoSize, Alignment,
1658 MLD->getAAInfo(), MLD->getRanges());
1659
1660 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1661 MMO, MLD->getAddressingMode(), ExtType,
1662 MLD->isExpandingLoad());
1663
1664 if (HiIsEmpty) {
1665 // The hi masked load has zero storage size. We therefore simply set it to
1666 // the low masked load and rely on subsequent removal from the chain.
1667 Hi = Lo;
1668 } else {
1669 // Generate hi masked load.
1670 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1671 MLD->isExpandingLoad());
1672 unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
1673
1674 MachinePointerInfo MPI;
1675 if (LoMemVT.isScalableVector())
1676 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
1677 else
1678 MPI = MLD->getPointerInfo().getWithOffset(
1679 LoMemVT.getStoreSize().getFixedSize());
1680
1681 MMO = DAG.getMachineFunction().getMachineMemOperand(
1682 MPI, MachineMemOperand::MOLoad, HiSize, Alignment, MLD->getAAInfo(),
1683 MLD->getRanges());
1684
1685 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
1686 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
1687 MLD->isExpandingLoad());
1688 }
1689
1690 // Build a factor node to remember that this load is independent of the
1691 // other one.
1692 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1693 Hi.getValue(1));
1694
1695 // Legalize the chain result - switch anything that used the old chain to
1696 // use the new one.
1697 ReplaceValueWith(SDValue(MLD, 1), Ch);
1698
1699 }
1700
SplitVecRes_MGATHER(MaskedGatherSDNode * MGT,SDValue & Lo,SDValue & Hi)1701 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1702 SDValue &Lo, SDValue &Hi) {
1703 EVT LoVT, HiVT;
1704 SDLoc dl(MGT);
1705 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1706
1707 SDValue Ch = MGT->getChain();
1708 SDValue Ptr = MGT->getBasePtr();
1709 SDValue Mask = MGT->getMask();
1710 SDValue PassThru = MGT->getPassThru();
1711 SDValue Index = MGT->getIndex();
1712 SDValue Scale = MGT->getScale();
1713 Align Alignment = MGT->getOriginalAlign();
1714
1715 // Split Mask operand
1716 SDValue MaskLo, MaskHi;
1717 if (Mask.getOpcode() == ISD::SETCC) {
1718 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1719 } else {
1720 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1721 GetSplitVector(Mask, MaskLo, MaskHi);
1722 else
1723 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1724 }
1725
1726 SDValue PassThruLo, PassThruHi;
1727 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1728 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1729 else
1730 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1731
1732 SDValue IndexHi, IndexLo;
1733 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1734 GetSplitVector(Index, IndexLo, IndexHi);
1735 else
1736 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1737
1738 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1739 MGT->getPointerInfo(), MachineMemOperand::MOLoad,
1740 MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
1741 MGT->getRanges());
1742
1743 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1744 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1745 MMO, MGT->getIndexType());
1746
1747 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1748 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1749 MMO, MGT->getIndexType());
1750
1751 // Build a factor node to remember that this load is independent of the
1752 // other one.
1753 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1754 Hi.getValue(1));
1755
1756 // Legalize the chain result - switch anything that used the old chain to
1757 // use the new one.
1758 ReplaceValueWith(SDValue(MGT, 1), Ch);
1759 }
1760
1761
SplitVecRes_SETCC(SDNode * N,SDValue & Lo,SDValue & Hi)1762 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1763 assert(N->getValueType(0).isVector() &&
1764 N->getOperand(0).getValueType().isVector() &&
1765 "Operand types must be vectors");
1766
1767 EVT LoVT, HiVT;
1768 SDLoc DL(N);
1769 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1770
1771 // If the input also splits, handle it directly. Otherwise split it by hand.
1772 SDValue LL, LH, RL, RH;
1773 if (getTypeAction(N->getOperand(0).getValueType()) ==
1774 TargetLowering::TypeSplitVector)
1775 GetSplitVector(N->getOperand(0), LL, LH);
1776 else
1777 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1778
1779 if (getTypeAction(N->getOperand(1).getValueType()) ==
1780 TargetLowering::TypeSplitVector)
1781 GetSplitVector(N->getOperand(1), RL, RH);
1782 else
1783 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1784
1785 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1786 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1787 }
1788
SplitVecRes_UnaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1789 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1790 SDValue &Hi) {
1791 // Get the dest types - they may not match the input types, e.g. int_to_fp.
1792 EVT LoVT, HiVT;
1793 SDLoc dl(N);
1794 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1795
1796 // If the input also splits, handle it directly for a compile time speedup.
1797 // Otherwise split it by hand.
1798 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1799 EVT InVT = N->getOperand(OpNo).getValueType();
1800 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1801 GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1802 else
1803 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1804
1805 if (N->getOpcode() == ISD::FP_ROUND) {
1806 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1),
1807 N->getFlags());
1808 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1),
1809 N->getFlags());
1810 } else {
1811 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags());
1812 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags());
1813 }
1814 }
1815
SplitVecRes_ExtendOp(SDNode * N,SDValue & Lo,SDValue & Hi)1816 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1817 SDValue &Hi) {
1818 SDLoc dl(N);
1819 EVT SrcVT = N->getOperand(0).getValueType();
1820 EVT DestVT = N->getValueType(0);
1821 EVT LoVT, HiVT;
1822 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1823
1824 // We can do better than a generic split operation if the extend is doing
1825 // more than just doubling the width of the elements and the following are
1826 // true:
1827 // - The number of vector elements is even,
1828 // - the source type is legal,
1829 // - the type of a split source is illegal,
1830 // - the type of an extended (by doubling element size) source is legal, and
1831 // - the type of that extended source when split is legal.
1832 //
1833 // This won't necessarily completely legalize the operation, but it will
1834 // more effectively move in the right direction and prevent falling down
1835 // to scalarization in many cases due to the input vector being split too
1836 // far.
1837 if (SrcVT.getVectorElementCount().isKnownEven() &&
1838 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
1839 LLVMContext &Ctx = *DAG.getContext();
1840 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1841 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1842
1843 EVT SplitLoVT, SplitHiVT;
1844 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1845 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1846 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1847 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1848 N->dump(&DAG); dbgs() << "\n");
1849 // Extend the source vector by one step.
1850 SDValue NewSrc =
1851 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1852 // Get the low and high halves of the new, extended one step, vector.
1853 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1854 // Extend those vector halves the rest of the way.
1855 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1856 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1857 return;
1858 }
1859 }
1860 // Fall back to the generic unary operator splitting otherwise.
1861 SplitVecRes_UnaryOp(N, Lo, Hi);
1862 }
1863
SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N,SDValue & Lo,SDValue & Hi)1864 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1865 SDValue &Lo, SDValue &Hi) {
1866 // The low and high parts of the original input give four input vectors.
1867 SDValue Inputs[4];
1868 SDLoc dl(N);
1869 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1870 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1871 EVT NewVT = Inputs[0].getValueType();
1872 unsigned NewElts = NewVT.getVectorNumElements();
1873
1874 // If Lo or Hi uses elements from at most two of the four input vectors, then
1875 // express it as a vector shuffle of those two inputs. Otherwise extract the
1876 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1877 SmallVector<int, 16> Ops;
1878 for (unsigned High = 0; High < 2; ++High) {
1879 SDValue &Output = High ? Hi : Lo;
1880
1881 // Build a shuffle mask for the output, discovering on the fly which
1882 // input vectors to use as shuffle operands (recorded in InputUsed).
1883 // If building a suitable shuffle vector proves too hard, then bail
1884 // out with useBuildVector set.
1885 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1886 unsigned FirstMaskIdx = High * NewElts;
1887 bool useBuildVector = false;
1888 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1889 // The mask element. This indexes into the input.
1890 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1891
1892 // The input vector this mask element indexes into.
1893 unsigned Input = (unsigned)Idx / NewElts;
1894
1895 if (Input >= array_lengthof(Inputs)) {
1896 // The mask element does not index into any input vector.
1897 Ops.push_back(-1);
1898 continue;
1899 }
1900
1901 // Turn the index into an offset from the start of the input vector.
1902 Idx -= Input * NewElts;
1903
1904 // Find or create a shuffle vector operand to hold this input.
1905 unsigned OpNo;
1906 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1907 if (InputUsed[OpNo] == Input) {
1908 // This input vector is already an operand.
1909 break;
1910 } else if (InputUsed[OpNo] == -1U) {
1911 // Create a new operand for this input vector.
1912 InputUsed[OpNo] = Input;
1913 break;
1914 }
1915 }
1916
1917 if (OpNo >= array_lengthof(InputUsed)) {
1918 // More than two input vectors used! Give up on trying to create a
1919 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1920 useBuildVector = true;
1921 break;
1922 }
1923
1924 // Add the mask index for the new shuffle vector.
1925 Ops.push_back(Idx + OpNo * NewElts);
1926 }
1927
1928 if (useBuildVector) {
1929 EVT EltVT = NewVT.getVectorElementType();
1930 SmallVector<SDValue, 16> SVOps;
1931
1932 // Extract the input elements by hand.
1933 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1934 // The mask element. This indexes into the input.
1935 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1936
1937 // The input vector this mask element indexes into.
1938 unsigned Input = (unsigned)Idx / NewElts;
1939
1940 if (Input >= array_lengthof(Inputs)) {
1941 // The mask element is "undef" or indexes off the end of the input.
1942 SVOps.push_back(DAG.getUNDEF(EltVT));
1943 continue;
1944 }
1945
1946 // Turn the index into an offset from the start of the input vector.
1947 Idx -= Input * NewElts;
1948
1949 // Extract the vector element by hand.
1950 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
1951 Inputs[Input],
1952 DAG.getVectorIdxConstant(Idx, dl)));
1953 }
1954
1955 // Construct the Lo/Hi output using a BUILD_VECTOR.
1956 Output = DAG.getBuildVector(NewVT, dl, SVOps);
1957 } else if (InputUsed[0] == -1U) {
1958 // No input vectors were used! The result is undefined.
1959 Output = DAG.getUNDEF(NewVT);
1960 } else {
1961 SDValue Op0 = Inputs[InputUsed[0]];
1962 // If only one input was used, use an undefined vector for the other.
1963 SDValue Op1 = InputUsed[1] == -1U ?
1964 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1965 // At least one input vector was used. Create a new shuffle vector.
1966 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1967 }
1968
1969 Ops.clear();
1970 }
1971 }
1972
SplitVecRes_VAARG(SDNode * N,SDValue & Lo,SDValue & Hi)1973 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
1974 EVT OVT = N->getValueType(0);
1975 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
1976 SDValue Chain = N->getOperand(0);
1977 SDValue Ptr = N->getOperand(1);
1978 SDValue SV = N->getOperand(2);
1979 SDLoc dl(N);
1980
1981 const Align Alignment =
1982 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
1983
1984 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
1985 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
1986 Chain = Hi.getValue(1);
1987
1988 // Modified the chain - switch anything that used the old chain to use
1989 // the new one.
1990 ReplaceValueWith(SDValue(N, 1), Chain);
1991 }
1992
1993
1994 //===----------------------------------------------------------------------===//
1995 // Operand Vector Splitting
1996 //===----------------------------------------------------------------------===//
1997
1998 /// This method is called when the specified operand of the specified node is
1999 /// found to need vector splitting. At this point, all of the result types of
2000 /// the node are known to be legal, but other operands of the node may need
2001 /// legalization as well as the specified one.
SplitVectorOperand(SDNode * N,unsigned OpNo)2002 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2003 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2004 SDValue Res = SDValue();
2005
2006 // See if the target wants to custom split this node.
2007 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2008 return false;
2009
2010 switch (N->getOpcode()) {
2011 default:
2012 #ifndef NDEBUG
2013 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2014 N->dump(&DAG);
2015 dbgs() << "\n";
2016 #endif
2017 report_fatal_error("Do not know how to split this operator's "
2018 "operand!\n");
2019
2020 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2021 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2022 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2023 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2024 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2025 case ISD::TRUNCATE:
2026 Res = SplitVecOp_TruncateHelper(N);
2027 break;
2028 case ISD::STRICT_FP_ROUND:
2029 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2030 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2031 case ISD::STORE:
2032 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2033 break;
2034 case ISD::MSTORE:
2035 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2036 break;
2037 case ISD::MSCATTER:
2038 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
2039 break;
2040 case ISD::MGATHER:
2041 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
2042 break;
2043 case ISD::VSELECT:
2044 Res = SplitVecOp_VSELECT(N, OpNo);
2045 break;
2046 case ISD::STRICT_SINT_TO_FP:
2047 case ISD::STRICT_UINT_TO_FP:
2048 case ISD::SINT_TO_FP:
2049 case ISD::UINT_TO_FP:
2050 if (N->getValueType(0).bitsLT(
2051 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2052 Res = SplitVecOp_TruncateHelper(N);
2053 else
2054 Res = SplitVecOp_UnaryOp(N);
2055 break;
2056 case ISD::FP_TO_SINT:
2057 case ISD::FP_TO_UINT:
2058 case ISD::STRICT_FP_TO_SINT:
2059 case ISD::STRICT_FP_TO_UINT:
2060 case ISD::STRICT_FP_EXTEND:
2061 case ISD::FP_EXTEND:
2062 case ISD::SIGN_EXTEND:
2063 case ISD::ZERO_EXTEND:
2064 case ISD::ANY_EXTEND:
2065 case ISD::FTRUNC:
2066 Res = SplitVecOp_UnaryOp(N);
2067 break;
2068
2069 case ISD::ANY_EXTEND_VECTOR_INREG:
2070 case ISD::SIGN_EXTEND_VECTOR_INREG:
2071 case ISD::ZERO_EXTEND_VECTOR_INREG:
2072 Res = SplitVecOp_ExtVecInRegOp(N);
2073 break;
2074
2075 case ISD::VECREDUCE_FADD:
2076 case ISD::VECREDUCE_FMUL:
2077 case ISD::VECREDUCE_ADD:
2078 case ISD::VECREDUCE_MUL:
2079 case ISD::VECREDUCE_AND:
2080 case ISD::VECREDUCE_OR:
2081 case ISD::VECREDUCE_XOR:
2082 case ISD::VECREDUCE_SMAX:
2083 case ISD::VECREDUCE_SMIN:
2084 case ISD::VECREDUCE_UMAX:
2085 case ISD::VECREDUCE_UMIN:
2086 case ISD::VECREDUCE_FMAX:
2087 case ISD::VECREDUCE_FMIN:
2088 Res = SplitVecOp_VECREDUCE(N, OpNo);
2089 break;
2090 case ISD::VECREDUCE_SEQ_FADD:
2091 case ISD::VECREDUCE_SEQ_FMUL:
2092 Res = SplitVecOp_VECREDUCE_SEQ(N);
2093 break;
2094 }
2095
2096 // If the result is null, the sub-method took care of registering results etc.
2097 if (!Res.getNode()) return false;
2098
2099 // If the result is N, the sub-method updated N in place. Tell the legalizer
2100 // core about this.
2101 if (Res.getNode() == N)
2102 return true;
2103
2104 if (N->isStrictFPOpcode())
2105 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2106 "Invalid operand expansion");
2107 else
2108 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2109 "Invalid operand expansion");
2110
2111 ReplaceValueWith(SDValue(N, 0), Res);
2112 return false;
2113 }
2114
SplitVecOp_VSELECT(SDNode * N,unsigned OpNo)2115 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2116 // The only possibility for an illegal operand is the mask, since result type
2117 // legalization would have handled this node already otherwise.
2118 assert(OpNo == 0 && "Illegal operand must be mask");
2119
2120 SDValue Mask = N->getOperand(0);
2121 SDValue Src0 = N->getOperand(1);
2122 SDValue Src1 = N->getOperand(2);
2123 EVT Src0VT = Src0.getValueType();
2124 SDLoc DL(N);
2125 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2126
2127 SDValue Lo, Hi;
2128 GetSplitVector(N->getOperand(0), Lo, Hi);
2129 assert(Lo.getValueType() == Hi.getValueType() &&
2130 "Lo and Hi have differing types");
2131
2132 EVT LoOpVT, HiOpVT;
2133 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2134 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2135
2136 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2137 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2138 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2139 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2140
2141 SDValue LoSelect =
2142 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2143 SDValue HiSelect =
2144 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2145
2146 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2147 }
2148
SplitVecOp_VECREDUCE(SDNode * N,unsigned OpNo)2149 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2150 EVT ResVT = N->getValueType(0);
2151 SDValue Lo, Hi;
2152 SDLoc dl(N);
2153
2154 SDValue VecOp = N->getOperand(OpNo);
2155 EVT VecVT = VecOp.getValueType();
2156 assert(VecVT.isVector() && "Can only split reduce vector operand");
2157 GetSplitVector(VecOp, Lo, Hi);
2158 EVT LoOpVT, HiOpVT;
2159 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2160
2161 // Use the appropriate scalar instruction on the split subvectors before
2162 // reducing the now partially reduced smaller vector.
2163 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2164 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2165 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2166 }
2167
SplitVecOp_VECREDUCE_SEQ(SDNode * N)2168 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2169 EVT ResVT = N->getValueType(0);
2170 SDValue Lo, Hi;
2171 SDLoc dl(N);
2172
2173 SDValue AccOp = N->getOperand(0);
2174 SDValue VecOp = N->getOperand(1);
2175 SDNodeFlags Flags = N->getFlags();
2176
2177 EVT VecVT = VecOp.getValueType();
2178 assert(VecVT.isVector() && "Can only split reduce vector operand");
2179 GetSplitVector(VecOp, Lo, Hi);
2180 EVT LoOpVT, HiOpVT;
2181 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2182
2183 // Reduce low half.
2184 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2185
2186 // Reduce high half, using low half result as initial value.
2187 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2188 }
2189
SplitVecOp_UnaryOp(SDNode * N)2190 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2191 // The result has a legal vector type, but the input needs splitting.
2192 EVT ResVT = N->getValueType(0);
2193 SDValue Lo, Hi;
2194 SDLoc dl(N);
2195 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2196 EVT InVT = Lo.getValueType();
2197
2198 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2199 InVT.getVectorElementCount());
2200
2201 if (N->isStrictFPOpcode()) {
2202 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2203 { N->getOperand(0), Lo });
2204 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2205 { N->getOperand(0), Hi });
2206
2207 // Build a factor node to remember that this operation is independent
2208 // of the other one.
2209 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2210 Hi.getValue(1));
2211
2212 // Legalize the chain result - switch anything that used the old chain to
2213 // use the new one.
2214 ReplaceValueWith(SDValue(N, 1), Ch);
2215 } else {
2216 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2217 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2218 }
2219
2220 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2221 }
2222
SplitVecOp_BITCAST(SDNode * N)2223 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2224 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2225 // end up being split all the way down to individual components. Convert the
2226 // split pieces into integers and reassemble.
2227 SDValue Lo, Hi;
2228 GetSplitVector(N->getOperand(0), Lo, Hi);
2229 Lo = BitConvertToInteger(Lo);
2230 Hi = BitConvertToInteger(Hi);
2231
2232 if (DAG.getDataLayout().isBigEndian())
2233 std::swap(Lo, Hi);
2234
2235 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2236 JoinIntegers(Lo, Hi));
2237 }
2238
SplitVecOp_EXTRACT_SUBVECTOR(SDNode * N)2239 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2240 // We know that the extracted result type is legal.
2241 EVT SubVT = N->getValueType(0);
2242
2243 if (SubVT.isScalableVector() !=
2244 N->getOperand(0).getValueType().isScalableVector())
2245 report_fatal_error("Extracting fixed from scalable not implemented");
2246
2247 SDValue Idx = N->getOperand(1);
2248 SDLoc dl(N);
2249 SDValue Lo, Hi;
2250
2251 if (SubVT.isScalableVector() !=
2252 N->getOperand(0).getValueType().isScalableVector())
2253 report_fatal_error("Extracting a fixed-length vector from an illegal "
2254 "scalable vector is not yet supported");
2255
2256 GetSplitVector(N->getOperand(0), Lo, Hi);
2257
2258 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2259 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2260
2261 if (IdxVal < LoElts) {
2262 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
2263 "Extracted subvector crosses vector split!");
2264 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2265 } else {
2266 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2267 DAG.getVectorIdxConstant(IdxVal - LoElts, dl));
2268 }
2269 }
2270
SplitVecOp_EXTRACT_VECTOR_ELT(SDNode * N)2271 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2272 SDValue Vec = N->getOperand(0);
2273 SDValue Idx = N->getOperand(1);
2274 EVT VecVT = Vec.getValueType();
2275
2276 if (isa<ConstantSDNode>(Idx)) {
2277 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2278
2279 SDValue Lo, Hi;
2280 GetSplitVector(Vec, Lo, Hi);
2281
2282 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2283
2284 if (IdxVal < LoElts)
2285 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2286 else if (!Vec.getValueType().isScalableVector())
2287 return SDValue(DAG.UpdateNodeOperands(N, Hi,
2288 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2289 Idx.getValueType())), 0);
2290 }
2291
2292 // See if the target wants to custom expand this node.
2293 if (CustomLowerNode(N, N->getValueType(0), true))
2294 return SDValue();
2295
2296 // Make the vector elements byte-addressable if they aren't already.
2297 SDLoc dl(N);
2298 EVT EltVT = VecVT.getVectorElementType();
2299 if (VecVT.getScalarSizeInBits() < 8) {
2300 EltVT = MVT::i8;
2301 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2302 VecVT.getVectorElementCount());
2303 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2304 }
2305
2306 // Store the vector to the stack.
2307 // In cases where the vector is illegal it will be broken down into parts
2308 // and stored in parts - we should use the alignment for the smallest part.
2309 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
2310 SDValue StackPtr =
2311 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
2312 auto &MF = DAG.getMachineFunction();
2313 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2314 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2315 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2316 SmallestAlign);
2317
2318 // Load back the required element.
2319 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2320
2321 // FIXME: This is to handle i1 vectors with elements promoted to i8.
2322 // i1 vector handling needs general improvement.
2323 if (N->getValueType(0).bitsLT(EltVT)) {
2324 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2325 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
2326 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2327 }
2328
2329 return DAG.getExtLoad(
2330 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2331 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
2332 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
2333 }
2334
SplitVecOp_ExtVecInRegOp(SDNode * N)2335 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2336 SDValue Lo, Hi;
2337
2338 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2339 // splitting the result has the same effect as splitting the input operand.
2340 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2341
2342 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2343 }
2344
SplitVecOp_MGATHER(MaskedGatherSDNode * MGT,unsigned OpNo)2345 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2346 unsigned OpNo) {
2347 EVT LoVT, HiVT;
2348 SDLoc dl(MGT);
2349 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2350
2351 SDValue Ch = MGT->getChain();
2352 SDValue Ptr = MGT->getBasePtr();
2353 SDValue Index = MGT->getIndex();
2354 SDValue Scale = MGT->getScale();
2355 SDValue Mask = MGT->getMask();
2356 SDValue PassThru = MGT->getPassThru();
2357 Align Alignment = MGT->getOriginalAlign();
2358
2359 SDValue MaskLo, MaskHi;
2360 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2361 // Split Mask operand
2362 GetSplitVector(Mask, MaskLo, MaskHi);
2363 else
2364 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2365
2366 EVT MemoryVT = MGT->getMemoryVT();
2367 EVT LoMemVT, HiMemVT;
2368 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2369
2370 SDValue PassThruLo, PassThruHi;
2371 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2372 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2373 else
2374 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2375
2376 SDValue IndexHi, IndexLo;
2377 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2378 GetSplitVector(Index, IndexLo, IndexHi);
2379 else
2380 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2381
2382 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2383 MGT->getPointerInfo(), MachineMemOperand::MOLoad,
2384 MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
2385 MGT->getRanges());
2386
2387 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2388 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2389 OpsLo, MMO, MGT->getIndexType());
2390
2391 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2392 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2393 OpsHi, MMO, MGT->getIndexType());
2394
2395 // Build a factor node to remember that this load is independent of the
2396 // other one.
2397 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2398 Hi.getValue(1));
2399
2400 // Legalize the chain result - switch anything that used the old chain to
2401 // use the new one.
2402 ReplaceValueWith(SDValue(MGT, 1), Ch);
2403
2404 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2405 Hi);
2406 ReplaceValueWith(SDValue(MGT, 0), Res);
2407 return SDValue();
2408 }
2409
SplitVecOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)2410 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2411 unsigned OpNo) {
2412 assert(N->isUnindexed() && "Indexed masked store of vector?");
2413 SDValue Ch = N->getChain();
2414 SDValue Ptr = N->getBasePtr();
2415 SDValue Offset = N->getOffset();
2416 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
2417 SDValue Mask = N->getMask();
2418 SDValue Data = N->getValue();
2419 Align Alignment = N->getOriginalAlign();
2420 SDLoc DL(N);
2421
2422 SDValue DataLo, DataHi;
2423 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2424 // Split Data operand
2425 GetSplitVector(Data, DataLo, DataHi);
2426 else
2427 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2428
2429 // Split Mask operand
2430 SDValue MaskLo, MaskHi;
2431 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2432 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2433 } else {
2434 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2435 GetSplitVector(Mask, MaskLo, MaskHi);
2436 else
2437 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2438 }
2439
2440 EVT MemoryVT = N->getMemoryVT();
2441 EVT LoMemVT, HiMemVT;
2442 bool HiIsEmpty = false;
2443 std::tie(LoMemVT, HiMemVT) =
2444 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
2445
2446 SDValue Lo, Hi, Res;
2447 unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
2448 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2449 N->getPointerInfo(), MachineMemOperand::MOStore, LoSize, Alignment,
2450 N->getAAInfo(), N->getRanges());
2451
2452 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
2453 N->getAddressingMode(), N->isTruncatingStore(),
2454 N->isCompressingStore());
2455
2456 if (HiIsEmpty) {
2457 // The hi masked store has zero storage size.
2458 // Only the lo masked store is needed.
2459 Res = Lo;
2460 } else {
2461
2462 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2463 N->isCompressingStore());
2464
2465 MachinePointerInfo MPI;
2466 if (LoMemVT.isScalableVector()) {
2467 Alignment = commonAlignment(
2468 Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
2469 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
2470 } else
2471 MPI = N->getPointerInfo().getWithOffset(
2472 LoMemVT.getStoreSize().getFixedSize());
2473
2474 unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
2475 MMO = DAG.getMachineFunction().getMachineMemOperand(
2476 MPI, MachineMemOperand::MOStore, HiSize, Alignment, N->getAAInfo(),
2477 N->getRanges());
2478
2479 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
2480 N->getAddressingMode(), N->isTruncatingStore(),
2481 N->isCompressingStore());
2482
2483 // Build a factor node to remember that this store is independent of the
2484 // other one.
2485 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2486 }
2487
2488 return Res;
2489 }
2490
SplitVecOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)2491 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2492 unsigned OpNo) {
2493 SDValue Ch = N->getChain();
2494 SDValue Ptr = N->getBasePtr();
2495 SDValue Mask = N->getMask();
2496 SDValue Index = N->getIndex();
2497 SDValue Scale = N->getScale();
2498 SDValue Data = N->getValue();
2499 EVT MemoryVT = N->getMemoryVT();
2500 Align Alignment = N->getOriginalAlign();
2501 SDLoc DL(N);
2502
2503 // Split all operands
2504
2505 EVT LoMemVT, HiMemVT;
2506 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2507
2508 SDValue DataLo, DataHi;
2509 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2510 // Split Data operand
2511 GetSplitVector(Data, DataLo, DataHi);
2512 else
2513 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2514
2515 // Split Mask operand
2516 SDValue MaskLo, MaskHi;
2517 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2518 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2519 } else {
2520 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2521 GetSplitVector(Mask, MaskLo, MaskHi);
2522 else
2523 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2524 }
2525
2526 SDValue IndexHi, IndexLo;
2527 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2528 GetSplitVector(Index, IndexLo, IndexHi);
2529 else
2530 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2531
2532 SDValue Lo;
2533 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2534 N->getPointerInfo(), MachineMemOperand::MOStore,
2535 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2536
2537 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2538 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT,
2539 DL, OpsLo, MMO, N->getIndexType(),
2540 N->isTruncatingStore());
2541
2542 // The order of the Scatter operation after split is well defined. The "Hi"
2543 // part comes after the "Lo". So these two operations should be chained one
2544 // after another.
2545 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2546 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT,
2547 DL, OpsHi, MMO, N->getIndexType(),
2548 N->isTruncatingStore());
2549 }
2550
SplitVecOp_STORE(StoreSDNode * N,unsigned OpNo)2551 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2552 assert(N->isUnindexed() && "Indexed store of vector?");
2553 assert(OpNo == 1 && "Can only split the stored value");
2554 SDLoc DL(N);
2555
2556 bool isTruncating = N->isTruncatingStore();
2557 SDValue Ch = N->getChain();
2558 SDValue Ptr = N->getBasePtr();
2559 EVT MemoryVT = N->getMemoryVT();
2560 Align Alignment = N->getOriginalAlign();
2561 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2562 AAMDNodes AAInfo = N->getAAInfo();
2563 SDValue Lo, Hi;
2564 GetSplitVector(N->getOperand(1), Lo, Hi);
2565
2566 EVT LoMemVT, HiMemVT;
2567 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2568
2569 // Scalarize if the split halves are not byte-sized.
2570 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2571 return TLI.scalarizeVectorStore(N, DAG);
2572
2573 if (isTruncating)
2574 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2575 Alignment, MMOFlags, AAInfo);
2576 else
2577 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2578 AAInfo);
2579
2580 MachinePointerInfo MPI;
2581 IncrementPointer(N, LoMemVT, MPI, Ptr);
2582
2583 if (isTruncating)
2584 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
2585 HiMemVT, Alignment, MMOFlags, AAInfo);
2586 else
2587 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
2588
2589 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2590 }
2591
SplitVecOp_CONCAT_VECTORS(SDNode * N)2592 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2593 SDLoc DL(N);
2594
2595 // The input operands all must have the same type, and we know the result
2596 // type is valid. Convert this to a buildvector which extracts all the
2597 // input elements.
2598 // TODO: If the input elements are power-two vectors, we could convert this to
2599 // a new CONCAT_VECTORS node with elements that are half-wide.
2600 SmallVector<SDValue, 32> Elts;
2601 EVT EltVT = N->getValueType(0).getVectorElementType();
2602 for (const SDValue &Op : N->op_values()) {
2603 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2604 i != e; ++i) {
2605 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2606 DAG.getVectorIdxConstant(i, DL)));
2607 }
2608 }
2609
2610 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2611 }
2612
SplitVecOp_TruncateHelper(SDNode * N)2613 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2614 // The result type is legal, but the input type is illegal. If splitting
2615 // ends up with the result type of each half still being legal, just
2616 // do that. If, however, that would result in an illegal result type,
2617 // we can try to get more clever with power-two vectors. Specifically,
2618 // split the input type, but also widen the result element size, then
2619 // concatenate the halves and truncate again. For example, consider a target
2620 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2621 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2622 // %inlo = v4i32 extract_subvector %in, 0
2623 // %inhi = v4i32 extract_subvector %in, 4
2624 // %lo16 = v4i16 trunc v4i32 %inlo
2625 // %hi16 = v4i16 trunc v4i32 %inhi
2626 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2627 // %res = v8i8 trunc v8i16 %in16
2628 //
2629 // Without this transform, the original truncate would end up being
2630 // scalarized, which is pretty much always a last resort.
2631 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
2632 SDValue InVec = N->getOperand(OpNo);
2633 EVT InVT = InVec->getValueType(0);
2634 EVT OutVT = N->getValueType(0);
2635 ElementCount NumElements = OutVT.getVectorElementCount();
2636 bool IsFloat = OutVT.isFloatingPoint();
2637
2638 unsigned InElementSize = InVT.getScalarSizeInBits();
2639 unsigned OutElementSize = OutVT.getScalarSizeInBits();
2640
2641 // Determine the split output VT. If its legal we can just split dirctly.
2642 EVT LoOutVT, HiOutVT;
2643 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2644 assert(LoOutVT == HiOutVT && "Unequal split?");
2645
2646 // If the input elements are only 1/2 the width of the result elements,
2647 // just use the normal splitting. Our trick only work if there's room
2648 // to split more than once.
2649 if (isTypeLegal(LoOutVT) ||
2650 InElementSize <= OutElementSize * 2)
2651 return SplitVecOp_UnaryOp(N);
2652 SDLoc DL(N);
2653
2654 // Don't touch if this will be scalarized.
2655 EVT FinalVT = InVT;
2656 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2657 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2658
2659 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2660 return SplitVecOp_UnaryOp(N);
2661
2662 // Get the split input vector.
2663 SDValue InLoVec, InHiVec;
2664 GetSplitVector(InVec, InLoVec, InHiVec);
2665
2666 // Truncate them to 1/2 the element size.
2667 //
2668 // This assumes the number of elements is a power of two; any vector that
2669 // isn't should be widened, not split.
2670 EVT HalfElementVT = IsFloat ?
2671 EVT::getFloatingPointVT(InElementSize/2) :
2672 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2673 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2674 NumElements.divideCoefficientBy(2));
2675
2676 SDValue HalfLo;
2677 SDValue HalfHi;
2678 SDValue Chain;
2679 if (N->isStrictFPOpcode()) {
2680 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2681 {N->getOperand(0), InLoVec});
2682 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2683 {N->getOperand(0), InHiVec});
2684 // Legalize the chain result - switch anything that used the old chain to
2685 // use the new one.
2686 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
2687 HalfHi.getValue(1));
2688 } else {
2689 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2690 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2691 }
2692 // Concatenate them to get the full intermediate truncation result.
2693 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2694 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2695 HalfHi);
2696 // Now finish up by truncating all the way down to the original result
2697 // type. This should normally be something that ends up being legal directly,
2698 // but in theory if a target has very wide vectors and an annoyingly
2699 // restricted set of legal types, this split can chain to build things up.
2700
2701 if (N->isStrictFPOpcode()) {
2702 SDValue Res = DAG.getNode(
2703 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
2704 {Chain, InterVec,
2705 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
2706 // Relink the chain
2707 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2708 return Res;
2709 }
2710
2711 return IsFloat
2712 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2713 DAG.getTargetConstant(
2714 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2715 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2716 }
2717
SplitVecOp_VSETCC(SDNode * N)2718 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2719 assert(N->getValueType(0).isVector() &&
2720 N->getOperand(0).getValueType().isVector() &&
2721 "Operand types must be vectors");
2722 // The result has a legal vector type, but the input needs splitting.
2723 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2724 SDLoc DL(N);
2725 GetSplitVector(N->getOperand(0), Lo0, Hi0);
2726 GetSplitVector(N->getOperand(1), Lo1, Hi1);
2727 auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
2728
2729 LLVMContext &Context = *DAG.getContext();
2730 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
2731 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
2732
2733 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2734 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2735 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2736
2737 EVT OpVT = N->getOperand(0).getValueType();
2738 ISD::NodeType ExtendCode =
2739 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
2740 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
2741 }
2742
2743
SplitVecOp_FP_ROUND(SDNode * N)2744 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2745 // The result has a legal vector type, but the input needs splitting.
2746 EVT ResVT = N->getValueType(0);
2747 SDValue Lo, Hi;
2748 SDLoc DL(N);
2749 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2750 EVT InVT = Lo.getValueType();
2751
2752 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2753 InVT.getVectorElementCount());
2754
2755 if (N->isStrictFPOpcode()) {
2756 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2757 { N->getOperand(0), Lo, N->getOperand(2) });
2758 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2759 { N->getOperand(0), Hi, N->getOperand(2) });
2760 // Legalize the chain result - switch anything that used the old chain to
2761 // use the new one.
2762 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2763 Lo.getValue(1), Hi.getValue(1));
2764 ReplaceValueWith(SDValue(N, 1), NewChain);
2765 } else {
2766 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2767 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2768 }
2769
2770 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2771 }
2772
SplitVecOp_FCOPYSIGN(SDNode * N)2773 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2774 // The result (and the first input) has a legal vector type, but the second
2775 // input needs splitting.
2776 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2777 }
2778
2779
2780 //===----------------------------------------------------------------------===//
2781 // Result Vector Widening
2782 //===----------------------------------------------------------------------===//
2783
WidenVectorResult(SDNode * N,unsigned ResNo)2784 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2785 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2786 dbgs() << "\n");
2787
2788 // See if the target wants to custom widen this node.
2789 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2790 return;
2791
2792 SDValue Res = SDValue();
2793 switch (N->getOpcode()) {
2794 default:
2795 #ifndef NDEBUG
2796 dbgs() << "WidenVectorResult #" << ResNo << ": ";
2797 N->dump(&DAG);
2798 dbgs() << "\n";
2799 #endif
2800 llvm_unreachable("Do not know how to widen the result of this operator!");
2801
2802 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2803 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2804 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2805 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2806 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2807 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2808 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2809 case ISD::SPLAT_VECTOR:
2810 case ISD::SCALAR_TO_VECTOR:
2811 Res = WidenVecRes_ScalarOp(N);
2812 break;
2813 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2814 case ISD::VSELECT:
2815 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2816 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2817 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2818 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2819 case ISD::VECTOR_SHUFFLE:
2820 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2821 break;
2822 case ISD::MLOAD:
2823 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2824 break;
2825 case ISD::MGATHER:
2826 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2827 break;
2828
2829 case ISD::ADD:
2830 case ISD::AND:
2831 case ISD::MUL:
2832 case ISD::MULHS:
2833 case ISD::MULHU:
2834 case ISD::OR:
2835 case ISD::SUB:
2836 case ISD::XOR:
2837 case ISD::SHL:
2838 case ISD::SRA:
2839 case ISD::SRL:
2840 case ISD::FMINNUM:
2841 case ISD::FMAXNUM:
2842 case ISD::FMINIMUM:
2843 case ISD::FMAXIMUM:
2844 case ISD::SMIN:
2845 case ISD::SMAX:
2846 case ISD::UMIN:
2847 case ISD::UMAX:
2848 case ISD::UADDSAT:
2849 case ISD::SADDSAT:
2850 case ISD::USUBSAT:
2851 case ISD::SSUBSAT:
2852 case ISD::SSHLSAT:
2853 case ISD::USHLSAT:
2854 case ISD::ROTL:
2855 case ISD::ROTR:
2856 Res = WidenVecRes_Binary(N);
2857 break;
2858
2859 case ISD::FADD:
2860 case ISD::FMUL:
2861 case ISD::FPOW:
2862 case ISD::FSUB:
2863 case ISD::FDIV:
2864 case ISD::FREM:
2865 case ISD::SDIV:
2866 case ISD::UDIV:
2867 case ISD::SREM:
2868 case ISD::UREM:
2869 Res = WidenVecRes_BinaryCanTrap(N);
2870 break;
2871
2872 case ISD::SMULFIX:
2873 case ISD::SMULFIXSAT:
2874 case ISD::UMULFIX:
2875 case ISD::UMULFIXSAT:
2876 // These are binary operations, but with an extra operand that shouldn't
2877 // be widened (the scale).
2878 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
2879 break;
2880
2881 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
2882 case ISD::STRICT_##DAGN:
2883 #include "llvm/IR/ConstrainedOps.def"
2884 Res = WidenVecRes_StrictFP(N);
2885 break;
2886
2887 case ISD::UADDO:
2888 case ISD::SADDO:
2889 case ISD::USUBO:
2890 case ISD::SSUBO:
2891 case ISD::UMULO:
2892 case ISD::SMULO:
2893 Res = WidenVecRes_OverflowOp(N, ResNo);
2894 break;
2895
2896 case ISD::FCOPYSIGN:
2897 Res = WidenVecRes_FCOPYSIGN(N);
2898 break;
2899
2900 case ISD::FPOWI:
2901 Res = WidenVecRes_POWI(N);
2902 break;
2903
2904 case ISD::ANY_EXTEND_VECTOR_INREG:
2905 case ISD::SIGN_EXTEND_VECTOR_INREG:
2906 case ISD::ZERO_EXTEND_VECTOR_INREG:
2907 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2908 break;
2909
2910 case ISD::ANY_EXTEND:
2911 case ISD::FP_EXTEND:
2912 case ISD::FP_ROUND:
2913 case ISD::FP_TO_SINT:
2914 case ISD::FP_TO_UINT:
2915 case ISD::SIGN_EXTEND:
2916 case ISD::SINT_TO_FP:
2917 case ISD::TRUNCATE:
2918 case ISD::UINT_TO_FP:
2919 case ISD::ZERO_EXTEND:
2920 Res = WidenVecRes_Convert(N);
2921 break;
2922
2923 case ISD::FABS:
2924 case ISD::FCEIL:
2925 case ISD::FCOS:
2926 case ISD::FEXP:
2927 case ISD::FEXP2:
2928 case ISD::FFLOOR:
2929 case ISD::FLOG:
2930 case ISD::FLOG10:
2931 case ISD::FLOG2:
2932 case ISD::FNEARBYINT:
2933 case ISD::FRINT:
2934 case ISD::FROUND:
2935 case ISD::FROUNDEVEN:
2936 case ISD::FSIN:
2937 case ISD::FSQRT:
2938 case ISD::FTRUNC: {
2939 // We're going to widen this vector op to a legal type by padding with undef
2940 // elements. If the wide vector op is eventually going to be expanded to
2941 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
2942 // libcalls on the undef elements.
2943 EVT VT = N->getValueType(0);
2944 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2945 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
2946 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
2947 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2948 break;
2949 }
2950 }
2951 // If the target has custom/legal support for the scalar FP intrinsic ops
2952 // (they are probably not destined to become libcalls), then widen those like
2953 // any other unary ops.
2954 LLVM_FALLTHROUGH;
2955
2956 case ISD::ABS:
2957 case ISD::BITREVERSE:
2958 case ISD::BSWAP:
2959 case ISD::CTLZ:
2960 case ISD::CTLZ_ZERO_UNDEF:
2961 case ISD::CTPOP:
2962 case ISD::CTTZ:
2963 case ISD::CTTZ_ZERO_UNDEF:
2964 case ISD::FNEG:
2965 case ISD::FREEZE:
2966 case ISD::FCANONICALIZE:
2967 Res = WidenVecRes_Unary(N);
2968 break;
2969 case ISD::FMA:
2970 case ISD::FSHL:
2971 case ISD::FSHR:
2972 Res = WidenVecRes_Ternary(N);
2973 break;
2974 }
2975
2976 // If Res is null, the sub-method took care of registering the result.
2977 if (Res.getNode())
2978 SetWidenedVector(SDValue(N, ResNo), Res);
2979 }
2980
WidenVecRes_Ternary(SDNode * N)2981 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2982 // Ternary op widening.
2983 SDLoc dl(N);
2984 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2985 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2986 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2987 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2988 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2989 }
2990
WidenVecRes_Binary(SDNode * N)2991 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2992 // Binary op widening.
2993 SDLoc dl(N);
2994 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2995 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2996 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2997 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2998 }
2999
WidenVecRes_BinaryWithExtraScalarOp(SDNode * N)3000 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
3001 // Binary op widening, but with an extra operand that shouldn't be widened.
3002 SDLoc dl(N);
3003 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3004 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3005 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3006 SDValue InOp3 = N->getOperand(2);
3007 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3008 N->getFlags());
3009 }
3010
3011 // Given a vector of operations that have been broken up to widen, see
3012 // if we can collect them together into the next widest legal VT. This
3013 // implementation is trap-safe.
CollectOpsToWiden(SelectionDAG & DAG,const TargetLowering & TLI,SmallVectorImpl<SDValue> & ConcatOps,unsigned ConcatEnd,EVT VT,EVT MaxVT,EVT WidenVT)3014 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
3015 SmallVectorImpl<SDValue> &ConcatOps,
3016 unsigned ConcatEnd, EVT VT, EVT MaxVT,
3017 EVT WidenVT) {
3018 // Check to see if we have a single operation with the widen type.
3019 if (ConcatEnd == 1) {
3020 VT = ConcatOps[0].getValueType();
3021 if (VT == WidenVT)
3022 return ConcatOps[0];
3023 }
3024
3025 SDLoc dl(ConcatOps[0]);
3026 EVT WidenEltVT = WidenVT.getVectorElementType();
3027
3028 // while (Some element of ConcatOps is not of type MaxVT) {
3029 // From the end of ConcatOps, collect elements of the same type and put
3030 // them into an op of the next larger supported type
3031 // }
3032 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
3033 int Idx = ConcatEnd - 1;
3034 VT = ConcatOps[Idx--].getValueType();
3035 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
3036 Idx--;
3037
3038 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
3039 EVT NextVT;
3040 do {
3041 NextSize *= 2;
3042 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
3043 } while (!TLI.isTypeLegal(NextVT));
3044
3045 if (!VT.isVector()) {
3046 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
3047 SDValue VecOp = DAG.getUNDEF(NextVT);
3048 unsigned NumToInsert = ConcatEnd - Idx - 1;
3049 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
3050 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
3051 ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
3052 }
3053 ConcatOps[Idx+1] = VecOp;
3054 ConcatEnd = Idx + 2;
3055 } else {
3056 // Vector type, create a CONCAT_VECTORS of type NextVT
3057 SDValue undefVec = DAG.getUNDEF(VT);
3058 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
3059 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3060 unsigned RealVals = ConcatEnd - Idx - 1;
3061 unsigned SubConcatEnd = 0;
3062 unsigned SubConcatIdx = Idx + 1;
3063 while (SubConcatEnd < RealVals)
3064 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
3065 while (SubConcatEnd < OpsToConcat)
3066 SubConcatOps[SubConcatEnd++] = undefVec;
3067 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
3068 NextVT, SubConcatOps);
3069 ConcatEnd = SubConcatIdx + 1;
3070 }
3071 }
3072
3073 // Check to see if we have a single operation with the widen type.
3074 if (ConcatEnd == 1) {
3075 VT = ConcatOps[0].getValueType();
3076 if (VT == WidenVT)
3077 return ConcatOps[0];
3078 }
3079
3080 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
3081 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
3082 if (NumOps != ConcatEnd ) {
3083 SDValue UndefVal = DAG.getUNDEF(MaxVT);
3084 for (unsigned j = ConcatEnd; j < NumOps; ++j)
3085 ConcatOps[j] = UndefVal;
3086 }
3087 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3088 makeArrayRef(ConcatOps.data(), NumOps));
3089 }
3090
WidenVecRes_BinaryCanTrap(SDNode * N)3091 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3092 // Binary op widening for operations that can trap.
3093 unsigned Opcode = N->getOpcode();
3094 SDLoc dl(N);
3095 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3096 EVT WidenEltVT = WidenVT.getVectorElementType();
3097 EVT VT = WidenVT;
3098 unsigned NumElts = VT.getVectorNumElements();
3099 const SDNodeFlags Flags = N->getFlags();
3100 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3101 NumElts = NumElts / 2;
3102 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3103 }
3104
3105 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
3106 // Operation doesn't trap so just widen as normal.
3107 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3108 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3109 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
3110 }
3111
3112 // No legal vector version so unroll the vector operation and then widen.
3113 if (NumElts == 1)
3114 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3115
3116 // Since the operation can trap, apply operation on the original vector.
3117 EVT MaxVT = VT;
3118 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3119 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3120 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3121
3122 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3123 unsigned ConcatEnd = 0; // Current ConcatOps index.
3124 int Idx = 0; // Current Idx into input vectors.
3125
3126 // NumElts := greatest legal vector size (at most WidenVT)
3127 // while (orig. vector has unhandled elements) {
3128 // take munches of size NumElts from the beginning and add to ConcatOps
3129 // NumElts := next smaller supported vector size or 1
3130 // }
3131 while (CurNumElts != 0) {
3132 while (CurNumElts >= NumElts) {
3133 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3134 DAG.getVectorIdxConstant(Idx, dl));
3135 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3136 DAG.getVectorIdxConstant(Idx, dl));
3137 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3138 Idx += NumElts;
3139 CurNumElts -= NumElts;
3140 }
3141 do {
3142 NumElts = NumElts / 2;
3143 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3144 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3145
3146 if (NumElts == 1) {
3147 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3148 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3149 InOp1, DAG.getVectorIdxConstant(Idx, dl));
3150 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3151 InOp2, DAG.getVectorIdxConstant(Idx, dl));
3152 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3153 EOp1, EOp2, Flags);
3154 }
3155 CurNumElts = 0;
3156 }
3157 }
3158
3159 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3160 }
3161
WidenVecRes_StrictFP(SDNode * N)3162 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3163 switch (N->getOpcode()) {
3164 case ISD::STRICT_FSETCC:
3165 case ISD::STRICT_FSETCCS:
3166 return WidenVecRes_STRICT_FSETCC(N);
3167 case ISD::STRICT_FP_EXTEND:
3168 case ISD::STRICT_FP_ROUND:
3169 case ISD::STRICT_FP_TO_SINT:
3170 case ISD::STRICT_FP_TO_UINT:
3171 case ISD::STRICT_SINT_TO_FP:
3172 case ISD::STRICT_UINT_TO_FP:
3173 return WidenVecRes_Convert_StrictFP(N);
3174 default:
3175 break;
3176 }
3177
3178 // StrictFP op widening for operations that can trap.
3179 unsigned NumOpers = N->getNumOperands();
3180 unsigned Opcode = N->getOpcode();
3181 SDLoc dl(N);
3182 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3183 EVT WidenEltVT = WidenVT.getVectorElementType();
3184 EVT VT = WidenVT;
3185 unsigned NumElts = VT.getVectorNumElements();
3186 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3187 NumElts = NumElts / 2;
3188 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3189 }
3190
3191 // No legal vector version so unroll the vector operation and then widen.
3192 if (NumElts == 1)
3193 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3194
3195 // Since the operation can trap, apply operation on the original vector.
3196 EVT MaxVT = VT;
3197 SmallVector<SDValue, 4> InOps;
3198 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3199
3200 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3201 SmallVector<SDValue, 16> Chains;
3202 unsigned ConcatEnd = 0; // Current ConcatOps index.
3203 int Idx = 0; // Current Idx into input vectors.
3204
3205 // The Chain is the first operand.
3206 InOps.push_back(N->getOperand(0));
3207
3208 // Now process the remaining operands.
3209 for (unsigned i = 1; i < NumOpers; ++i) {
3210 SDValue Oper = N->getOperand(i);
3211
3212 if (Oper.getValueType().isVector()) {
3213 assert(Oper.getValueType() == N->getValueType(0) &&
3214 "Invalid operand type to widen!");
3215 Oper = GetWidenedVector(Oper);
3216 }
3217
3218 InOps.push_back(Oper);
3219 }
3220
3221 // NumElts := greatest legal vector size (at most WidenVT)
3222 // while (orig. vector has unhandled elements) {
3223 // take munches of size NumElts from the beginning and add to ConcatOps
3224 // NumElts := next smaller supported vector size or 1
3225 // }
3226 while (CurNumElts != 0) {
3227 while (CurNumElts >= NumElts) {
3228 SmallVector<SDValue, 4> EOps;
3229
3230 for (unsigned i = 0; i < NumOpers; ++i) {
3231 SDValue Op = InOps[i];
3232
3233 if (Op.getValueType().isVector())
3234 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3235 DAG.getVectorIdxConstant(Idx, dl));
3236
3237 EOps.push_back(Op);
3238 }
3239
3240 EVT OperVT[] = {VT, MVT::Other};
3241 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3242 ConcatOps[ConcatEnd++] = Oper;
3243 Chains.push_back(Oper.getValue(1));
3244 Idx += NumElts;
3245 CurNumElts -= NumElts;
3246 }
3247 do {
3248 NumElts = NumElts / 2;
3249 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3250 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3251
3252 if (NumElts == 1) {
3253 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3254 SmallVector<SDValue, 4> EOps;
3255
3256 for (unsigned i = 0; i < NumOpers; ++i) {
3257 SDValue Op = InOps[i];
3258
3259 if (Op.getValueType().isVector())
3260 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3261 DAG.getVectorIdxConstant(Idx, dl));
3262
3263 EOps.push_back(Op);
3264 }
3265
3266 EVT WidenVT[] = {WidenEltVT, MVT::Other};
3267 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3268 ConcatOps[ConcatEnd++] = Oper;
3269 Chains.push_back(Oper.getValue(1));
3270 }
3271 CurNumElts = 0;
3272 }
3273 }
3274
3275 // Build a factor node to remember all the Ops that have been created.
3276 SDValue NewChain;
3277 if (Chains.size() == 1)
3278 NewChain = Chains[0];
3279 else
3280 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3281 ReplaceValueWith(SDValue(N, 1), NewChain);
3282
3283 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3284 }
3285
WidenVecRes_OverflowOp(SDNode * N,unsigned ResNo)3286 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3287 SDLoc DL(N);
3288 EVT ResVT = N->getValueType(0);
3289 EVT OvVT = N->getValueType(1);
3290 EVT WideResVT, WideOvVT;
3291 SDValue WideLHS, WideRHS;
3292
3293 // TODO: This might result in a widen/split loop.
3294 if (ResNo == 0) {
3295 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3296 WideOvVT = EVT::getVectorVT(
3297 *DAG.getContext(), OvVT.getVectorElementType(),
3298 WideResVT.getVectorNumElements());
3299
3300 WideLHS = GetWidenedVector(N->getOperand(0));
3301 WideRHS = GetWidenedVector(N->getOperand(1));
3302 } else {
3303 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3304 WideResVT = EVT::getVectorVT(
3305 *DAG.getContext(), ResVT.getVectorElementType(),
3306 WideOvVT.getVectorNumElements());
3307
3308 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3309 WideLHS = DAG.getNode(
3310 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3311 N->getOperand(0), Zero);
3312 WideRHS = DAG.getNode(
3313 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3314 N->getOperand(1), Zero);
3315 }
3316
3317 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3318 SDNode *WideNode = DAG.getNode(
3319 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3320
3321 // Replace the other vector result not being explicitly widened here.
3322 unsigned OtherNo = 1 - ResNo;
3323 EVT OtherVT = N->getValueType(OtherNo);
3324 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3325 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3326 } else {
3327 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3328 SDValue OtherVal = DAG.getNode(
3329 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3330 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3331 }
3332
3333 return SDValue(WideNode, ResNo);
3334 }
3335
WidenVecRes_Convert(SDNode * N)3336 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3337 LLVMContext &Ctx = *DAG.getContext();
3338 SDValue InOp = N->getOperand(0);
3339 SDLoc DL(N);
3340
3341 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
3342 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3343
3344 EVT InVT = InOp.getValueType();
3345
3346 unsigned Opcode = N->getOpcode();
3347 const SDNodeFlags Flags = N->getFlags();
3348
3349 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
3350 // not equal that of WidenVT.
3351 if (N->getOpcode() == ISD::ZERO_EXTEND &&
3352 getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
3353 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
3354 WidenVT.getScalarSizeInBits()) {
3355 InOp = ZExtPromotedInteger(InOp);
3356 InVT = InOp.getValueType();
3357 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
3358 Opcode = ISD::TRUNCATE;
3359 }
3360
3361 EVT InEltVT = InVT.getVectorElementType();
3362 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenNumElts);
3363 unsigned InVTNumElts = InVT.getVectorNumElements();
3364
3365 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3366 InOp = GetWidenedVector(N->getOperand(0));
3367 InVT = InOp.getValueType();
3368 InVTNumElts = InVT.getVectorNumElements();
3369 if (InVTNumElts == WidenNumElts) {
3370 if (N->getNumOperands() == 1)
3371 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3372 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3373 }
3374 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3375 // If both input and result vector types are of same width, extend
3376 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3377 // accepts fewer elements in the result than in the input.
3378 if (Opcode == ISD::ANY_EXTEND)
3379 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3380 if (Opcode == ISD::SIGN_EXTEND)
3381 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3382 if (Opcode == ISD::ZERO_EXTEND)
3383 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3384 }
3385 }
3386
3387 if (TLI.isTypeLegal(InWidenVT)) {
3388 // Because the result and the input are different vector types, widening
3389 // the result could create a legal type but widening the input might make
3390 // it an illegal type that might lead to repeatedly splitting the input
3391 // and then widening it. To avoid this, we widen the input only if
3392 // it results in a legal type.
3393 if (WidenNumElts % InVTNumElts == 0) {
3394 // Widen the input and call convert on the widened input vector.
3395 unsigned NumConcat = WidenNumElts/InVTNumElts;
3396 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3397 Ops[0] = InOp;
3398 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3399 if (N->getNumOperands() == 1)
3400 return DAG.getNode(Opcode, DL, WidenVT, InVec);
3401 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3402 }
3403
3404 if (InVTNumElts % WidenNumElts == 0) {
3405 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3406 DAG.getVectorIdxConstant(0, DL));
3407 // Extract the input and convert the shorten input vector.
3408 if (N->getNumOperands() == 1)
3409 return DAG.getNode(Opcode, DL, WidenVT, InVal);
3410 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3411 }
3412 }
3413
3414 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3415 EVT EltVT = WidenVT.getVectorElementType();
3416 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3417 // Use the original element count so we don't do more scalar opts than
3418 // necessary.
3419 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3420 for (unsigned i=0; i < MinElts; ++i) {
3421 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3422 DAG.getVectorIdxConstant(i, DL));
3423 if (N->getNumOperands() == 1)
3424 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3425 else
3426 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3427 }
3428
3429 return DAG.getBuildVector(WidenVT, DL, Ops);
3430 }
3431
WidenVecRes_Convert_StrictFP(SDNode * N)3432 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3433 SDValue InOp = N->getOperand(1);
3434 SDLoc DL(N);
3435 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3436
3437 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3438 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3439
3440 EVT InVT = InOp.getValueType();
3441 EVT InEltVT = InVT.getVectorElementType();
3442
3443 unsigned Opcode = N->getOpcode();
3444
3445 // FIXME: Optimizations need to be implemented here.
3446
3447 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3448 EVT EltVT = WidenVT.getVectorElementType();
3449 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
3450 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3451 SmallVector<SDValue, 32> OpChains;
3452 // Use the original element count so we don't do more scalar opts than
3453 // necessary.
3454 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3455 for (unsigned i=0; i < MinElts; ++i) {
3456 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3457 DAG.getVectorIdxConstant(i, DL));
3458 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3459 OpChains.push_back(Ops[i].getValue(1));
3460 }
3461 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3462 ReplaceValueWith(SDValue(N, 1), NewChain);
3463
3464 return DAG.getBuildVector(WidenVT, DL, Ops);
3465 }
3466
WidenVecRes_EXTEND_VECTOR_INREG(SDNode * N)3467 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3468 unsigned Opcode = N->getOpcode();
3469 SDValue InOp = N->getOperand(0);
3470 SDLoc DL(N);
3471
3472 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3473 EVT WidenSVT = WidenVT.getVectorElementType();
3474 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3475
3476 EVT InVT = InOp.getValueType();
3477 EVT InSVT = InVT.getVectorElementType();
3478 unsigned InVTNumElts = InVT.getVectorNumElements();
3479
3480 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3481 InOp = GetWidenedVector(InOp);
3482 InVT = InOp.getValueType();
3483 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3484 switch (Opcode) {
3485 case ISD::ANY_EXTEND_VECTOR_INREG:
3486 case ISD::SIGN_EXTEND_VECTOR_INREG:
3487 case ISD::ZERO_EXTEND_VECTOR_INREG:
3488 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3489 }
3490 }
3491 }
3492
3493 // Unroll, extend the scalars and rebuild the vector.
3494 SmallVector<SDValue, 16> Ops;
3495 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3496 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3497 DAG.getVectorIdxConstant(i, DL));
3498 switch (Opcode) {
3499 case ISD::ANY_EXTEND_VECTOR_INREG:
3500 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3501 break;
3502 case ISD::SIGN_EXTEND_VECTOR_INREG:
3503 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3504 break;
3505 case ISD::ZERO_EXTEND_VECTOR_INREG:
3506 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3507 break;
3508 default:
3509 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3510 }
3511 Ops.push_back(Val);
3512 }
3513
3514 while (Ops.size() != WidenNumElts)
3515 Ops.push_back(DAG.getUNDEF(WidenSVT));
3516
3517 return DAG.getBuildVector(WidenVT, DL, Ops);
3518 }
3519
WidenVecRes_FCOPYSIGN(SDNode * N)3520 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3521 // If this is an FCOPYSIGN with same input types, we can treat it as a
3522 // normal (can trap) binary op.
3523 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3524 return WidenVecRes_BinaryCanTrap(N);
3525
3526 // If the types are different, fall back to unrolling.
3527 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3528 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3529 }
3530
WidenVecRes_POWI(SDNode * N)3531 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3532 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3533 SDValue InOp = GetWidenedVector(N->getOperand(0));
3534 SDValue ShOp = N->getOperand(1);
3535 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3536 }
3537
WidenVecRes_Unary(SDNode * N)3538 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3539 // Unary op widening.
3540 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3541 SDValue InOp = GetWidenedVector(N->getOperand(0));
3542 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3543 }
3544
WidenVecRes_InregOp(SDNode * N)3545 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3546 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3547 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3548 cast<VTSDNode>(N->getOperand(1))->getVT()
3549 .getVectorElementType(),
3550 WidenVT.getVectorNumElements());
3551 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3552 return DAG.getNode(N->getOpcode(), SDLoc(N),
3553 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3554 }
3555
WidenVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)3556 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3557 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3558 return GetWidenedVector(WidenVec);
3559 }
3560
WidenVecRes_BITCAST(SDNode * N)3561 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3562 SDValue InOp = N->getOperand(0);
3563 EVT InVT = InOp.getValueType();
3564 EVT VT = N->getValueType(0);
3565 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3566 SDLoc dl(N);
3567
3568 switch (getTypeAction(InVT)) {
3569 case TargetLowering::TypeLegal:
3570 break;
3571 case TargetLowering::TypeScalarizeScalableVector:
3572 report_fatal_error("Scalarization of scalable vectors is not supported.");
3573 case TargetLowering::TypePromoteInteger: {
3574 // If the incoming type is a vector that is being promoted, then
3575 // we know that the elements are arranged differently and that we
3576 // must perform the conversion using a stack slot.
3577 if (InVT.isVector())
3578 break;
3579
3580 // If the InOp is promoted to the same size, convert it. Otherwise,
3581 // fall out of the switch and widen the promoted input.
3582 SDValue NInOp = GetPromotedInteger(InOp);
3583 EVT NInVT = NInOp.getValueType();
3584 if (WidenVT.bitsEq(NInVT)) {
3585 // For big endian targets we need to shift the input integer or the
3586 // interesting bits will end up at the wrong place.
3587 if (DAG.getDataLayout().isBigEndian()) {
3588 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
3589 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
3590 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
3591 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
3592 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
3593 }
3594 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
3595 }
3596 InOp = NInOp;
3597 InVT = NInVT;
3598 break;
3599 }
3600 case TargetLowering::TypeSoftenFloat:
3601 case TargetLowering::TypePromoteFloat:
3602 case TargetLowering::TypeSoftPromoteHalf:
3603 case TargetLowering::TypeExpandInteger:
3604 case TargetLowering::TypeExpandFloat:
3605 case TargetLowering::TypeScalarizeVector:
3606 case TargetLowering::TypeSplitVector:
3607 break;
3608 case TargetLowering::TypeWidenVector:
3609 // If the InOp is widened to the same size, convert it. Otherwise, fall
3610 // out of the switch and widen the widened input.
3611 InOp = GetWidenedVector(InOp);
3612 InVT = InOp.getValueType();
3613 if (WidenVT.bitsEq(InVT))
3614 // The input widens to the same size. Convert to the widen value.
3615 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3616 break;
3617 }
3618
3619 unsigned WidenSize = WidenVT.getSizeInBits();
3620 unsigned InSize = InVT.getSizeInBits();
3621 // x86mmx is not an acceptable vector element type, so don't try.
3622 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3623 // Determine new input vector type. The new input vector type will use
3624 // the same element type (if its a vector) or use the input type as a
3625 // vector. It is the same size as the type to widen to.
3626 EVT NewInVT;
3627 unsigned NewNumElts = WidenSize / InSize;
3628 if (InVT.isVector()) {
3629 EVT InEltVT = InVT.getVectorElementType();
3630 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3631 WidenSize / InEltVT.getSizeInBits());
3632 } else {
3633 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3634 }
3635
3636 if (TLI.isTypeLegal(NewInVT)) {
3637 SDValue NewVec;
3638 if (InVT.isVector()) {
3639 // Because the result and the input are different vector types, widening
3640 // the result could create a legal type but widening the input might make
3641 // it an illegal type that might lead to repeatedly splitting the input
3642 // and then widening it. To avoid this, we widen the input only if
3643 // it results in a legal type.
3644 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3645 Ops[0] = InOp;
3646
3647 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3648 } else {
3649 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3650 }
3651 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3652 }
3653 }
3654
3655 return CreateStackStoreLoad(InOp, WidenVT);
3656 }
3657
WidenVecRes_BUILD_VECTOR(SDNode * N)3658 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3659 SDLoc dl(N);
3660 // Build a vector with undefined for the new nodes.
3661 EVT VT = N->getValueType(0);
3662
3663 // Integer BUILD_VECTOR operands may be larger than the node's vector element
3664 // type. The UNDEFs need to have the same type as the existing operands.
3665 EVT EltVT = N->getOperand(0).getValueType();
3666 unsigned NumElts = VT.getVectorNumElements();
3667
3668 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3669 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3670
3671 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3672 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3673 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3674
3675 return DAG.getBuildVector(WidenVT, dl, NewOps);
3676 }
3677
WidenVecRes_CONCAT_VECTORS(SDNode * N)3678 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3679 EVT InVT = N->getOperand(0).getValueType();
3680 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3681 SDLoc dl(N);
3682 unsigned NumOperands = N->getNumOperands();
3683
3684 bool InputWidened = false; // Indicates we need to widen the input.
3685 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3686 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
3687 unsigned NumInElts = InVT.getVectorMinNumElements();
3688 if (WidenNumElts % NumInElts == 0) {
3689 // Add undef vectors to widen to correct length.
3690 unsigned NumConcat = WidenNumElts / NumInElts;
3691 SDValue UndefVal = DAG.getUNDEF(InVT);
3692 SmallVector<SDValue, 16> Ops(NumConcat);
3693 for (unsigned i=0; i < NumOperands; ++i)
3694 Ops[i] = N->getOperand(i);
3695 for (unsigned i = NumOperands; i != NumConcat; ++i)
3696 Ops[i] = UndefVal;
3697 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3698 }
3699 } else {
3700 InputWidened = true;
3701 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3702 // The inputs and the result are widen to the same value.
3703 unsigned i;
3704 for (i=1; i < NumOperands; ++i)
3705 if (!N->getOperand(i).isUndef())
3706 break;
3707
3708 if (i == NumOperands)
3709 // Everything but the first operand is an UNDEF so just return the
3710 // widened first operand.
3711 return GetWidenedVector(N->getOperand(0));
3712
3713 if (NumOperands == 2) {
3714 assert(!WidenVT.isScalableVector() &&
3715 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
3716 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3717 unsigned NumInElts = InVT.getVectorNumElements();
3718
3719 // Replace concat of two operands with a shuffle.
3720 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3721 for (unsigned i = 0; i < NumInElts; ++i) {
3722 MaskOps[i] = i;
3723 MaskOps[i + NumInElts] = i + WidenNumElts;
3724 }
3725 return DAG.getVectorShuffle(WidenVT, dl,
3726 GetWidenedVector(N->getOperand(0)),
3727 GetWidenedVector(N->getOperand(1)),
3728 MaskOps);
3729 }
3730 }
3731 }
3732
3733 assert(!WidenVT.isScalableVector() &&
3734 "Cannot use build vectors to widen CONCAT_VECTOR result");
3735 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3736 unsigned NumInElts = InVT.getVectorNumElements();
3737
3738 // Fall back to use extracts and build vector.
3739 EVT EltVT = WidenVT.getVectorElementType();
3740 SmallVector<SDValue, 16> Ops(WidenNumElts);
3741 unsigned Idx = 0;
3742 for (unsigned i=0; i < NumOperands; ++i) {
3743 SDValue InOp = N->getOperand(i);
3744 if (InputWidened)
3745 InOp = GetWidenedVector(InOp);
3746 for (unsigned j = 0; j < NumInElts; ++j)
3747 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3748 DAG.getVectorIdxConstant(j, dl));
3749 }
3750 SDValue UndefVal = DAG.getUNDEF(EltVT);
3751 for (; Idx < WidenNumElts; ++Idx)
3752 Ops[Idx] = UndefVal;
3753 return DAG.getBuildVector(WidenVT, dl, Ops);
3754 }
3755
WidenVecRes_EXTRACT_SUBVECTOR(SDNode * N)3756 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3757 EVT VT = N->getValueType(0);
3758 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3759 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3760 SDValue InOp = N->getOperand(0);
3761 SDValue Idx = N->getOperand(1);
3762 SDLoc dl(N);
3763
3764 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3765 InOp = GetWidenedVector(InOp);
3766
3767 EVT InVT = InOp.getValueType();
3768
3769 // Check if we can just return the input vector after widening.
3770 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3771 if (IdxVal == 0 && InVT == WidenVT)
3772 return InOp;
3773
3774 // Check if we can extract from the vector.
3775 unsigned InNumElts = InVT.getVectorNumElements();
3776 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3777 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3778
3779 // We could try widening the input to the right length but for now, extract
3780 // the original elements, fill the rest with undefs and build a vector.
3781 SmallVector<SDValue, 16> Ops(WidenNumElts);
3782 EVT EltVT = VT.getVectorElementType();
3783 unsigned NumElts = VT.getVectorNumElements();
3784 unsigned i;
3785 for (i = 0; i < NumElts; ++i)
3786 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3787 DAG.getVectorIdxConstant(IdxVal + i, dl));
3788
3789 SDValue UndefVal = DAG.getUNDEF(EltVT);
3790 for (; i < WidenNumElts; ++i)
3791 Ops[i] = UndefVal;
3792 return DAG.getBuildVector(WidenVT, dl, Ops);
3793 }
3794
WidenVecRes_INSERT_VECTOR_ELT(SDNode * N)3795 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3796 SDValue InOp = GetWidenedVector(N->getOperand(0));
3797 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3798 InOp.getValueType(), InOp,
3799 N->getOperand(1), N->getOperand(2));
3800 }
3801
WidenVecRes_LOAD(SDNode * N)3802 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3803 LoadSDNode *LD = cast<LoadSDNode>(N);
3804 ISD::LoadExtType ExtType = LD->getExtensionType();
3805
3806 // A vector must always be stored in memory as-is, i.e. without any padding
3807 // between the elements, since various code depend on it, e.g. in the
3808 // handling of a bitcast of a vector type to int, which may be done with a
3809 // vector store followed by an integer load. A vector that does not have
3810 // elements that are byte-sized must therefore be stored as an integer
3811 // built out of the extracted vector elements.
3812 if (!LD->getMemoryVT().isByteSized()) {
3813 SDValue Value, NewChain;
3814 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
3815 ReplaceValueWith(SDValue(LD, 0), Value);
3816 ReplaceValueWith(SDValue(LD, 1), NewChain);
3817 return SDValue();
3818 }
3819
3820 SDValue Result;
3821 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3822 if (ExtType != ISD::NON_EXTLOAD)
3823 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
3824 else
3825 Result = GenWidenVectorLoads(LdChain, LD);
3826
3827 // If we generate a single load, we can use that for the chain. Otherwise,
3828 // build a factor node to remember the multiple loads are independent and
3829 // chain to that.
3830 SDValue NewChain;
3831 if (LdChain.size() == 1)
3832 NewChain = LdChain[0];
3833 else
3834 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3835
3836 // Modified the chain - switch anything that used the old chain to use
3837 // the new one.
3838 ReplaceValueWith(SDValue(N, 1), NewChain);
3839
3840 return Result;
3841 }
3842
WidenVecRes_MLOAD(MaskedLoadSDNode * N)3843 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3844
3845 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3846 SDValue Mask = N->getMask();
3847 EVT MaskVT = Mask.getValueType();
3848 SDValue PassThru = GetWidenedVector(N->getPassThru());
3849 ISD::LoadExtType ExtType = N->getExtensionType();
3850 SDLoc dl(N);
3851
3852 // The mask should be widened as well
3853 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3854 MaskVT.getVectorElementType(),
3855 WidenVT.getVectorNumElements());
3856 Mask = ModifyToType(Mask, WideMaskVT, true);
3857
3858 SDValue Res = DAG.getMaskedLoad(
3859 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
3860 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
3861 ExtType, N->isExpandingLoad());
3862 // Legalize the chain result - switch anything that used the old chain to
3863 // use the new one.
3864 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3865 return Res;
3866 }
3867
WidenVecRes_MGATHER(MaskedGatherSDNode * N)3868 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3869
3870 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3871 SDValue Mask = N->getMask();
3872 EVT MaskVT = Mask.getValueType();
3873 SDValue PassThru = GetWidenedVector(N->getPassThru());
3874 SDValue Scale = N->getScale();
3875 unsigned NumElts = WideVT.getVectorNumElements();
3876 SDLoc dl(N);
3877
3878 // The mask should be widened as well
3879 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3880 MaskVT.getVectorElementType(),
3881 WideVT.getVectorNumElements());
3882 Mask = ModifyToType(Mask, WideMaskVT, true);
3883
3884 // Widen the Index operand
3885 SDValue Index = N->getIndex();
3886 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3887 Index.getValueType().getScalarType(),
3888 NumElts);
3889 Index = ModifyToType(Index, WideIndexVT);
3890 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3891 Scale };
3892 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3893 N->getMemoryVT(), dl, Ops,
3894 N->getMemOperand(), N->getIndexType());
3895
3896 // Legalize the chain result - switch anything that used the old chain to
3897 // use the new one.
3898 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3899 return Res;
3900 }
3901
WidenVecRes_ScalarOp(SDNode * N)3902 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
3903 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3904 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
3905 }
3906
3907 // Return true is this is a SETCC node or a strict version of it.
isSETCCOp(unsigned Opcode)3908 static inline bool isSETCCOp(unsigned Opcode) {
3909 switch (Opcode) {
3910 case ISD::SETCC:
3911 case ISD::STRICT_FSETCC:
3912 case ISD::STRICT_FSETCCS:
3913 return true;
3914 }
3915 return false;
3916 }
3917
3918 // Return true if this is a node that could have two SETCCs as operands.
isLogicalMaskOp(unsigned Opcode)3919 static inline bool isLogicalMaskOp(unsigned Opcode) {
3920 switch (Opcode) {
3921 case ISD::AND:
3922 case ISD::OR:
3923 case ISD::XOR:
3924 return true;
3925 }
3926 return false;
3927 }
3928
3929 // If N is a SETCC or a strict variant of it, return the type
3930 // of the compare operands.
getSETCCOperandType(SDValue N)3931 static inline EVT getSETCCOperandType(SDValue N) {
3932 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3933 return N->getOperand(OpNo).getValueType();
3934 }
3935
3936 // This is used just for the assert in convertMask(). Check that this either
3937 // a SETCC or a previously handled SETCC by convertMask().
3938 #ifndef NDEBUG
isSETCCorConvertedSETCC(SDValue N)3939 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3940 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3941 N = N.getOperand(0);
3942 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3943 for (unsigned i = 1; i < N->getNumOperands(); ++i)
3944 if (!N->getOperand(i)->isUndef())
3945 return false;
3946 N = N.getOperand(0);
3947 }
3948
3949 if (N.getOpcode() == ISD::TRUNCATE)
3950 N = N.getOperand(0);
3951 else if (N.getOpcode() == ISD::SIGN_EXTEND)
3952 N = N.getOperand(0);
3953
3954 if (isLogicalMaskOp(N.getOpcode()))
3955 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3956 isSETCCorConvertedSETCC(N.getOperand(1));
3957
3958 return (isSETCCOp(N.getOpcode()) ||
3959 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
3960 }
3961 #endif
3962
3963 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3964 // to ToMaskVT if needed with vector extension or truncation.
convertMask(SDValue InMask,EVT MaskVT,EVT ToMaskVT)3965 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3966 EVT ToMaskVT) {
3967 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3968 // FIXME: This code seems to be too restrictive, we might consider
3969 // generalizing it or dropping it.
3970 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3971
3972 // Make a new Mask node, with a legal result VT.
3973 SDValue Mask;
3974 SmallVector<SDValue, 4> Ops;
3975 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3976 Ops.push_back(InMask->getOperand(i));
3977 if (InMask->isStrictFPOpcode()) {
3978 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
3979 { MaskVT, MVT::Other }, Ops);
3980 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
3981 }
3982 else
3983 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3984
3985 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3986 // extend or truncate is needed.
3987 LLVMContext &Ctx = *DAG.getContext();
3988 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3989 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3990 if (MaskScalarBits < ToMaskScalBits) {
3991 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3992 MaskVT.getVectorNumElements());
3993 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3994 } else if (MaskScalarBits > ToMaskScalBits) {
3995 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3996 MaskVT.getVectorNumElements());
3997 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3998 }
3999
4000 assert(Mask->getValueType(0).getScalarSizeInBits() ==
4001 ToMaskVT.getScalarSizeInBits() &&
4002 "Mask should have the right element size by now.");
4003
4004 // Adjust Mask to the right number of elements.
4005 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
4006 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
4007 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
4008 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
4009 ZeroIdx);
4010 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
4011 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
4012 EVT SubVT = Mask->getValueType(0);
4013 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
4014 SubOps[0] = Mask;
4015 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
4016 }
4017
4018 assert((Mask->getValueType(0) == ToMaskVT) &&
4019 "A mask of ToMaskVT should have been produced by now.");
4020
4021 return Mask;
4022 }
4023
4024 // This method tries to handle some special cases for the vselect mask
4025 // and if needed adjusting the mask vector type to match that of the VSELECT.
4026 // Without it, many cases end up with scalarization of the SETCC, with many
4027 // unnecessary instructions.
WidenVSELECTMask(SDNode * N)4028 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
4029 LLVMContext &Ctx = *DAG.getContext();
4030 SDValue Cond = N->getOperand(0);
4031
4032 if (N->getOpcode() != ISD::VSELECT)
4033 return SDValue();
4034
4035 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
4036 return SDValue();
4037
4038 // If this is a splitted VSELECT that was previously already handled, do
4039 // nothing.
4040 EVT CondVT = Cond->getValueType(0);
4041 if (CondVT.getScalarSizeInBits() != 1)
4042 return SDValue();
4043
4044 EVT VSelVT = N->getValueType(0);
4045 // Only handle vector types which are a power of 2.
4046 if (!isPowerOf2_64(VSelVT.getSizeInBits()))
4047 return SDValue();
4048
4049 // Don't touch if this will be scalarized.
4050 EVT FinalVT = VSelVT;
4051 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
4052 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
4053
4054 if (FinalVT.getVectorNumElements() == 1)
4055 return SDValue();
4056
4057 // If there is support for an i1 vector mask, don't touch.
4058 if (isSETCCOp(Cond.getOpcode())) {
4059 EVT SetCCOpVT = getSETCCOperandType(Cond);
4060 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
4061 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
4062 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
4063 if (SetCCResVT.getScalarSizeInBits() == 1)
4064 return SDValue();
4065 } else if (CondVT.getScalarType() == MVT::i1) {
4066 // If there is support for an i1 vector mask (or only scalar i1 conditions),
4067 // don't touch.
4068 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
4069 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
4070
4071 if (CondVT.getScalarType() == MVT::i1)
4072 return SDValue();
4073 }
4074
4075 // Widen the vselect result type if needed.
4076 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector)
4077 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
4078
4079 // The mask of the VSELECT should have integer elements.
4080 EVT ToMaskVT = VSelVT;
4081 if (!ToMaskVT.getScalarType().isInteger())
4082 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
4083
4084 SDValue Mask;
4085 if (isSETCCOp(Cond->getOpcode())) {
4086 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
4087 Mask = convertMask(Cond, MaskVT, ToMaskVT);
4088 } else if (isLogicalMaskOp(Cond->getOpcode()) &&
4089 isSETCCOp(Cond->getOperand(0).getOpcode()) &&
4090 isSETCCOp(Cond->getOperand(1).getOpcode())) {
4091 // Cond is (AND/OR/XOR (SETCC, SETCC))
4092 SDValue SETCC0 = Cond->getOperand(0);
4093 SDValue SETCC1 = Cond->getOperand(1);
4094 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
4095 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
4096 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
4097 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
4098 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
4099 EVT MaskVT;
4100 // If the two SETCCs have different VTs, either extend/truncate one of
4101 // them to the other "towards" ToMaskVT, or truncate one and extend the
4102 // other to ToMaskVT.
4103 if (ScalarBits0 != ScalarBits1) {
4104 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
4105 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
4106 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
4107 MaskVT = WideVT;
4108 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
4109 MaskVT = NarrowVT;
4110 else
4111 MaskVT = ToMaskVT;
4112 } else
4113 // If the two SETCCs have the same VT, don't change it.
4114 MaskVT = VT0;
4115
4116 // Make new SETCCs and logical nodes.
4117 SETCC0 = convertMask(SETCC0, VT0, MaskVT);
4118 SETCC1 = convertMask(SETCC1, VT1, MaskVT);
4119 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
4120
4121 // Convert the logical op for VSELECT if needed.
4122 Mask = convertMask(Cond, MaskVT, ToMaskVT);
4123 } else
4124 return SDValue();
4125
4126 return Mask;
4127 }
4128
WidenVecRes_SELECT(SDNode * N)4129 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
4130 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4131 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4132
4133 SDValue Cond1 = N->getOperand(0);
4134 EVT CondVT = Cond1.getValueType();
4135 if (CondVT.isVector()) {
4136 if (SDValue WideCond = WidenVSELECTMask(N)) {
4137 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4138 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4139 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4140 return DAG.getNode(N->getOpcode(), SDLoc(N),
4141 WidenVT, WideCond, InOp1, InOp2);
4142 }
4143
4144 EVT CondEltVT = CondVT.getVectorElementType();
4145 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
4146 CondEltVT, WidenNumElts);
4147 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
4148 Cond1 = GetWidenedVector(Cond1);
4149
4150 // If we have to split the condition there is no point in widening the
4151 // select. This would result in an cycle of widening the select ->
4152 // widening the condition operand -> splitting the condition operand ->
4153 // splitting the select -> widening the select. Instead split this select
4154 // further and widen the resulting type.
4155 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
4156 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
4157 SDValue Res = ModifyToType(SplitSelect, WidenVT);
4158 return Res;
4159 }
4160
4161 if (Cond1.getValueType() != CondWidenVT)
4162 Cond1 = ModifyToType(Cond1, CondWidenVT);
4163 }
4164
4165 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4166 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4167 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4168 return DAG.getNode(N->getOpcode(), SDLoc(N),
4169 WidenVT, Cond1, InOp1, InOp2);
4170 }
4171
WidenVecRes_SELECT_CC(SDNode * N)4172 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
4173 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
4174 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
4175 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
4176 InOp1.getValueType(), N->getOperand(0),
4177 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
4178 }
4179
WidenVecRes_UNDEF(SDNode * N)4180 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
4181 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4182 return DAG.getUNDEF(WidenVT);
4183 }
4184
WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N)4185 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
4186 EVT VT = N->getValueType(0);
4187 SDLoc dl(N);
4188
4189 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4190 unsigned NumElts = VT.getVectorNumElements();
4191 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4192
4193 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4194 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4195
4196 // Adjust mask based on new input vector length.
4197 SmallVector<int, 16> NewMask;
4198 for (unsigned i = 0; i != NumElts; ++i) {
4199 int Idx = N->getMaskElt(i);
4200 if (Idx < (int)NumElts)
4201 NewMask.push_back(Idx);
4202 else
4203 NewMask.push_back(Idx - NumElts + WidenNumElts);
4204 }
4205 for (unsigned i = NumElts; i != WidenNumElts; ++i)
4206 NewMask.push_back(-1);
4207 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
4208 }
4209
WidenVecRes_SETCC(SDNode * N)4210 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4211 assert(N->getValueType(0).isVector() &&
4212 N->getOperand(0).getValueType().isVector() &&
4213 "Operands must be vectors");
4214 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4215 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4216
4217 SDValue InOp1 = N->getOperand(0);
4218 EVT InVT = InOp1.getValueType();
4219 assert(InVT.isVector() && "can not widen non-vector type");
4220 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
4221 InVT.getVectorElementType(), WidenNumElts);
4222
4223 // The input and output types often differ here, and it could be that while
4224 // we'd prefer to widen the result type, the input operands have been split.
4225 // In this case, we also need to split the result of this node as well.
4226 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
4227 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4228 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4229 return Res;
4230 }
4231
4232 // If the inputs also widen, handle them directly. Otherwise widen by hand.
4233 SDValue InOp2 = N->getOperand(1);
4234 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4235 InOp1 = GetWidenedVector(InOp1);
4236 InOp2 = GetWidenedVector(InOp2);
4237 } else {
4238 InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
4239 InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
4240 }
4241
4242 // Assume that the input and output will be widen appropriately. If not,
4243 // we will have to unroll it at some point.
4244 assert(InOp1.getValueType() == WidenInVT &&
4245 InOp2.getValueType() == WidenInVT &&
4246 "Input not widened to expected type!");
4247 (void)WidenInVT;
4248 return DAG.getNode(ISD::SETCC, SDLoc(N),
4249 WidenVT, InOp1, InOp2, N->getOperand(2));
4250 }
4251
WidenVecRes_STRICT_FSETCC(SDNode * N)4252 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
4253 assert(N->getValueType(0).isVector() &&
4254 N->getOperand(1).getValueType().isVector() &&
4255 "Operands must be vectors");
4256 EVT VT = N->getValueType(0);
4257 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4258 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4259 unsigned NumElts = VT.getVectorNumElements();
4260 EVT EltVT = VT.getVectorElementType();
4261
4262 SDLoc dl(N);
4263 SDValue Chain = N->getOperand(0);
4264 SDValue LHS = N->getOperand(1);
4265 SDValue RHS = N->getOperand(2);
4266 SDValue CC = N->getOperand(3);
4267 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
4268
4269 // Fully unroll and reassemble.
4270 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT));
4271 SmallVector<SDValue, 8> Chains(NumElts);
4272 for (unsigned i = 0; i != NumElts; ++i) {
4273 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
4274 DAG.getVectorIdxConstant(i, dl));
4275 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
4276 DAG.getVectorIdxConstant(i, dl));
4277
4278 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
4279 {Chain, LHSElem, RHSElem, CC});
4280 Chains[i] = Scalars[i].getValue(1);
4281 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
4282 DAG.getBoolConstant(true, dl, EltVT, VT),
4283 DAG.getBoolConstant(false, dl, EltVT, VT));
4284 }
4285
4286 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4287 ReplaceValueWith(SDValue(N, 1), NewChain);
4288
4289 return DAG.getBuildVector(WidenVT, dl, Scalars);
4290 }
4291
4292 //===----------------------------------------------------------------------===//
4293 // Widen Vector Operand
4294 //===----------------------------------------------------------------------===//
WidenVectorOperand(SDNode * N,unsigned OpNo)4295 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
4296 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
4297 dbgs() << "\n");
4298 SDValue Res = SDValue();
4299
4300 // See if the target wants to custom widen this node.
4301 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4302 return false;
4303
4304 switch (N->getOpcode()) {
4305 default:
4306 #ifndef NDEBUG
4307 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
4308 N->dump(&DAG);
4309 dbgs() << "\n";
4310 #endif
4311 llvm_unreachable("Do not know how to widen this operator's operand!");
4312
4313 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
4314 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
4315 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
4316 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
4317 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
4318 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
4319 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
4320 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
4321 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
4322 case ISD::STRICT_FSETCC:
4323 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
4324 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
4325 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
4326
4327 case ISD::ANY_EXTEND:
4328 case ISD::SIGN_EXTEND:
4329 case ISD::ZERO_EXTEND:
4330 Res = WidenVecOp_EXTEND(N);
4331 break;
4332
4333 case ISD::FP_EXTEND:
4334 case ISD::STRICT_FP_EXTEND:
4335 case ISD::FP_ROUND:
4336 case ISD::STRICT_FP_ROUND:
4337 case ISD::FP_TO_SINT:
4338 case ISD::STRICT_FP_TO_SINT:
4339 case ISD::FP_TO_UINT:
4340 case ISD::STRICT_FP_TO_UINT:
4341 case ISD::SINT_TO_FP:
4342 case ISD::STRICT_SINT_TO_FP:
4343 case ISD::UINT_TO_FP:
4344 case ISD::STRICT_UINT_TO_FP:
4345 case ISD::TRUNCATE:
4346 Res = WidenVecOp_Convert(N);
4347 break;
4348
4349 case ISD::VECREDUCE_FADD:
4350 case ISD::VECREDUCE_FMUL:
4351 case ISD::VECREDUCE_ADD:
4352 case ISD::VECREDUCE_MUL:
4353 case ISD::VECREDUCE_AND:
4354 case ISD::VECREDUCE_OR:
4355 case ISD::VECREDUCE_XOR:
4356 case ISD::VECREDUCE_SMAX:
4357 case ISD::VECREDUCE_SMIN:
4358 case ISD::VECREDUCE_UMAX:
4359 case ISD::VECREDUCE_UMIN:
4360 case ISD::VECREDUCE_FMAX:
4361 case ISD::VECREDUCE_FMIN:
4362 Res = WidenVecOp_VECREDUCE(N);
4363 break;
4364 case ISD::VECREDUCE_SEQ_FADD:
4365 case ISD::VECREDUCE_SEQ_FMUL:
4366 Res = WidenVecOp_VECREDUCE_SEQ(N);
4367 break;
4368 }
4369
4370 // If Res is null, the sub-method took care of registering the result.
4371 if (!Res.getNode()) return false;
4372
4373 // If the result is N, the sub-method updated N in place. Tell the legalizer
4374 // core about this.
4375 if (Res.getNode() == N)
4376 return true;
4377
4378
4379 if (N->isStrictFPOpcode())
4380 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4381 "Invalid operand expansion");
4382 else
4383 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4384 "Invalid operand expansion");
4385
4386 ReplaceValueWith(SDValue(N, 0), Res);
4387 return false;
4388 }
4389
WidenVecOp_EXTEND(SDNode * N)4390 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4391 SDLoc DL(N);
4392 EVT VT = N->getValueType(0);
4393
4394 SDValue InOp = N->getOperand(0);
4395 assert(getTypeAction(InOp.getValueType()) ==
4396 TargetLowering::TypeWidenVector &&
4397 "Unexpected type action");
4398 InOp = GetWidenedVector(InOp);
4399 assert(VT.getVectorNumElements() <
4400 InOp.getValueType().getVectorNumElements() &&
4401 "Input wasn't widened!");
4402
4403 // We may need to further widen the operand until it has the same total
4404 // vector size as the result.
4405 EVT InVT = InOp.getValueType();
4406 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
4407 EVT InEltVT = InVT.getVectorElementType();
4408 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
4409 EVT FixedVT = (MVT::SimpleValueType)i;
4410 EVT FixedEltVT = FixedVT.getVectorElementType();
4411 if (TLI.isTypeLegal(FixedVT) &&
4412 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
4413 FixedEltVT == InEltVT) {
4414 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
4415 "Not enough elements in the fixed type for the operand!");
4416 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
4417 "We can't have the same type as we started with!");
4418 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
4419 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT,
4420 DAG.getUNDEF(FixedVT), InOp,
4421 DAG.getVectorIdxConstant(0, DL));
4422 else
4423 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
4424 DAG.getVectorIdxConstant(0, DL));
4425 break;
4426 }
4427 }
4428 InVT = InOp.getValueType();
4429 if (InVT.getSizeInBits() != VT.getSizeInBits())
4430 // We couldn't find a legal vector type that was a widening of the input
4431 // and could be extended in-register to the result type, so we have to
4432 // scalarize.
4433 return WidenVecOp_Convert(N);
4434 }
4435
4436 // Use special DAG nodes to represent the operation of extending the
4437 // low lanes.
4438 switch (N->getOpcode()) {
4439 default:
4440 llvm_unreachable("Extend legalization on extend operation!");
4441 case ISD::ANY_EXTEND:
4442 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
4443 case ISD::SIGN_EXTEND:
4444 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
4445 case ISD::ZERO_EXTEND:
4446 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
4447 }
4448 }
4449
WidenVecOp_FCOPYSIGN(SDNode * N)4450 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4451 // The result (and first input) is legal, but the second input is illegal.
4452 // We can't do much to fix that, so just unroll and let the extracts off of
4453 // the second input be widened as needed later.
4454 return DAG.UnrollVectorOp(N);
4455 }
4456
WidenVecOp_Convert(SDNode * N)4457 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4458 // Since the result is legal and the input is illegal.
4459 EVT VT = N->getValueType(0);
4460 EVT EltVT = VT.getVectorElementType();
4461 SDLoc dl(N);
4462 unsigned NumElts = VT.getVectorNumElements();
4463 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
4464 assert(getTypeAction(InOp.getValueType()) ==
4465 TargetLowering::TypeWidenVector &&
4466 "Unexpected type action");
4467 InOp = GetWidenedVector(InOp);
4468 EVT InVT = InOp.getValueType();
4469 unsigned Opcode = N->getOpcode();
4470
4471 // See if a widened result type would be legal, if so widen the node.
4472 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
4473 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4474 InVT.getVectorNumElements());
4475 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
4476 SDValue Res;
4477 if (N->isStrictFPOpcode()) {
4478 if (Opcode == ISD::STRICT_FP_ROUND)
4479 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4480 { N->getOperand(0), InOp, N->getOperand(2) });
4481 else
4482 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4483 { N->getOperand(0), InOp });
4484 // Legalize the chain result - switch anything that used the old chain to
4485 // use the new one.
4486 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4487 } else {
4488 if (Opcode == ISD::FP_ROUND)
4489 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1));
4490 else
4491 Res = DAG.getNode(Opcode, dl, WideVT, InOp);
4492 }
4493 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
4494 DAG.getVectorIdxConstant(0, dl));
4495 }
4496
4497 EVT InEltVT = InVT.getVectorElementType();
4498
4499 // Unroll the convert into some scalar code and create a nasty build vector.
4500 SmallVector<SDValue, 16> Ops(NumElts);
4501 if (N->isStrictFPOpcode()) {
4502 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4503 SmallVector<SDValue, 32> OpChains;
4504 for (unsigned i=0; i < NumElts; ++i) {
4505 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4506 DAG.getVectorIdxConstant(i, dl));
4507 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
4508 OpChains.push_back(Ops[i].getValue(1));
4509 }
4510 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4511 ReplaceValueWith(SDValue(N, 1), NewChain);
4512 } else {
4513 for (unsigned i = 0; i < NumElts; ++i)
4514 Ops[i] = DAG.getNode(Opcode, dl, EltVT,
4515 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT,
4516 InOp, DAG.getVectorIdxConstant(i, dl)));
4517 }
4518
4519 return DAG.getBuildVector(VT, dl, Ops);
4520 }
4521
WidenVecOp_BITCAST(SDNode * N)4522 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4523 EVT VT = N->getValueType(0);
4524 SDValue InOp = GetWidenedVector(N->getOperand(0));
4525 EVT InWidenVT = InOp.getValueType();
4526 SDLoc dl(N);
4527
4528 // Check if we can convert between two legal vector types and extract.
4529 unsigned InWidenSize = InWidenVT.getSizeInBits();
4530 unsigned Size = VT.getSizeInBits();
4531 // x86mmx is not an acceptable vector element type, so don't try.
4532 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
4533 unsigned NewNumElts = InWidenSize / Size;
4534 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4535 if (TLI.isTypeLegal(NewVT)) {
4536 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4537 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
4538 DAG.getVectorIdxConstant(0, dl));
4539 }
4540 }
4541
4542 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
4543 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
4544 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
4545 // having to copy via memory.
4546 if (VT.isVector()) {
4547 EVT EltVT = VT.getVectorElementType();
4548 unsigned EltSize = EltVT.getSizeInBits();
4549 if (InWidenSize % EltSize == 0) {
4550 unsigned NewNumElts = InWidenSize / EltSize;
4551 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
4552 if (TLI.isTypeLegal(NewVT)) {
4553 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4554 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
4555 DAG.getVectorIdxConstant(0, dl));
4556 }
4557 }
4558 }
4559
4560 return CreateStackStoreLoad(InOp, VT);
4561 }
4562
WidenVecOp_CONCAT_VECTORS(SDNode * N)4563 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4564 EVT VT = N->getValueType(0);
4565 EVT EltVT = VT.getVectorElementType();
4566 EVT InVT = N->getOperand(0).getValueType();
4567 SDLoc dl(N);
4568
4569 // If the widen width for this operand is the same as the width of the concat
4570 // and all but the first operand is undef, just use the widened operand.
4571 unsigned NumOperands = N->getNumOperands();
4572 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4573 unsigned i;
4574 for (i = 1; i < NumOperands; ++i)
4575 if (!N->getOperand(i).isUndef())
4576 break;
4577
4578 if (i == NumOperands)
4579 return GetWidenedVector(N->getOperand(0));
4580 }
4581
4582 // Otherwise, fall back to a nasty build vector.
4583 unsigned NumElts = VT.getVectorNumElements();
4584 SmallVector<SDValue, 16> Ops(NumElts);
4585
4586 unsigned NumInElts = InVT.getVectorNumElements();
4587
4588 unsigned Idx = 0;
4589 for (unsigned i=0; i < NumOperands; ++i) {
4590 SDValue InOp = N->getOperand(i);
4591 assert(getTypeAction(InOp.getValueType()) ==
4592 TargetLowering::TypeWidenVector &&
4593 "Unexpected type action");
4594 InOp = GetWidenedVector(InOp);
4595 for (unsigned j = 0; j < NumInElts; ++j)
4596 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4597 DAG.getVectorIdxConstant(j, dl));
4598 }
4599 return DAG.getBuildVector(VT, dl, Ops);
4600 }
4601
WidenVecOp_EXTRACT_SUBVECTOR(SDNode * N)4602 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4603 SDValue InOp = GetWidenedVector(N->getOperand(0));
4604 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4605 N->getValueType(0), InOp, N->getOperand(1));
4606 }
4607
WidenVecOp_EXTRACT_VECTOR_ELT(SDNode * N)4608 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4609 SDValue InOp = GetWidenedVector(N->getOperand(0));
4610 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4611 N->getValueType(0), InOp, N->getOperand(1));
4612 }
4613
WidenVecOp_STORE(SDNode * N)4614 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4615 // We have to widen the value, but we want only to store the original
4616 // vector type.
4617 StoreSDNode *ST = cast<StoreSDNode>(N);
4618
4619 if (!ST->getMemoryVT().getScalarType().isByteSized())
4620 return TLI.scalarizeVectorStore(ST, DAG);
4621
4622 if (ST->isTruncatingStore())
4623 return TLI.scalarizeVectorStore(ST, DAG);
4624
4625 SmallVector<SDValue, 16> StChain;
4626 GenWidenVectorStores(StChain, ST);
4627
4628 if (StChain.size() == 1)
4629 return StChain[0];
4630 else
4631 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4632 }
4633
WidenVecOp_MSTORE(SDNode * N,unsigned OpNo)4634 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4635 assert((OpNo == 1 || OpNo == 3) &&
4636 "Can widen only data or mask operand of mstore");
4637 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
4638 SDValue Mask = MST->getMask();
4639 EVT MaskVT = Mask.getValueType();
4640 SDValue StVal = MST->getValue();
4641 SDLoc dl(N);
4642
4643 if (OpNo == 1) {
4644 // Widen the value.
4645 StVal = GetWidenedVector(StVal);
4646
4647 // The mask should be widened as well.
4648 EVT WideVT = StVal.getValueType();
4649 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4650 MaskVT.getVectorElementType(),
4651 WideVT.getVectorNumElements());
4652 Mask = ModifyToType(Mask, WideMaskVT, true);
4653 } else {
4654 // Widen the mask.
4655 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4656 Mask = ModifyToType(Mask, WideMaskVT, true);
4657
4658 EVT ValueVT = StVal.getValueType();
4659 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4660 ValueVT.getVectorElementType(),
4661 WideMaskVT.getVectorNumElements());
4662 StVal = ModifyToType(StVal, WideVT);
4663 }
4664
4665 assert(Mask.getValueType().getVectorNumElements() ==
4666 StVal.getValueType().getVectorNumElements() &&
4667 "Mask and data vectors should have the same number of elements");
4668 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4669 MST->getOffset(), Mask, MST->getMemoryVT(),
4670 MST->getMemOperand(), MST->getAddressingMode(),
4671 false, MST->isCompressingStore());
4672 }
4673
WidenVecOp_MGATHER(SDNode * N,unsigned OpNo)4674 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4675 assert(OpNo == 4 && "Can widen only the index of mgather");
4676 auto *MG = cast<MaskedGatherSDNode>(N);
4677 SDValue DataOp = MG->getPassThru();
4678 SDValue Mask = MG->getMask();
4679 SDValue Scale = MG->getScale();
4680
4681 // Just widen the index. It's allowed to have extra elements.
4682 SDValue Index = GetWidenedVector(MG->getIndex());
4683
4684 SDLoc dl(N);
4685 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4686 Scale};
4687 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4688 MG->getMemOperand(), MG->getIndexType());
4689 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4690 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4691 return SDValue();
4692 }
4693
WidenVecOp_MSCATTER(SDNode * N,unsigned OpNo)4694 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4695 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
4696 SDValue DataOp = MSC->getValue();
4697 SDValue Mask = MSC->getMask();
4698 SDValue Index = MSC->getIndex();
4699 SDValue Scale = MSC->getScale();
4700
4701 if (OpNo == 1) {
4702 DataOp = GetWidenedVector(DataOp);
4703 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
4704
4705 // Widen index.
4706 EVT IndexVT = Index.getValueType();
4707 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4708 IndexVT.getVectorElementType(), NumElts);
4709 Index = ModifyToType(Index, WideIndexVT);
4710
4711 // The mask should be widened as well.
4712 EVT MaskVT = Mask.getValueType();
4713 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4714 MaskVT.getVectorElementType(), NumElts);
4715 Mask = ModifyToType(Mask, WideMaskVT, true);
4716 } else if (OpNo == 4) {
4717 // Just widen the index. It's allowed to have extra elements.
4718 Index = GetWidenedVector(Index);
4719 } else
4720 llvm_unreachable("Can't widen this operand of mscatter");
4721
4722 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4723 Scale};
4724 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
4725 MSC->getMemoryVT(), SDLoc(N), Ops,
4726 MSC->getMemOperand(), MSC->getIndexType(),
4727 MSC->isTruncatingStore());
4728 }
4729
WidenVecOp_SETCC(SDNode * N)4730 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4731 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4732 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4733 SDLoc dl(N);
4734 EVT VT = N->getValueType(0);
4735
4736 // WARNING: In this code we widen the compare instruction with garbage.
4737 // This garbage may contain denormal floats which may be slow. Is this a real
4738 // concern ? Should we zero the unused lanes if this is a float compare ?
4739
4740 // Get a new SETCC node to compare the newly widened operands.
4741 // Only some of the compared elements are legal.
4742 EVT SVT = getSetCCResultType(InOp0.getValueType());
4743 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
4744 if (VT.getScalarType() == MVT::i1)
4745 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4746 SVT.getVectorNumElements());
4747
4748 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4749 SVT, InOp0, InOp1, N->getOperand(2));
4750
4751 // Extract the needed results from the result vector.
4752 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
4753 SVT.getVectorElementType(),
4754 VT.getVectorNumElements());
4755 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
4756 DAG.getVectorIdxConstant(0, dl));
4757
4758 EVT OpVT = N->getOperand(0).getValueType();
4759 ISD::NodeType ExtendCode =
4760 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
4761 return DAG.getNode(ExtendCode, dl, VT, CC);
4762 }
4763
WidenVecOp_STRICT_FSETCC(SDNode * N)4764 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
4765 SDValue Chain = N->getOperand(0);
4766 SDValue LHS = GetWidenedVector(N->getOperand(1));
4767 SDValue RHS = GetWidenedVector(N->getOperand(2));
4768 SDValue CC = N->getOperand(3);
4769 SDLoc dl(N);
4770
4771 EVT VT = N->getValueType(0);
4772 EVT EltVT = VT.getVectorElementType();
4773 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
4774 unsigned NumElts = VT.getVectorNumElements();
4775
4776 // Unroll into a build vector.
4777 SmallVector<SDValue, 8> Scalars(NumElts);
4778 SmallVector<SDValue, 8> Chains(NumElts);
4779
4780 for (unsigned i = 0; i != NumElts; ++i) {
4781 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
4782 DAG.getVectorIdxConstant(i, dl));
4783 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
4784 DAG.getVectorIdxConstant(i, dl));
4785
4786 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
4787 {Chain, LHSElem, RHSElem, CC});
4788 Chains[i] = Scalars[i].getValue(1);
4789 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
4790 DAG.getBoolConstant(true, dl, EltVT, VT),
4791 DAG.getBoolConstant(false, dl, EltVT, VT));
4792 }
4793
4794 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4795 ReplaceValueWith(SDValue(N, 1), NewChain);
4796
4797 return DAG.getBuildVector(VT, dl, Scalars);
4798 }
4799
WidenVecOp_VECREDUCE(SDNode * N)4800 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
4801 SDLoc dl(N);
4802 SDValue Op = GetWidenedVector(N->getOperand(0));
4803 EVT OrigVT = N->getOperand(0).getValueType();
4804 EVT WideVT = Op.getValueType();
4805 EVT ElemVT = OrigVT.getVectorElementType();
4806 SDNodeFlags Flags = N->getFlags();
4807
4808 unsigned Opc = N->getOpcode();
4809 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
4810 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
4811 assert(NeutralElem && "Neutral element must exist");
4812
4813 // Pad the vector with the neutral element.
4814 unsigned OrigElts = OrigVT.getVectorNumElements();
4815 unsigned WideElts = WideVT.getVectorNumElements();
4816 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
4817 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
4818 DAG.getVectorIdxConstant(Idx, dl));
4819
4820 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
4821 }
4822
WidenVecOp_VECREDUCE_SEQ(SDNode * N)4823 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
4824 SDLoc dl(N);
4825 SDValue AccOp = N->getOperand(0);
4826 SDValue VecOp = N->getOperand(1);
4827 SDValue Op = GetWidenedVector(VecOp);
4828
4829 EVT OrigVT = VecOp.getValueType();
4830 EVT WideVT = Op.getValueType();
4831 EVT ElemVT = OrigVT.getVectorElementType();
4832 SDNodeFlags Flags = N->getFlags();
4833
4834 unsigned Opc = N->getOpcode();
4835 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
4836 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
4837
4838 // Pad the vector with the neutral element.
4839 unsigned OrigElts = OrigVT.getVectorNumElements();
4840 unsigned WideElts = WideVT.getVectorNumElements();
4841 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
4842 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
4843 DAG.getVectorIdxConstant(Idx, dl));
4844
4845 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags);
4846 }
4847
WidenVecOp_VSELECT(SDNode * N)4848 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
4849 // This only gets called in the case that the left and right inputs and
4850 // result are of a legal odd vector type, and the condition is illegal i1 of
4851 // the same odd width that needs widening.
4852 EVT VT = N->getValueType(0);
4853 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
4854
4855 SDValue Cond = GetWidenedVector(N->getOperand(0));
4856 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
4857 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
4858 SDLoc DL(N);
4859
4860 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
4861 LeftIn, RightIn);
4862 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
4863 DAG.getVectorIdxConstant(0, DL));
4864 }
4865
4866 //===----------------------------------------------------------------------===//
4867 // Vector Widening Utilities
4868 //===----------------------------------------------------------------------===//
4869
4870 // Utility function to find the type to chop up a widen vector for load/store
4871 // TLI: Target lowering used to determine legal types.
4872 // Width: Width left need to load/store.
4873 // WidenVT: The widen vector type to load to/store from
4874 // Align: If 0, don't allow use of a wider type
4875 // WidenEx: If Align is not 0, the amount additional we can load/store from.
4876
FindMemType(SelectionDAG & DAG,const TargetLowering & TLI,unsigned Width,EVT WidenVT,unsigned Align=0,unsigned WidenEx=0)4877 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
4878 unsigned Width, EVT WidenVT,
4879 unsigned Align = 0, unsigned WidenEx = 0) {
4880 EVT WidenEltVT = WidenVT.getVectorElementType();
4881 const bool Scalable = WidenVT.isScalableVector();
4882 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize();
4883 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
4884 unsigned AlignInBits = Align*8;
4885
4886 // If we have one element to load/store, return it.
4887 EVT RetVT = WidenEltVT;
4888 if (!Scalable && Width == WidenEltWidth)
4889 return RetVT;
4890
4891 // See if there is larger legal integer than the element type to load/store.
4892 unsigned VT;
4893 // Don't bother looking for an integer type if the vector is scalable, skip
4894 // to vector types.
4895 if (!Scalable) {
4896 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
4897 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
4898 EVT MemVT((MVT::SimpleValueType) VT);
4899 unsigned MemVTWidth = MemVT.getSizeInBits();
4900 if (MemVT.getSizeInBits() <= WidenEltWidth)
4901 break;
4902 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4903 if ((Action == TargetLowering::TypeLegal ||
4904 Action == TargetLowering::TypePromoteInteger) &&
4905 (WidenWidth % MemVTWidth) == 0 &&
4906 isPowerOf2_32(WidenWidth / MemVTWidth) &&
4907 (MemVTWidth <= Width ||
4908 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4909 if (MemVTWidth == WidenWidth)
4910 return MemVT;
4911 RetVT = MemVT;
4912 break;
4913 }
4914 }
4915 }
4916
4917 // See if there is a larger vector type to load/store that has the same vector
4918 // element type and is evenly divisible with the WidenVT.
4919 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
4920 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
4921 EVT MemVT = (MVT::SimpleValueType) VT;
4922 // Skip vector MVTs which don't match the scalable property of WidenVT.
4923 if (Scalable != MemVT.isScalableVector())
4924 continue;
4925 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize();
4926 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4927 if ((Action == TargetLowering::TypeLegal ||
4928 Action == TargetLowering::TypePromoteInteger) &&
4929 WidenEltVT == MemVT.getVectorElementType() &&
4930 (WidenWidth % MemVTWidth) == 0 &&
4931 isPowerOf2_32(WidenWidth / MemVTWidth) &&
4932 (MemVTWidth <= Width ||
4933 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4934 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
4935 return MemVT;
4936 }
4937 }
4938
4939 if (Scalable)
4940 report_fatal_error("Using element-wise loads and stores for widening "
4941 "operations is not supported for scalable vectors");
4942 return RetVT;
4943 }
4944
4945 // Builds a vector type from scalar loads
4946 // VecTy: Resulting Vector type
4947 // LDOps: Load operators to build a vector type
4948 // [Start,End) the list of loads to use.
BuildVectorFromScalar(SelectionDAG & DAG,EVT VecTy,SmallVectorImpl<SDValue> & LdOps,unsigned Start,unsigned End)4949 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
4950 SmallVectorImpl<SDValue> &LdOps,
4951 unsigned Start, unsigned End) {
4952 SDLoc dl(LdOps[Start]);
4953 EVT LdTy = LdOps[Start].getValueType();
4954 unsigned Width = VecTy.getSizeInBits();
4955 unsigned NumElts = Width / LdTy.getSizeInBits();
4956 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
4957
4958 unsigned Idx = 1;
4959 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4960
4961 for (unsigned i = Start + 1; i != End; ++i) {
4962 EVT NewLdTy = LdOps[i].getValueType();
4963 if (NewLdTy != LdTy) {
4964 NumElts = Width / NewLdTy.getSizeInBits();
4965 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
4966 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
4967 // Readjust position and vector position based on new load type.
4968 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
4969 LdTy = NewLdTy;
4970 }
4971 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
4972 DAG.getVectorIdxConstant(Idx++, dl));
4973 }
4974 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
4975 }
4976
GenWidenVectorLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD)4977 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4978 LoadSDNode *LD) {
4979 // The strategy assumes that we can efficiently load power-of-two widths.
4980 // The routine chops the vector into the largest vector loads with the same
4981 // element type or scalar loads and then recombines it to the widen vector
4982 // type.
4983 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4984 EVT LdVT = LD->getMemoryVT();
4985 SDLoc dl(LD);
4986 assert(LdVT.isVector() && WidenVT.isVector());
4987 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
4988 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
4989
4990 // Load information
4991 SDValue Chain = LD->getChain();
4992 SDValue BasePtr = LD->getBasePtr();
4993 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4994 AAMDNodes AAInfo = LD->getAAInfo();
4995
4996 TypeSize LdWidth = LdVT.getSizeInBits();
4997 TypeSize WidenWidth = WidenVT.getSizeInBits();
4998 TypeSize WidthDiff = WidenWidth - LdWidth;
4999 // Allow wider loads if they are sufficiently aligned to avoid memory faults
5000 // and if the original load is simple.
5001 unsigned LdAlign = (!LD->isSimple()) ? 0 : LD->getAlignment();
5002
5003 // Find the vector type that can load from.
5004 EVT NewVT = FindMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign,
5005 WidthDiff.getKnownMinSize());
5006 TypeSize NewVTWidth = NewVT.getSizeInBits();
5007 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
5008 LD->getOriginalAlign(), MMOFlags, AAInfo);
5009 LdChain.push_back(LdOp.getValue(1));
5010
5011 // Check if we can load the element with one instruction.
5012 if (TypeSize::isKnownLE(LdWidth, NewVTWidth)) {
5013 if (!NewVT.isVector()) {
5014 unsigned NumElts = WidenWidth.getFixedSize() / NewVTWidth.getFixedSize();
5015 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
5016 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
5017 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
5018 }
5019 if (NewVT == WidenVT)
5020 return LdOp;
5021
5022 // TODO: We don't currently have any tests that exercise this code path.
5023 assert(WidenWidth.getFixedSize() % NewVTWidth.getFixedSize() == 0);
5024 unsigned NumConcat = WidenWidth.getFixedSize() / NewVTWidth.getFixedSize();
5025 SmallVector<SDValue, 16> ConcatOps(NumConcat);
5026 SDValue UndefVal = DAG.getUNDEF(NewVT);
5027 ConcatOps[0] = LdOp;
5028 for (unsigned i = 1; i != NumConcat; ++i)
5029 ConcatOps[i] = UndefVal;
5030 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
5031 }
5032
5033 // Load vector by using multiple loads from largest vector to scalar.
5034 SmallVector<SDValue, 16> LdOps;
5035 LdOps.push_back(LdOp);
5036
5037 uint64_t ScaledOffset = 0;
5038 MachinePointerInfo MPI = LD->getPointerInfo();
5039 do {
5040 LdWidth -= NewVTWidth;
5041 IncrementPointer(cast<LoadSDNode>(LdOp), NewVT, MPI, BasePtr,
5042 &ScaledOffset);
5043
5044 if (TypeSize::isKnownLT(LdWidth, NewVTWidth)) {
5045 // The current type we are using is too large. Find a better size.
5046 NewVT = FindMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign,
5047 WidthDiff.getKnownMinSize());
5048 NewVTWidth = NewVT.getSizeInBits();
5049 }
5050
5051 Align NewAlign = ScaledOffset == 0
5052 ? LD->getOriginalAlign()
5053 : commonAlignment(LD->getAlign(), ScaledOffset);
5054 SDValue L =
5055 DAG.getLoad(NewVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags, AAInfo);
5056 LdChain.push_back(L.getValue(1));
5057
5058 LdOps.push_back(L);
5059 LdOp = L;
5060 } while (TypeSize::isKnownGT(LdWidth, NewVTWidth));
5061
5062 // Build the vector from the load operations.
5063 unsigned End = LdOps.size();
5064 if (!LdOps[0].getValueType().isVector())
5065 // All the loads are scalar loads.
5066 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
5067
5068 // If the load contains vectors, build the vector using concat vector.
5069 // All of the vectors used to load are power-of-2, and the scalar loads can be
5070 // combined to make a power-of-2 vector.
5071 SmallVector<SDValue, 16> ConcatOps(End);
5072 int i = End - 1;
5073 int Idx = End;
5074 EVT LdTy = LdOps[i].getValueType();
5075 // First, combine the scalar loads to a vector.
5076 if (!LdTy.isVector()) {
5077 for (--i; i >= 0; --i) {
5078 LdTy = LdOps[i].getValueType();
5079 if (LdTy.isVector())
5080 break;
5081 }
5082 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
5083 }
5084
5085 ConcatOps[--Idx] = LdOps[i];
5086 for (--i; i >= 0; --i) {
5087 EVT NewLdTy = LdOps[i].getValueType();
5088 if (NewLdTy != LdTy) {
5089 // Create a larger vector.
5090 TypeSize LdTySize = LdTy.getSizeInBits();
5091 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
5092 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
5093 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinSize()));
5094 unsigned NumOps =
5095 NewLdTySize.getKnownMinSize() / LdTySize.getKnownMinSize();
5096 SmallVector<SDValue, 16> WidenOps(NumOps);
5097 unsigned j = 0;
5098 for (; j != End-Idx; ++j)
5099 WidenOps[j] = ConcatOps[Idx+j];
5100 for (; j != NumOps; ++j)
5101 WidenOps[j] = DAG.getUNDEF(LdTy);
5102
5103 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
5104 WidenOps);
5105 Idx = End - 1;
5106 LdTy = NewLdTy;
5107 }
5108 ConcatOps[--Idx] = LdOps[i];
5109 }
5110
5111 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
5112 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
5113 makeArrayRef(&ConcatOps[Idx], End - Idx));
5114
5115 // We need to fill the rest with undefs to build the vector.
5116 unsigned NumOps =
5117 WidenWidth.getKnownMinSize() / LdTy.getSizeInBits().getKnownMinSize();
5118 SmallVector<SDValue, 16> WidenOps(NumOps);
5119 SDValue UndefVal = DAG.getUNDEF(LdTy);
5120 {
5121 unsigned i = 0;
5122 for (; i != End-Idx; ++i)
5123 WidenOps[i] = ConcatOps[Idx+i];
5124 for (; i != NumOps; ++i)
5125 WidenOps[i] = UndefVal;
5126 }
5127 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
5128 }
5129
5130 SDValue
GenWidenVectorExtLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD,ISD::LoadExtType ExtType)5131 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
5132 LoadSDNode *LD,
5133 ISD::LoadExtType ExtType) {
5134 // For extension loads, it may not be more efficient to chop up the vector
5135 // and then extend it. Instead, we unroll the load and build a new vector.
5136 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
5137 EVT LdVT = LD->getMemoryVT();
5138 SDLoc dl(LD);
5139 assert(LdVT.isVector() && WidenVT.isVector());
5140 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
5141
5142 // Load information
5143 SDValue Chain = LD->getChain();
5144 SDValue BasePtr = LD->getBasePtr();
5145 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
5146 AAMDNodes AAInfo = LD->getAAInfo();
5147
5148 if (LdVT.isScalableVector())
5149 report_fatal_error("Generating widen scalable extending vector loads is "
5150 "not yet supported");
5151
5152 EVT EltVT = WidenVT.getVectorElementType();
5153 EVT LdEltVT = LdVT.getVectorElementType();
5154 unsigned NumElts = LdVT.getVectorNumElements();
5155
5156 // Load each element and widen.
5157 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5158 SmallVector<SDValue, 16> Ops(WidenNumElts);
5159 unsigned Increment = LdEltVT.getSizeInBits() / 8;
5160 Ops[0] =
5161 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
5162 LdEltVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
5163 LdChain.push_back(Ops[0].getValue(1));
5164 unsigned i = 0, Offset = Increment;
5165 for (i=1; i < NumElts; ++i, Offset += Increment) {
5166 SDValue NewBasePtr =
5167 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::Fixed(Offset));
5168 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
5169 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
5170 LD->getOriginalAlign(), MMOFlags, AAInfo);
5171 LdChain.push_back(Ops[i].getValue(1));
5172 }
5173
5174 // Fill the rest with undefs.
5175 SDValue UndefVal = DAG.getUNDEF(EltVT);
5176 for (; i != WidenNumElts; ++i)
5177 Ops[i] = UndefVal;
5178
5179 return DAG.getBuildVector(WidenVT, dl, Ops);
5180 }
5181
GenWidenVectorStores(SmallVectorImpl<SDValue> & StChain,StoreSDNode * ST)5182 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
5183 StoreSDNode *ST) {
5184 // The strategy assumes that we can efficiently store power-of-two widths.
5185 // The routine chops the vector into the largest vector stores with the same
5186 // element type or scalar stores.
5187 SDValue Chain = ST->getChain();
5188 SDValue BasePtr = ST->getBasePtr();
5189 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
5190 AAMDNodes AAInfo = ST->getAAInfo();
5191 SDValue ValOp = GetWidenedVector(ST->getValue());
5192 SDLoc dl(ST);
5193
5194 EVT StVT = ST->getMemoryVT();
5195 TypeSize StWidth = StVT.getSizeInBits();
5196 EVT ValVT = ValOp.getValueType();
5197 TypeSize ValWidth = ValVT.getSizeInBits();
5198 EVT ValEltVT = ValVT.getVectorElementType();
5199 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
5200 assert(StVT.getVectorElementType() == ValEltVT);
5201 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
5202 "Mismatch between store and value types");
5203
5204 int Idx = 0; // current index to store
5205
5206 MachinePointerInfo MPI = ST->getPointerInfo();
5207 uint64_t ScaledOffset = 0;
5208 while (StWidth.isNonZero()) {
5209 // Find the largest vector type we can store with.
5210 EVT NewVT = FindMemType(DAG, TLI, StWidth.getKnownMinSize(), ValVT);
5211 TypeSize NewVTWidth = NewVT.getSizeInBits();
5212
5213 if (NewVT.isVector()) {
5214 unsigned NumVTElts = NewVT.getVectorMinNumElements();
5215 do {
5216 Align NewAlign = ScaledOffset == 0
5217 ? ST->getOriginalAlign()
5218 : commonAlignment(ST->getAlign(), ScaledOffset);
5219 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
5220 DAG.getVectorIdxConstant(Idx, dl));
5221 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign,
5222 MMOFlags, AAInfo);
5223 StChain.push_back(PartStore);
5224
5225 StWidth -= NewVTWidth;
5226 Idx += NumVTElts;
5227
5228 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr,
5229 &ScaledOffset);
5230 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
5231 } else {
5232 // Cast the vector to the scalar type we can store.
5233 unsigned NumElts = ValWidth.getFixedSize() / NewVTWidth.getFixedSize();
5234 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
5235 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
5236 // Readjust index position based on new vector type.
5237 Idx = Idx * ValEltWidth / NewVTWidth.getFixedSize();
5238 do {
5239 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
5240 DAG.getVectorIdxConstant(Idx++, dl));
5241 SDValue PartStore =
5242 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getOriginalAlign(),
5243 MMOFlags, AAInfo);
5244 StChain.push_back(PartStore);
5245
5246 StWidth -= NewVTWidth;
5247 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr);
5248 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
5249 // Restore index back to be relative to the original widen element type.
5250 Idx = Idx * NewVTWidth.getFixedSize() / ValEltWidth;
5251 }
5252 }
5253 }
5254
5255 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
5256 /// input vector must have the same element type as NVT.
5257 /// FillWithZeroes specifies that the vector should be widened with zeroes.
ModifyToType(SDValue InOp,EVT NVT,bool FillWithZeroes)5258 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5259 bool FillWithZeroes) {
5260 // Note that InOp might have been widened so it might already have
5261 // the right width or it might need be narrowed.
5262 EVT InVT = InOp.getValueType();
5263 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
5264 "input and widen element type must match");
5265 SDLoc dl(InOp);
5266
5267 // Check if InOp already has the right width.
5268 if (InVT == NVT)
5269 return InOp;
5270
5271 unsigned InNumElts = InVT.getVectorNumElements();
5272 unsigned WidenNumElts = NVT.getVectorNumElements();
5273 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
5274 unsigned NumConcat = WidenNumElts / InNumElts;
5275 SmallVector<SDValue, 16> Ops(NumConcat);
5276 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
5277 DAG.getUNDEF(InVT);
5278 Ops[0] = InOp;
5279 for (unsigned i = 1; i != NumConcat; ++i)
5280 Ops[i] = FillVal;
5281
5282 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
5283 }
5284
5285 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
5286 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
5287 DAG.getVectorIdxConstant(0, dl));
5288
5289 // Fall back to extract and build.
5290 SmallVector<SDValue, 16> Ops(WidenNumElts);
5291 EVT EltVT = NVT.getVectorElementType();
5292 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
5293 unsigned Idx;
5294 for (Idx = 0; Idx < MinNumElts; ++Idx)
5295 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5296 DAG.getVectorIdxConstant(Idx, dl));
5297
5298 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
5299 DAG.getUNDEF(EltVT);
5300 for ( ; Idx < WidenNumElts; ++Idx)
5301 Ops[Idx] = FillVal;
5302 return DAG.getBuildVector(NVT, dl, Ops);
5303 }
5304