1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "LegalizeTypes.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/Support/ErrorHandling.h"
25 #include "llvm/Support/raw_ostream.h"
26 #include "llvm/Support/TypeSize.h"
27 using namespace llvm;
28
29 #define DEBUG_TYPE "legalize-types"
30
31 //===----------------------------------------------------------------------===//
32 // Result Vector Scalarization: <1 x ty> -> ty.
33 //===----------------------------------------------------------------------===//
34
ScalarizeVectorResult(SDNode * N,unsigned ResNo)35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
36 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
37 dbgs() << "\n");
38 SDValue R = SDValue();
39
40 switch (N->getOpcode()) {
41 default:
42 #ifndef NDEBUG
43 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
44 N->dump(&DAG);
45 dbgs() << "\n";
46 #endif
47 report_fatal_error("Do not know how to scalarize the result of this "
48 "operator!\n");
49
50 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
51 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
52 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
53 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
54 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
55 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
56 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
57 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
58 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
59 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
60 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
61 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
62 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
63 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
64 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
65 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
66 case ISD::ANY_EXTEND_VECTOR_INREG:
67 case ISD::SIGN_EXTEND_VECTOR_INREG:
68 case ISD::ZERO_EXTEND_VECTOR_INREG:
69 R = ScalarizeVecRes_VecInregOp(N);
70 break;
71 case ISD::ABS:
72 case ISD::ANY_EXTEND:
73 case ISD::BITREVERSE:
74 case ISD::BSWAP:
75 case ISD::CTLZ:
76 case ISD::CTLZ_ZERO_UNDEF:
77 case ISD::CTPOP:
78 case ISD::CTTZ:
79 case ISD::CTTZ_ZERO_UNDEF:
80 case ISD::FABS:
81 case ISD::FCEIL:
82 case ISD::FCOS:
83 case ISD::FEXP:
84 case ISD::FEXP2:
85 case ISD::FFLOOR:
86 case ISD::FLOG:
87 case ISD::FLOG10:
88 case ISD::FLOG2:
89 case ISD::FNEARBYINT:
90 case ISD::FNEG:
91 case ISD::FP_EXTEND:
92 case ISD::FP_TO_SINT:
93 case ISD::FP_TO_UINT:
94 case ISD::FRINT:
95 case ISD::FROUND:
96 case ISD::FSIN:
97 case ISD::FSQRT:
98 case ISD::FTRUNC:
99 case ISD::SIGN_EXTEND:
100 case ISD::SINT_TO_FP:
101 case ISD::TRUNCATE:
102 case ISD::UINT_TO_FP:
103 case ISD::ZERO_EXTEND:
104 case ISD::FCANONICALIZE:
105 R = ScalarizeVecRes_UnaryOp(N);
106 break;
107
108 case ISD::ADD:
109 case ISD::AND:
110 case ISD::FADD:
111 case ISD::FCOPYSIGN:
112 case ISD::FDIV:
113 case ISD::FMUL:
114 case ISD::FMINNUM:
115 case ISD::FMAXNUM:
116 case ISD::FMINNUM_IEEE:
117 case ISD::FMAXNUM_IEEE:
118 case ISD::FMINIMUM:
119 case ISD::FMAXIMUM:
120 case ISD::SMIN:
121 case ISD::SMAX:
122 case ISD::UMIN:
123 case ISD::UMAX:
124
125 case ISD::SADDSAT:
126 case ISD::UADDSAT:
127 case ISD::SSUBSAT:
128 case ISD::USUBSAT:
129
130 case ISD::FPOW:
131 case ISD::FREM:
132 case ISD::FSUB:
133 case ISD::MUL:
134 case ISD::OR:
135 case ISD::SDIV:
136 case ISD::SREM:
137 case ISD::SUB:
138 case ISD::UDIV:
139 case ISD::UREM:
140 case ISD::XOR:
141 case ISD::SHL:
142 case ISD::SRA:
143 case ISD::SRL:
144 R = ScalarizeVecRes_BinOp(N);
145 break;
146 case ISD::FMA:
147 R = ScalarizeVecRes_TernaryOp(N);
148 break;
149
150 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
151 case ISD::STRICT_##DAGN:
152 #include "llvm/IR/ConstrainedOps.def"
153 R = ScalarizeVecRes_StrictFPOp(N);
154 break;
155
156 case ISD::UADDO:
157 case ISD::SADDO:
158 case ISD::USUBO:
159 case ISD::SSUBO:
160 case ISD::UMULO:
161 case ISD::SMULO:
162 R = ScalarizeVecRes_OverflowOp(N, ResNo);
163 break;
164 case ISD::SMULFIX:
165 case ISD::SMULFIXSAT:
166 case ISD::UMULFIX:
167 case ISD::UMULFIXSAT:
168 case ISD::SDIVFIX:
169 case ISD::UDIVFIX:
170 R = ScalarizeVecRes_FIX(N);
171 break;
172 }
173
174 // If R is null, the sub-method took care of registering the result.
175 if (R.getNode())
176 SetScalarizedVector(SDValue(N, ResNo), R);
177 }
178
ScalarizeVecRes_BinOp(SDNode * N)179 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
180 SDValue LHS = GetScalarizedVector(N->getOperand(0));
181 SDValue RHS = GetScalarizedVector(N->getOperand(1));
182 return DAG.getNode(N->getOpcode(), SDLoc(N),
183 LHS.getValueType(), LHS, RHS, N->getFlags());
184 }
185
ScalarizeVecRes_TernaryOp(SDNode * N)186 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
187 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
188 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
189 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
190 return DAG.getNode(N->getOpcode(), SDLoc(N),
191 Op0.getValueType(), Op0, Op1, Op2);
192 }
193
ScalarizeVecRes_FIX(SDNode * N)194 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
195 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
196 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
197 SDValue Op2 = N->getOperand(2);
198 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
199 Op2);
200 }
201
ScalarizeVecRes_StrictFPOp(SDNode * N)202 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
203 EVT VT = N->getValueType(0).getVectorElementType();
204 unsigned NumOpers = N->getNumOperands();
205 SDValue Chain = N->getOperand(0);
206 EVT ValueVTs[] = {VT, MVT::Other};
207 SDLoc dl(N);
208
209 SmallVector<SDValue, 4> Opers(NumOpers);
210
211 // The Chain is the first operand.
212 Opers[0] = Chain;
213
214 // Now process the remaining operands.
215 for (unsigned i = 1; i < NumOpers; ++i) {
216 SDValue Oper = N->getOperand(i);
217
218 if (Oper.getValueType().isVector())
219 Oper = GetScalarizedVector(Oper);
220
221 Opers[i] = Oper;
222 }
223
224 SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
225
226 // Legalize the chain result - switch anything that used the old chain to
227 // use the new one.
228 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
229 return Result;
230 }
231
ScalarizeVecRes_OverflowOp(SDNode * N,unsigned ResNo)232 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
233 unsigned ResNo) {
234 SDLoc DL(N);
235 EVT ResVT = N->getValueType(0);
236 EVT OvVT = N->getValueType(1);
237
238 SDValue ScalarLHS, ScalarRHS;
239 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
240 ScalarLHS = GetScalarizedVector(N->getOperand(0));
241 ScalarRHS = GetScalarizedVector(N->getOperand(1));
242 } else {
243 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
244 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
245 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
246 ScalarLHS = ElemsLHS[0];
247 ScalarRHS = ElemsRHS[0];
248 }
249
250 SDVTList ScalarVTs = DAG.getVTList(
251 ResVT.getVectorElementType(), OvVT.getVectorElementType());
252 SDNode *ScalarNode = DAG.getNode(
253 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
254
255 // Replace the other vector result not being explicitly scalarized here.
256 unsigned OtherNo = 1 - ResNo;
257 EVT OtherVT = N->getValueType(OtherNo);
258 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
259 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
260 } else {
261 SDValue OtherVal = DAG.getNode(
262 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
263 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
264 }
265
266 return SDValue(ScalarNode, ResNo);
267 }
268
ScalarizeVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)269 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
270 unsigned ResNo) {
271 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
272 return GetScalarizedVector(Op);
273 }
274
ScalarizeVecRes_BITCAST(SDNode * N)275 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
276 SDValue Op = N->getOperand(0);
277 if (Op.getValueType().isVector()
278 && Op.getValueType().getVectorNumElements() == 1
279 && !isSimpleLegalType(Op.getValueType()))
280 Op = GetScalarizedVector(Op);
281 EVT NewVT = N->getValueType(0).getVectorElementType();
282 return DAG.getNode(ISD::BITCAST, SDLoc(N),
283 NewVT, Op);
284 }
285
ScalarizeVecRes_BUILD_VECTOR(SDNode * N)286 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
287 EVT EltVT = N->getValueType(0).getVectorElementType();
288 SDValue InOp = N->getOperand(0);
289 // The BUILD_VECTOR operands may be of wider element types and
290 // we may need to truncate them back to the requested return type.
291 if (EltVT.isInteger())
292 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
293 return InOp;
294 }
295
ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode * N)296 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
297 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
298 N->getValueType(0).getVectorElementType(),
299 N->getOperand(0), N->getOperand(1));
300 }
301
ScalarizeVecRes_FP_ROUND(SDNode * N)302 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
303 EVT NewVT = N->getValueType(0).getVectorElementType();
304 SDValue Op = GetScalarizedVector(N->getOperand(0));
305 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
306 NewVT, Op, N->getOperand(1));
307 }
308
ScalarizeVecRes_FPOWI(SDNode * N)309 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
310 SDValue Op = GetScalarizedVector(N->getOperand(0));
311 return DAG.getNode(ISD::FPOWI, SDLoc(N),
312 Op.getValueType(), Op, N->getOperand(1));
313 }
314
ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode * N)315 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
316 // The value to insert may have a wider type than the vector element type,
317 // so be sure to truncate it to the element type if necessary.
318 SDValue Op = N->getOperand(1);
319 EVT EltVT = N->getValueType(0).getVectorElementType();
320 if (Op.getValueType() != EltVT)
321 // FIXME: Can this happen for floating point types?
322 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
323 return Op;
324 }
325
ScalarizeVecRes_LOAD(LoadSDNode * N)326 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
327 assert(N->isUnindexed() && "Indexed vector load?");
328
329 SDValue Result = DAG.getLoad(
330 ISD::UNINDEXED, N->getExtensionType(),
331 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
332 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
333 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
334 N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
335 N->getAAInfo());
336
337 // Legalize the chain result - switch anything that used the old chain to
338 // use the new one.
339 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
340 return Result;
341 }
342
ScalarizeVecRes_UnaryOp(SDNode * N)343 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
344 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
345 EVT DestVT = N->getValueType(0).getVectorElementType();
346 SDValue Op = N->getOperand(0);
347 EVT OpVT = Op.getValueType();
348 SDLoc DL(N);
349 // The result needs scalarizing, but it's not a given that the source does.
350 // This is a workaround for targets where it's impossible to scalarize the
351 // result of a conversion, because the source type is legal.
352 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
353 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
354 // legal and was not scalarized.
355 // See the similar logic in ScalarizeVecRes_SETCC
356 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
357 Op = GetScalarizedVector(Op);
358 } else {
359 EVT VT = OpVT.getVectorElementType();
360 Op = DAG.getNode(
361 ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
362 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
363 }
364 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
365 }
366
ScalarizeVecRes_InregOp(SDNode * N)367 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
368 EVT EltVT = N->getValueType(0).getVectorElementType();
369 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
370 SDValue LHS = GetScalarizedVector(N->getOperand(0));
371 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
372 LHS, DAG.getValueType(ExtVT));
373 }
374
ScalarizeVecRes_VecInregOp(SDNode * N)375 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
376 SDLoc DL(N);
377 SDValue Op = N->getOperand(0);
378
379 EVT OpVT = Op.getValueType();
380 EVT OpEltVT = OpVT.getVectorElementType();
381 EVT EltVT = N->getValueType(0).getVectorElementType();
382
383 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
384 Op = GetScalarizedVector(Op);
385 } else {
386 Op = DAG.getNode(
387 ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
388 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
389 }
390
391 switch (N->getOpcode()) {
392 case ISD::ANY_EXTEND_VECTOR_INREG:
393 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
394 case ISD::SIGN_EXTEND_VECTOR_INREG:
395 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
396 case ISD::ZERO_EXTEND_VECTOR_INREG:
397 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
398 }
399
400 llvm_unreachable("Illegal extend_vector_inreg opcode");
401 }
402
ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode * N)403 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
404 // If the operand is wider than the vector element type then it is implicitly
405 // truncated. Make that explicit here.
406 EVT EltVT = N->getValueType(0).getVectorElementType();
407 SDValue InOp = N->getOperand(0);
408 if (InOp.getValueType() != EltVT)
409 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
410 return InOp;
411 }
412
ScalarizeVecRes_VSELECT(SDNode * N)413 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
414 SDValue Cond = N->getOperand(0);
415 EVT OpVT = Cond.getValueType();
416 SDLoc DL(N);
417 // The vselect result and true/value operands needs scalarizing, but it's
418 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
419 // See the similar logic in ScalarizeVecRes_SETCC
420 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
421 Cond = GetScalarizedVector(Cond);
422 } else {
423 EVT VT = OpVT.getVectorElementType();
424 Cond = DAG.getNode(
425 ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
426 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
427 }
428
429 SDValue LHS = GetScalarizedVector(N->getOperand(1));
430 TargetLowering::BooleanContent ScalarBool =
431 TLI.getBooleanContents(false, false);
432 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
433
434 // If integer and float booleans have different contents then we can't
435 // reliably optimize in all cases. There is a full explanation for this in
436 // DAGCombiner::visitSELECT() where the same issue affects folding
437 // (select C, 0, 1) to (xor C, 1).
438 if (TLI.getBooleanContents(false, false) !=
439 TLI.getBooleanContents(false, true)) {
440 // At least try the common case where the boolean is generated by a
441 // comparison.
442 if (Cond->getOpcode() == ISD::SETCC) {
443 EVT OpVT = Cond->getOperand(0).getValueType();
444 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
445 VecBool = TLI.getBooleanContents(OpVT);
446 } else
447 ScalarBool = TargetLowering::UndefinedBooleanContent;
448 }
449
450 EVT CondVT = Cond.getValueType();
451 if (ScalarBool != VecBool) {
452 switch (ScalarBool) {
453 case TargetLowering::UndefinedBooleanContent:
454 break;
455 case TargetLowering::ZeroOrOneBooleanContent:
456 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
457 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
458 // Vector read from all ones, scalar expects a single 1 so mask.
459 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
460 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
461 break;
462 case TargetLowering::ZeroOrNegativeOneBooleanContent:
463 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
464 VecBool == TargetLowering::ZeroOrOneBooleanContent);
465 // Vector reads from a one, scalar from all ones so sign extend.
466 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
467 Cond, DAG.getValueType(MVT::i1));
468 break;
469 }
470 }
471
472 // Truncate the condition if needed
473 auto BoolVT = getSetCCResultType(CondVT);
474 if (BoolVT.bitsLT(CondVT))
475 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
476
477 return DAG.getSelect(SDLoc(N),
478 LHS.getValueType(), Cond, LHS,
479 GetScalarizedVector(N->getOperand(2)));
480 }
481
ScalarizeVecRes_SELECT(SDNode * N)482 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
483 SDValue LHS = GetScalarizedVector(N->getOperand(1));
484 return DAG.getSelect(SDLoc(N),
485 LHS.getValueType(), N->getOperand(0), LHS,
486 GetScalarizedVector(N->getOperand(2)));
487 }
488
ScalarizeVecRes_SELECT_CC(SDNode * N)489 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
490 SDValue LHS = GetScalarizedVector(N->getOperand(2));
491 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
492 N->getOperand(0), N->getOperand(1),
493 LHS, GetScalarizedVector(N->getOperand(3)),
494 N->getOperand(4));
495 }
496
ScalarizeVecRes_UNDEF(SDNode * N)497 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
498 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
499 }
500
ScalarizeVecRes_VECTOR_SHUFFLE(SDNode * N)501 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
502 // Figure out if the scalar is the LHS or RHS and return it.
503 SDValue Arg = N->getOperand(2).getOperand(0);
504 if (Arg.isUndef())
505 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
506 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
507 return GetScalarizedVector(N->getOperand(Op));
508 }
509
ScalarizeVecRes_SETCC(SDNode * N)510 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
511 assert(N->getValueType(0).isVector() &&
512 N->getOperand(0).getValueType().isVector() &&
513 "Operand types must be vectors");
514 SDValue LHS = N->getOperand(0);
515 SDValue RHS = N->getOperand(1);
516 EVT OpVT = LHS.getValueType();
517 EVT NVT = N->getValueType(0).getVectorElementType();
518 SDLoc DL(N);
519
520 // The result needs scalarizing, but it's not a given that the source does.
521 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
522 LHS = GetScalarizedVector(LHS);
523 RHS = GetScalarizedVector(RHS);
524 } else {
525 EVT VT = OpVT.getVectorElementType();
526 LHS = DAG.getNode(
527 ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
528 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
529 RHS = DAG.getNode(
530 ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
531 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
532 }
533
534 // Turn it into a scalar SETCC.
535 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
536 N->getOperand(2));
537 // Vectors may have a different boolean contents to scalars. Promote the
538 // value appropriately.
539 ISD::NodeType ExtendCode =
540 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
541 return DAG.getNode(ExtendCode, DL, NVT, Res);
542 }
543
544
545 //===----------------------------------------------------------------------===//
546 // Operand Vector Scalarization <1 x ty> -> ty.
547 //===----------------------------------------------------------------------===//
548
ScalarizeVectorOperand(SDNode * N,unsigned OpNo)549 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
550 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
551 dbgs() << "\n");
552 SDValue Res = SDValue();
553
554 if (!Res.getNode()) {
555 switch (N->getOpcode()) {
556 default:
557 #ifndef NDEBUG
558 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
559 N->dump(&DAG);
560 dbgs() << "\n";
561 #endif
562 report_fatal_error("Do not know how to scalarize this operator's "
563 "operand!\n");
564 case ISD::BITCAST:
565 Res = ScalarizeVecOp_BITCAST(N);
566 break;
567 case ISD::ANY_EXTEND:
568 case ISD::ZERO_EXTEND:
569 case ISD::SIGN_EXTEND:
570 case ISD::TRUNCATE:
571 case ISD::FP_TO_SINT:
572 case ISD::FP_TO_UINT:
573 case ISD::SINT_TO_FP:
574 case ISD::UINT_TO_FP:
575 Res = ScalarizeVecOp_UnaryOp(N);
576 break;
577 case ISD::STRICT_SINT_TO_FP:
578 case ISD::STRICT_UINT_TO_FP:
579 case ISD::STRICT_FP_TO_SINT:
580 case ISD::STRICT_FP_TO_UINT:
581 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
582 break;
583 case ISD::CONCAT_VECTORS:
584 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
585 break;
586 case ISD::EXTRACT_VECTOR_ELT:
587 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
588 break;
589 case ISD::VSELECT:
590 Res = ScalarizeVecOp_VSELECT(N);
591 break;
592 case ISD::SETCC:
593 Res = ScalarizeVecOp_VSETCC(N);
594 break;
595 case ISD::STORE:
596 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
597 break;
598 case ISD::STRICT_FP_ROUND:
599 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
600 break;
601 case ISD::FP_ROUND:
602 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
603 break;
604 case ISD::VECREDUCE_FADD:
605 case ISD::VECREDUCE_FMUL:
606 case ISD::VECREDUCE_ADD:
607 case ISD::VECREDUCE_MUL:
608 case ISD::VECREDUCE_AND:
609 case ISD::VECREDUCE_OR:
610 case ISD::VECREDUCE_XOR:
611 case ISD::VECREDUCE_SMAX:
612 case ISD::VECREDUCE_SMIN:
613 case ISD::VECREDUCE_UMAX:
614 case ISD::VECREDUCE_UMIN:
615 case ISD::VECREDUCE_FMAX:
616 case ISD::VECREDUCE_FMIN:
617 Res = ScalarizeVecOp_VECREDUCE(N);
618 break;
619 }
620 }
621
622 // If the result is null, the sub-method took care of registering results etc.
623 if (!Res.getNode()) return false;
624
625 // If the result is N, the sub-method updated N in place. Tell the legalizer
626 // core about this.
627 if (Res.getNode() == N)
628 return true;
629
630 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
631 "Invalid operand expansion");
632
633 ReplaceValueWith(SDValue(N, 0), Res);
634 return false;
635 }
636
637 /// If the value to convert is a vector that needs to be scalarized, it must be
638 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_BITCAST(SDNode * N)639 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
640 SDValue Elt = GetScalarizedVector(N->getOperand(0));
641 return DAG.getNode(ISD::BITCAST, SDLoc(N),
642 N->getValueType(0), Elt);
643 }
644
645 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
646 /// Do the operation on the element instead.
ScalarizeVecOp_UnaryOp(SDNode * N)647 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
648 assert(N->getValueType(0).getVectorNumElements() == 1 &&
649 "Unexpected vector type!");
650 SDValue Elt = GetScalarizedVector(N->getOperand(0));
651 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
652 N->getValueType(0).getScalarType(), Elt);
653 // Revectorize the result so the types line up with what the uses of this
654 // expression expect.
655 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
656 }
657
658 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
659 /// Do the strict FP operation on the element instead.
ScalarizeVecOp_UnaryOp_StrictFP(SDNode * N)660 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
661 assert(N->getValueType(0).getVectorNumElements() == 1 &&
662 "Unexpected vector type!");
663 SDValue Elt = GetScalarizedVector(N->getOperand(1));
664 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
665 { N->getValueType(0).getScalarType(), MVT::Other },
666 { N->getOperand(0), Elt });
667 // Legalize the chain result - switch anything that used the old chain to
668 // use the new one.
669 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
670 // Revectorize the result so the types line up with what the uses of this
671 // expression expect.
672 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
673
674 // Do our own replacement and return SDValue() to tell the caller that we
675 // handled all replacements since caller can only handle a single result.
676 ReplaceValueWith(SDValue(N, 0), Res);
677 return SDValue();
678 }
679
680 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
ScalarizeVecOp_CONCAT_VECTORS(SDNode * N)681 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
682 SmallVector<SDValue, 8> Ops(N->getNumOperands());
683 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
684 Ops[i] = GetScalarizedVector(N->getOperand(i));
685 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
686 }
687
688 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
689 /// so just return the element, ignoring the index.
ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode * N)690 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
691 EVT VT = N->getValueType(0);
692 SDValue Res = GetScalarizedVector(N->getOperand(0));
693 if (Res.getValueType() != VT)
694 Res = VT.isFloatingPoint()
695 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
696 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
697 return Res;
698 }
699
700 /// If the input condition is a vector that needs to be scalarized, it must be
701 /// <1 x i1>, so just convert to a normal ISD::SELECT
702 /// (still with vector output type since that was acceptable if we got here).
ScalarizeVecOp_VSELECT(SDNode * N)703 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
704 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
705 EVT VT = N->getValueType(0);
706
707 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
708 N->getOperand(2));
709 }
710
711 /// If the operand is a vector that needs to be scalarized then the
712 /// result must be v1i1, so just convert to a scalar SETCC and wrap
713 /// with a scalar_to_vector since the res type is legal if we got here
ScalarizeVecOp_VSETCC(SDNode * N)714 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
715 assert(N->getValueType(0).isVector() &&
716 N->getOperand(0).getValueType().isVector() &&
717 "Operand types must be vectors");
718 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
719
720 EVT VT = N->getValueType(0);
721 SDValue LHS = GetScalarizedVector(N->getOperand(0));
722 SDValue RHS = GetScalarizedVector(N->getOperand(1));
723
724 EVT OpVT = N->getOperand(0).getValueType();
725 EVT NVT = VT.getVectorElementType();
726 SDLoc DL(N);
727 // Turn it into a scalar SETCC.
728 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
729 N->getOperand(2));
730
731 // Vectors may have a different boolean contents to scalars. Promote the
732 // value appropriately.
733 ISD::NodeType ExtendCode =
734 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
735
736 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
737
738 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
739 }
740
741 /// If the value to store is a vector that needs to be scalarized, it must be
742 /// <1 x ty>. Just store the element.
ScalarizeVecOp_STORE(StoreSDNode * N,unsigned OpNo)743 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
744 assert(N->isUnindexed() && "Indexed store of one-element vector?");
745 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
746 SDLoc dl(N);
747
748 if (N->isTruncatingStore())
749 return DAG.getTruncStore(
750 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
751 N->getBasePtr(), N->getPointerInfo(),
752 N->getMemoryVT().getVectorElementType(), N->getAlignment(),
753 N->getMemOperand()->getFlags(), N->getAAInfo());
754
755 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
756 N->getBasePtr(), N->getPointerInfo(),
757 N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
758 N->getAAInfo());
759 }
760
761 /// If the value to round is a vector that needs to be scalarized, it must be
762 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_ROUND(SDNode * N,unsigned OpNo)763 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
764 SDValue Elt = GetScalarizedVector(N->getOperand(0));
765 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
766 N->getValueType(0).getVectorElementType(), Elt,
767 N->getOperand(1));
768 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
769 }
770
ScalarizeVecOp_STRICT_FP_ROUND(SDNode * N,unsigned OpNo)771 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
772 unsigned OpNo) {
773 assert(OpNo == 1 && "Wrong operand for scalarization!");
774 SDValue Elt = GetScalarizedVector(N->getOperand(1));
775 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
776 { N->getValueType(0).getVectorElementType(),
777 MVT::Other },
778 { N->getOperand(0), Elt, N->getOperand(2) });
779 // Legalize the chain result - switch anything that used the old chain to
780 // use the new one.
781 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
782
783 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
784
785 // Do our own replacement and return SDValue() to tell the caller that we
786 // handled all replacements since caller can only handle a single result.
787 ReplaceValueWith(SDValue(N, 0), Res);
788 return SDValue();
789 }
790
ScalarizeVecOp_VECREDUCE(SDNode * N)791 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
792 SDValue Res = GetScalarizedVector(N->getOperand(0));
793 // Result type may be wider than element type.
794 if (Res.getValueType() != N->getValueType(0))
795 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
796 return Res;
797 }
798
799 //===----------------------------------------------------------------------===//
800 // Result Vector Splitting
801 //===----------------------------------------------------------------------===//
802
803 /// This method is called when the specified result of the specified node is
804 /// found to need vector splitting. At this point, the node may also have
805 /// invalid operands or may have other results that need legalization, we just
806 /// know that (at least) one result needs vector splitting.
SplitVectorResult(SDNode * N,unsigned ResNo)807 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
808 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
809 SDValue Lo, Hi;
810
811 // See if the target wants to custom expand this node.
812 if (CustomLowerNode(N, N->getValueType(ResNo), true))
813 return;
814
815 switch (N->getOpcode()) {
816 default:
817 #ifndef NDEBUG
818 dbgs() << "SplitVectorResult #" << ResNo << ": ";
819 N->dump(&DAG);
820 dbgs() << "\n";
821 #endif
822 report_fatal_error("Do not know how to split the result of this "
823 "operator!\n");
824
825 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
826 case ISD::VSELECT:
827 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
828 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
829 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
830 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
831 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
832 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
833 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
834 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
835 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
836 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
837 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
838 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
839 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
840 case ISD::LOAD:
841 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
842 break;
843 case ISD::MLOAD:
844 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
845 break;
846 case ISD::MGATHER:
847 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
848 break;
849 case ISD::SETCC:
850 SplitVecRes_SETCC(N, Lo, Hi);
851 break;
852 case ISD::VECTOR_SHUFFLE:
853 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
854 break;
855 case ISD::VAARG:
856 SplitVecRes_VAARG(N, Lo, Hi);
857 break;
858
859 case ISD::ANY_EXTEND_VECTOR_INREG:
860 case ISD::SIGN_EXTEND_VECTOR_INREG:
861 case ISD::ZERO_EXTEND_VECTOR_INREG:
862 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
863 break;
864
865 case ISD::ABS:
866 case ISD::BITREVERSE:
867 case ISD::BSWAP:
868 case ISD::CTLZ:
869 case ISD::CTTZ:
870 case ISD::CTLZ_ZERO_UNDEF:
871 case ISD::CTTZ_ZERO_UNDEF:
872 case ISD::CTPOP:
873 case ISD::FABS:
874 case ISD::FCEIL:
875 case ISD::FCOS:
876 case ISD::FEXP:
877 case ISD::FEXP2:
878 case ISD::FFLOOR:
879 case ISD::FLOG:
880 case ISD::FLOG10:
881 case ISD::FLOG2:
882 case ISD::FNEARBYINT:
883 case ISD::FNEG:
884 case ISD::FP_EXTEND:
885 case ISD::FP_ROUND:
886 case ISD::FP_TO_SINT:
887 case ISD::FP_TO_UINT:
888 case ISD::FRINT:
889 case ISD::FROUND:
890 case ISD::FSIN:
891 case ISD::FSQRT:
892 case ISD::FTRUNC:
893 case ISD::SINT_TO_FP:
894 case ISD::TRUNCATE:
895 case ISD::UINT_TO_FP:
896 case ISD::FCANONICALIZE:
897 SplitVecRes_UnaryOp(N, Lo, Hi);
898 break;
899
900 case ISD::ANY_EXTEND:
901 case ISD::SIGN_EXTEND:
902 case ISD::ZERO_EXTEND:
903 SplitVecRes_ExtendOp(N, Lo, Hi);
904 break;
905
906 case ISD::ADD:
907 case ISD::SUB:
908 case ISD::MUL:
909 case ISD::MULHS:
910 case ISD::MULHU:
911 case ISD::FADD:
912 case ISD::FSUB:
913 case ISD::FMUL:
914 case ISD::FMINNUM:
915 case ISD::FMAXNUM:
916 case ISD::FMINIMUM:
917 case ISD::FMAXIMUM:
918 case ISD::SDIV:
919 case ISD::UDIV:
920 case ISD::FDIV:
921 case ISD::FPOW:
922 case ISD::AND:
923 case ISD::OR:
924 case ISD::XOR:
925 case ISD::SHL:
926 case ISD::SRA:
927 case ISD::SRL:
928 case ISD::UREM:
929 case ISD::SREM:
930 case ISD::FREM:
931 case ISD::SMIN:
932 case ISD::SMAX:
933 case ISD::UMIN:
934 case ISD::UMAX:
935 case ISD::SADDSAT:
936 case ISD::UADDSAT:
937 case ISD::SSUBSAT:
938 case ISD::USUBSAT:
939 SplitVecRes_BinOp(N, Lo, Hi);
940 break;
941 case ISD::FMA:
942 SplitVecRes_TernaryOp(N, Lo, Hi);
943 break;
944
945 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
946 case ISD::STRICT_##DAGN:
947 #include "llvm/IR/ConstrainedOps.def"
948 SplitVecRes_StrictFPOp(N, Lo, Hi);
949 break;
950
951 case ISD::UADDO:
952 case ISD::SADDO:
953 case ISD::USUBO:
954 case ISD::SSUBO:
955 case ISD::UMULO:
956 case ISD::SMULO:
957 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
958 break;
959 case ISD::SMULFIX:
960 case ISD::SMULFIXSAT:
961 case ISD::UMULFIX:
962 case ISD::UMULFIXSAT:
963 case ISD::SDIVFIX:
964 case ISD::UDIVFIX:
965 SplitVecRes_FIX(N, Lo, Hi);
966 break;
967 }
968
969 // If Lo/Hi is null, the sub-method took care of registering results etc.
970 if (Lo.getNode())
971 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
972 }
973
SplitVecRes_BinOp(SDNode * N,SDValue & Lo,SDValue & Hi)974 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
975 SDValue &Hi) {
976 SDValue LHSLo, LHSHi;
977 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
978 SDValue RHSLo, RHSHi;
979 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
980 SDLoc dl(N);
981
982 const SDNodeFlags Flags = N->getFlags();
983 unsigned Opcode = N->getOpcode();
984 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
985 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
986 }
987
SplitVecRes_TernaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)988 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
989 SDValue &Hi) {
990 SDValue Op0Lo, Op0Hi;
991 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
992 SDValue Op1Lo, Op1Hi;
993 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
994 SDValue Op2Lo, Op2Hi;
995 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
996 SDLoc dl(N);
997
998 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
999 Op0Lo, Op1Lo, Op2Lo);
1000 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
1001 Op0Hi, Op1Hi, Op2Hi);
1002 }
1003
SplitVecRes_FIX(SDNode * N,SDValue & Lo,SDValue & Hi)1004 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1005 SDValue LHSLo, LHSHi;
1006 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1007 SDValue RHSLo, RHSHi;
1008 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1009 SDLoc dl(N);
1010 SDValue Op2 = N->getOperand(2);
1011
1012 unsigned Opcode = N->getOpcode();
1013 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2);
1014 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2);
1015 }
1016
SplitVecRes_BITCAST(SDNode * N,SDValue & Lo,SDValue & Hi)1017 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1018 SDValue &Hi) {
1019 // We know the result is a vector. The input may be either a vector or a
1020 // scalar value.
1021 EVT LoVT, HiVT;
1022 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1023 SDLoc dl(N);
1024
1025 SDValue InOp = N->getOperand(0);
1026 EVT InVT = InOp.getValueType();
1027
1028 // Handle some special cases efficiently.
1029 switch (getTypeAction(InVT)) {
1030 case TargetLowering::TypeLegal:
1031 case TargetLowering::TypePromoteInteger:
1032 case TargetLowering::TypePromoteFloat:
1033 case TargetLowering::TypeSoftenFloat:
1034 case TargetLowering::TypeScalarizeVector:
1035 case TargetLowering::TypeWidenVector:
1036 break;
1037 case TargetLowering::TypeExpandInteger:
1038 case TargetLowering::TypeExpandFloat:
1039 // A scalar to vector conversion, where the scalar needs expansion.
1040 // If the vector is being split in two then we can just convert the
1041 // expanded pieces.
1042 if (LoVT == HiVT) {
1043 GetExpandedOp(InOp, Lo, Hi);
1044 if (DAG.getDataLayout().isBigEndian())
1045 std::swap(Lo, Hi);
1046 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1047 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1048 return;
1049 }
1050 break;
1051 case TargetLowering::TypeSplitVector:
1052 // If the input is a vector that needs to be split, convert each split
1053 // piece of the input now.
1054 GetSplitVector(InOp, Lo, Hi);
1055 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1056 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1057 return;
1058 }
1059
1060 // In the general case, convert the input to an integer and split it by hand.
1061 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1062 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1063 if (DAG.getDataLayout().isBigEndian())
1064 std::swap(LoIntVT, HiIntVT);
1065
1066 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1067
1068 if (DAG.getDataLayout().isBigEndian())
1069 std::swap(Lo, Hi);
1070 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1071 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1072 }
1073
SplitVecRes_BUILD_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1074 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1075 SDValue &Hi) {
1076 EVT LoVT, HiVT;
1077 SDLoc dl(N);
1078 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1079 unsigned LoNumElts = LoVT.getVectorNumElements();
1080 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1081 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1082
1083 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1084 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1085 }
1086
SplitVecRes_CONCAT_VECTORS(SDNode * N,SDValue & Lo,SDValue & Hi)1087 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1088 SDValue &Hi) {
1089 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1090 SDLoc dl(N);
1091 unsigned NumSubvectors = N->getNumOperands() / 2;
1092 if (NumSubvectors == 1) {
1093 Lo = N->getOperand(0);
1094 Hi = N->getOperand(1);
1095 return;
1096 }
1097
1098 EVT LoVT, HiVT;
1099 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1100
1101 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1102 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1103
1104 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1105 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1106 }
1107
SplitVecRes_EXTRACT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1108 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1109 SDValue &Hi) {
1110 SDValue Vec = N->getOperand(0);
1111 SDValue Idx = N->getOperand(1);
1112 SDLoc dl(N);
1113
1114 EVT LoVT, HiVT;
1115 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1116
1117 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1118 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1119 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1120 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
1121 TLI.getVectorIdxTy(DAG.getDataLayout())));
1122 }
1123
SplitVecRes_INSERT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1124 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1125 SDValue &Hi) {
1126 SDValue Vec = N->getOperand(0);
1127 SDValue SubVec = N->getOperand(1);
1128 SDValue Idx = N->getOperand(2);
1129 SDLoc dl(N);
1130 GetSplitVector(Vec, Lo, Hi);
1131
1132 EVT VecVT = Vec.getValueType();
1133 unsigned VecElems = VecVT.getVectorNumElements();
1134 unsigned SubElems = SubVec.getValueType().getVectorNumElements();
1135
1136 // If we know the index is 0, and we know the subvector doesn't cross the
1137 // boundary between the halves, we can avoid spilling the vector, and insert
1138 // into the lower half of the split vector directly.
1139 // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
1140 // the index is constant and there is no boundary crossing. But those cases
1141 // don't seem to get hit in practice.
1142 if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
1143 unsigned IdxVal = ConstIdx->getZExtValue();
1144 if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
1145 EVT LoVT, HiVT;
1146 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1147 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1148 return;
1149 }
1150 }
1151
1152 // Spill the vector to the stack.
1153 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1154 SDValue Store =
1155 DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1156
1157 // Store the new subvector into the specified index.
1158 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1159 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1160 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1161 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
1162
1163 // Load the Lo part from the stack slot.
1164 Lo =
1165 DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1166
1167 // Increment the pointer to the other part.
1168 unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1169 StackPtr = DAG.getMemBasePlusOffset(StackPtr, IncrementSize, dl);
1170
1171 // Load the Hi part from the stack slot.
1172 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1173 MinAlign(Alignment, IncrementSize));
1174 }
1175
SplitVecRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)1176 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1177 SDValue &Hi) {
1178 SDLoc dl(N);
1179 GetSplitVector(N->getOperand(0), Lo, Hi);
1180 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1181 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1182 }
1183
SplitVecRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1184 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1185 SDValue &Hi) {
1186 SDValue LHSLo, LHSHi;
1187 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1188 SDLoc DL(N);
1189
1190 SDValue RHSLo, RHSHi;
1191 SDValue RHS = N->getOperand(1);
1192 EVT RHSVT = RHS.getValueType();
1193 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1194 GetSplitVector(RHS, RHSLo, RHSHi);
1195 else
1196 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1197
1198
1199 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1200 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1201 }
1202
SplitVecRes_InregOp(SDNode * N,SDValue & Lo,SDValue & Hi)1203 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1204 SDValue &Hi) {
1205 SDValue LHSLo, LHSHi;
1206 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1207 SDLoc dl(N);
1208
1209 EVT LoVT, HiVT;
1210 std::tie(LoVT, HiVT) =
1211 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1212
1213 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1214 DAG.getValueType(LoVT));
1215 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1216 DAG.getValueType(HiVT));
1217 }
1218
SplitVecRes_ExtVecInRegOp(SDNode * N,SDValue & Lo,SDValue & Hi)1219 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1220 SDValue &Hi) {
1221 unsigned Opcode = N->getOpcode();
1222 SDValue N0 = N->getOperand(0);
1223
1224 SDLoc dl(N);
1225 SDValue InLo, InHi;
1226
1227 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1228 GetSplitVector(N0, InLo, InHi);
1229 else
1230 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1231
1232 EVT InLoVT = InLo.getValueType();
1233 unsigned InNumElements = InLoVT.getVectorNumElements();
1234
1235 EVT OutLoVT, OutHiVT;
1236 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1237 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1238 assert((2 * OutNumElements) <= InNumElements &&
1239 "Illegal extend vector in reg split");
1240
1241 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1242 // input vector (i.e. we only use InLo):
1243 // OutLo will extend the first OutNumElements from InLo.
1244 // OutHi will extend the next OutNumElements from InLo.
1245
1246 // Shuffle the elements from InLo for OutHi into the bottom elements to
1247 // create a 'fake' InHi.
1248 SmallVector<int, 8> SplitHi(InNumElements, -1);
1249 for (unsigned i = 0; i != OutNumElements; ++i)
1250 SplitHi[i] = i + OutNumElements;
1251 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1252
1253 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1254 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1255 }
1256
SplitVecRes_StrictFPOp(SDNode * N,SDValue & Lo,SDValue & Hi)1257 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1258 SDValue &Hi) {
1259 unsigned NumOps = N->getNumOperands();
1260 SDValue Chain = N->getOperand(0);
1261 EVT LoVT, HiVT;
1262 SDLoc dl(N);
1263 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1264
1265 SmallVector<SDValue, 4> OpsLo(NumOps);
1266 SmallVector<SDValue, 4> OpsHi(NumOps);
1267
1268 // The Chain is the first operand.
1269 OpsLo[0] = Chain;
1270 OpsHi[0] = Chain;
1271
1272 // Now process the remaining operands.
1273 for (unsigned i = 1; i < NumOps; ++i) {
1274 SDValue Op = N->getOperand(i);
1275 SDValue OpLo = Op;
1276 SDValue OpHi = Op;
1277
1278 EVT InVT = Op.getValueType();
1279 if (InVT.isVector()) {
1280 // If the input also splits, handle it directly for a
1281 // compile time speedup. Otherwise split it by hand.
1282 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1283 GetSplitVector(Op, OpLo, OpHi);
1284 else
1285 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1286 }
1287
1288 OpsLo[i] = OpLo;
1289 OpsHi[i] = OpHi;
1290 }
1291
1292 EVT LoValueVTs[] = {LoVT, MVT::Other};
1293 EVT HiValueVTs[] = {HiVT, MVT::Other};
1294 Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
1295 Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
1296
1297 // Build a factor node to remember that this Op is independent of the
1298 // other one.
1299 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1300 Lo.getValue(1), Hi.getValue(1));
1301
1302 // Legalize the chain result - switch anything that used the old chain to
1303 // use the new one.
1304 ReplaceValueWith(SDValue(N, 1), Chain);
1305 }
1306
UnrollVectorOp_StrictFP(SDNode * N,unsigned ResNE)1307 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1308 SDValue Chain = N->getOperand(0);
1309 EVT VT = N->getValueType(0);
1310 unsigned NE = VT.getVectorNumElements();
1311 EVT EltVT = VT.getVectorElementType();
1312 SDLoc dl(N);
1313
1314 SmallVector<SDValue, 8> Scalars;
1315 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1316
1317 // If ResNE is 0, fully unroll the vector op.
1318 if (ResNE == 0)
1319 ResNE = NE;
1320 else if (NE > ResNE)
1321 NE = ResNE;
1322
1323 //The results of each unrolled operation, including the chain.
1324 EVT ChainVTs[] = {EltVT, MVT::Other};
1325 SmallVector<SDValue, 8> Chains;
1326
1327 unsigned i;
1328 for (i = 0; i != NE; ++i) {
1329 Operands[0] = Chain;
1330 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1331 SDValue Operand = N->getOperand(j);
1332 EVT OperandVT = Operand.getValueType();
1333 if (OperandVT.isVector()) {
1334 EVT OperandEltVT = OperandVT.getVectorElementType();
1335 Operands[j] =
1336 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
1337 DAG.getConstant(i, dl, TLI.getVectorIdxTy(
1338 DAG.getDataLayout())));
1339 } else {
1340 Operands[j] = Operand;
1341 }
1342 }
1343 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1344 Scalar.getNode()->setFlags(N->getFlags());
1345
1346 //Add in the scalar as well as its chain value to the
1347 //result vectors.
1348 Scalars.push_back(Scalar);
1349 Chains.push_back(Scalar.getValue(1));
1350 }
1351
1352 for (; i < ResNE; ++i)
1353 Scalars.push_back(DAG.getUNDEF(EltVT));
1354
1355 // Build a new factor node to connect the chain back together.
1356 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1357 ReplaceValueWith(SDValue(N, 1), Chain);
1358
1359 // Create a new BUILD_VECTOR node
1360 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1361 return DAG.getBuildVector(VecVT, dl, Scalars);
1362 }
1363
SplitVecRes_OverflowOp(SDNode * N,unsigned ResNo,SDValue & Lo,SDValue & Hi)1364 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1365 SDValue &Lo, SDValue &Hi) {
1366 SDLoc dl(N);
1367 EVT ResVT = N->getValueType(0);
1368 EVT OvVT = N->getValueType(1);
1369 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1370 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1371 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1372
1373 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1374 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1375 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1376 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1377 } else {
1378 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1379 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1380 }
1381
1382 unsigned Opcode = N->getOpcode();
1383 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1384 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1385 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1386 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1387
1388 Lo = SDValue(LoNode, ResNo);
1389 Hi = SDValue(HiNode, ResNo);
1390
1391 // Replace the other vector result not being explicitly split here.
1392 unsigned OtherNo = 1 - ResNo;
1393 EVT OtherVT = N->getValueType(OtherNo);
1394 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1395 SetSplitVector(SDValue(N, OtherNo),
1396 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1397 } else {
1398 SDValue OtherVal = DAG.getNode(
1399 ISD::CONCAT_VECTORS, dl, OtherVT,
1400 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1401 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1402 }
1403 }
1404
SplitVecRes_INSERT_VECTOR_ELT(SDNode * N,SDValue & Lo,SDValue & Hi)1405 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1406 SDValue &Hi) {
1407 SDValue Vec = N->getOperand(0);
1408 SDValue Elt = N->getOperand(1);
1409 SDValue Idx = N->getOperand(2);
1410 SDLoc dl(N);
1411 GetSplitVector(Vec, Lo, Hi);
1412
1413 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1414 unsigned IdxVal = CIdx->getZExtValue();
1415 unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1416 if (IdxVal < LoNumElts)
1417 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1418 Lo.getValueType(), Lo, Elt, Idx);
1419 else
1420 Hi =
1421 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1422 DAG.getConstant(IdxVal - LoNumElts, dl,
1423 TLI.getVectorIdxTy(DAG.getDataLayout())));
1424 return;
1425 }
1426
1427 // See if the target wants to custom expand this node.
1428 if (CustomLowerNode(N, N->getValueType(0), true))
1429 return;
1430
1431 // Make the vector elements byte-addressable if they aren't already.
1432 EVT VecVT = Vec.getValueType();
1433 EVT EltVT = VecVT.getVectorElementType();
1434 if (VecVT.getScalarSizeInBits() < 8) {
1435 EltVT = MVT::i8;
1436 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1437 VecVT.getVectorNumElements());
1438 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1439 // Extend the element type to match if needed.
1440 if (EltVT.bitsGT(Elt.getValueType()))
1441 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1442 }
1443
1444 // Spill the vector to the stack.
1445 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1446 auto &MF = DAG.getMachineFunction();
1447 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1448 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1449 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1450
1451 // Store the new element. This may be larger than the vector element type,
1452 // so use a truncating store.
1453 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1454 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1455 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1456 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1457 MachinePointerInfo::getUnknownStack(MF), EltVT);
1458
1459 EVT LoVT, HiVT;
1460 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1461
1462 // Load the Lo part from the stack slot.
1463 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1464
1465 // Increment the pointer to the other part.
1466 unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1467 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1468 DAG.getConstant(IncrementSize, dl,
1469 StackPtr.getValueType()));
1470
1471 // Load the Hi part from the stack slot.
1472 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1473 PtrInfo.getWithOffset(IncrementSize),
1474 MinAlign(Alignment, IncrementSize));
1475
1476 // If we adjusted the original type, we need to truncate the results.
1477 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1478 if (LoVT != Lo.getValueType())
1479 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1480 if (HiVT != Hi.getValueType())
1481 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1482 }
1483
SplitVecRes_SCALAR_TO_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1484 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1485 SDValue &Hi) {
1486 EVT LoVT, HiVT;
1487 SDLoc dl(N);
1488 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1489 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1490 Hi = DAG.getUNDEF(HiVT);
1491 }
1492
SplitVecRes_LOAD(LoadSDNode * LD,SDValue & Lo,SDValue & Hi)1493 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1494 SDValue &Hi) {
1495 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1496 EVT LoVT, HiVT;
1497 SDLoc dl(LD);
1498 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1499
1500 ISD::LoadExtType ExtType = LD->getExtensionType();
1501 SDValue Ch = LD->getChain();
1502 SDValue Ptr = LD->getBasePtr();
1503 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1504 EVT MemoryVT = LD->getMemoryVT();
1505 unsigned Alignment = LD->getOriginalAlignment();
1506 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1507 AAMDNodes AAInfo = LD->getAAInfo();
1508
1509 EVT LoMemVT, HiMemVT;
1510 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1511
1512 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1513 LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1514
1515 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1516 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1517 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1518 LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1519 Alignment, MMOFlags, AAInfo);
1520
1521 // Build a factor node to remember that this load is independent of the
1522 // other one.
1523 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1524 Hi.getValue(1));
1525
1526 // Legalize the chain result - switch anything that used the old chain to
1527 // use the new one.
1528 ReplaceValueWith(SDValue(LD, 1), Ch);
1529 }
1530
SplitVecRes_MLOAD(MaskedLoadSDNode * MLD,SDValue & Lo,SDValue & Hi)1531 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1532 SDValue &Lo, SDValue &Hi) {
1533 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1534 EVT LoVT, HiVT;
1535 SDLoc dl(MLD);
1536 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1537
1538 SDValue Ch = MLD->getChain();
1539 SDValue Ptr = MLD->getBasePtr();
1540 SDValue Offset = MLD->getOffset();
1541 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1542 SDValue Mask = MLD->getMask();
1543 SDValue PassThru = MLD->getPassThru();
1544 unsigned Alignment = MLD->getOriginalAlignment();
1545 ISD::LoadExtType ExtType = MLD->getExtensionType();
1546
1547 // Split Mask operand
1548 SDValue MaskLo, MaskHi;
1549 if (Mask.getOpcode() == ISD::SETCC) {
1550 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1551 } else {
1552 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1553 GetSplitVector(Mask, MaskLo, MaskHi);
1554 else
1555 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1556 }
1557
1558 EVT MemoryVT = MLD->getMemoryVT();
1559 EVT LoMemVT, HiMemVT;
1560 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1561
1562 SDValue PassThruLo, PassThruHi;
1563 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1564 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1565 else
1566 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1567
1568 MachineMemOperand *MMO = DAG.getMachineFunction().
1569 getMachineMemOperand(MLD->getPointerInfo(),
1570 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
1571 Alignment, MLD->getAAInfo(), MLD->getRanges());
1572
1573 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1574 MMO, MLD->getAddressingMode(), ExtType,
1575 MLD->isExpandingLoad());
1576
1577 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1578 MLD->isExpandingLoad());
1579 unsigned HiOffset = LoMemVT.getStoreSize();
1580
1581 MMO = DAG.getMachineFunction().getMachineMemOperand(
1582 MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad,
1583 HiMemVT.getStoreSize(), Alignment, MLD->getAAInfo(),
1584 MLD->getRanges());
1585
1586 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi, HiMemVT,
1587 MMO, MLD->getAddressingMode(), ExtType,
1588 MLD->isExpandingLoad());
1589
1590 // Build a factor node to remember that this load is independent of the
1591 // other one.
1592 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1593 Hi.getValue(1));
1594
1595 // Legalize the chain result - switch anything that used the old chain to
1596 // use the new one.
1597 ReplaceValueWith(SDValue(MLD, 1), Ch);
1598
1599 }
1600
SplitVecRes_MGATHER(MaskedGatherSDNode * MGT,SDValue & Lo,SDValue & Hi)1601 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1602 SDValue &Lo, SDValue &Hi) {
1603 EVT LoVT, HiVT;
1604 SDLoc dl(MGT);
1605 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1606
1607 SDValue Ch = MGT->getChain();
1608 SDValue Ptr = MGT->getBasePtr();
1609 SDValue Mask = MGT->getMask();
1610 SDValue PassThru = MGT->getPassThru();
1611 SDValue Index = MGT->getIndex();
1612 SDValue Scale = MGT->getScale();
1613 unsigned Alignment = MGT->getOriginalAlignment();
1614
1615 // Split Mask operand
1616 SDValue MaskLo, MaskHi;
1617 if (Mask.getOpcode() == ISD::SETCC) {
1618 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1619 } else {
1620 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1621 GetSplitVector(Mask, MaskLo, MaskHi);
1622 else
1623 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1624 }
1625
1626 EVT MemoryVT = MGT->getMemoryVT();
1627 EVT LoMemVT, HiMemVT;
1628 // Split MemoryVT
1629 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1630
1631 SDValue PassThruLo, PassThruHi;
1632 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1633 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1634 else
1635 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1636
1637 SDValue IndexHi, IndexLo;
1638 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1639 GetSplitVector(Index, IndexLo, IndexHi);
1640 else
1641 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1642
1643 MachineMemOperand *MMO = DAG.getMachineFunction().
1644 getMachineMemOperand(MGT->getPointerInfo(),
1645 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
1646 Alignment, MGT->getAAInfo(), MGT->getRanges());
1647
1648 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1649 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1650 MMO, MGT->getIndexType());
1651
1652 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1653 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1654 MMO, MGT->getIndexType());
1655
1656 // Build a factor node to remember that this load is independent of the
1657 // other one.
1658 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1659 Hi.getValue(1));
1660
1661 // Legalize the chain result - switch anything that used the old chain to
1662 // use the new one.
1663 ReplaceValueWith(SDValue(MGT, 1), Ch);
1664 }
1665
1666
SplitVecRes_SETCC(SDNode * N,SDValue & Lo,SDValue & Hi)1667 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1668 assert(N->getValueType(0).isVector() &&
1669 N->getOperand(0).getValueType().isVector() &&
1670 "Operand types must be vectors");
1671
1672 EVT LoVT, HiVT;
1673 SDLoc DL(N);
1674 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1675
1676 // If the input also splits, handle it directly. Otherwise split it by hand.
1677 SDValue LL, LH, RL, RH;
1678 if (getTypeAction(N->getOperand(0).getValueType()) ==
1679 TargetLowering::TypeSplitVector)
1680 GetSplitVector(N->getOperand(0), LL, LH);
1681 else
1682 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1683
1684 if (getTypeAction(N->getOperand(1).getValueType()) ==
1685 TargetLowering::TypeSplitVector)
1686 GetSplitVector(N->getOperand(1), RL, RH);
1687 else
1688 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1689
1690 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1691 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1692 }
1693
SplitVecRes_UnaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1694 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1695 SDValue &Hi) {
1696 // Get the dest types - they may not match the input types, e.g. int_to_fp.
1697 EVT LoVT, HiVT;
1698 SDLoc dl(N);
1699 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1700
1701 // If the input also splits, handle it directly for a compile time speedup.
1702 // Otherwise split it by hand.
1703 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1704 EVT InVT = N->getOperand(OpNo).getValueType();
1705 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1706 GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1707 else
1708 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1709
1710 if (N->getOpcode() == ISD::FP_ROUND) {
1711 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1712 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1713 } else {
1714 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1715 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1716 }
1717 }
1718
SplitVecRes_ExtendOp(SDNode * N,SDValue & Lo,SDValue & Hi)1719 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1720 SDValue &Hi) {
1721 SDLoc dl(N);
1722 EVT SrcVT = N->getOperand(0).getValueType();
1723 EVT DestVT = N->getValueType(0);
1724 EVT LoVT, HiVT;
1725 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1726
1727 // We can do better than a generic split operation if the extend is doing
1728 // more than just doubling the width of the elements and the following are
1729 // true:
1730 // - The number of vector elements is even,
1731 // - the source type is legal,
1732 // - the type of a split source is illegal,
1733 // - the type of an extended (by doubling element size) source is legal, and
1734 // - the type of that extended source when split is legal.
1735 //
1736 // This won't necessarily completely legalize the operation, but it will
1737 // more effectively move in the right direction and prevent falling down
1738 // to scalarization in many cases due to the input vector being split too
1739 // far.
1740 unsigned NumElements = SrcVT.getVectorNumElements();
1741 if ((NumElements & 1) == 0 &&
1742 SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1743 LLVMContext &Ctx = *DAG.getContext();
1744 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1745 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1746
1747 EVT SplitLoVT, SplitHiVT;
1748 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1749 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1750 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1751 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1752 N->dump(&DAG); dbgs() << "\n");
1753 // Extend the source vector by one step.
1754 SDValue NewSrc =
1755 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1756 // Get the low and high halves of the new, extended one step, vector.
1757 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1758 // Extend those vector halves the rest of the way.
1759 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1760 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1761 return;
1762 }
1763 }
1764 // Fall back to the generic unary operator splitting otherwise.
1765 SplitVecRes_UnaryOp(N, Lo, Hi);
1766 }
1767
SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N,SDValue & Lo,SDValue & Hi)1768 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1769 SDValue &Lo, SDValue &Hi) {
1770 // The low and high parts of the original input give four input vectors.
1771 SDValue Inputs[4];
1772 SDLoc dl(N);
1773 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1774 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1775 EVT NewVT = Inputs[0].getValueType();
1776 unsigned NewElts = NewVT.getVectorNumElements();
1777
1778 // If Lo or Hi uses elements from at most two of the four input vectors, then
1779 // express it as a vector shuffle of those two inputs. Otherwise extract the
1780 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1781 SmallVector<int, 16> Ops;
1782 for (unsigned High = 0; High < 2; ++High) {
1783 SDValue &Output = High ? Hi : Lo;
1784
1785 // Build a shuffle mask for the output, discovering on the fly which
1786 // input vectors to use as shuffle operands (recorded in InputUsed).
1787 // If building a suitable shuffle vector proves too hard, then bail
1788 // out with useBuildVector set.
1789 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1790 unsigned FirstMaskIdx = High * NewElts;
1791 bool useBuildVector = false;
1792 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1793 // The mask element. This indexes into the input.
1794 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1795
1796 // The input vector this mask element indexes into.
1797 unsigned Input = (unsigned)Idx / NewElts;
1798
1799 if (Input >= array_lengthof(Inputs)) {
1800 // The mask element does not index into any input vector.
1801 Ops.push_back(-1);
1802 continue;
1803 }
1804
1805 // Turn the index into an offset from the start of the input vector.
1806 Idx -= Input * NewElts;
1807
1808 // Find or create a shuffle vector operand to hold this input.
1809 unsigned OpNo;
1810 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
1811 if (InputUsed[OpNo] == Input) {
1812 // This input vector is already an operand.
1813 break;
1814 } else if (InputUsed[OpNo] == -1U) {
1815 // Create a new operand for this input vector.
1816 InputUsed[OpNo] = Input;
1817 break;
1818 }
1819 }
1820
1821 if (OpNo >= array_lengthof(InputUsed)) {
1822 // More than two input vectors used! Give up on trying to create a
1823 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
1824 useBuildVector = true;
1825 break;
1826 }
1827
1828 // Add the mask index for the new shuffle vector.
1829 Ops.push_back(Idx + OpNo * NewElts);
1830 }
1831
1832 if (useBuildVector) {
1833 EVT EltVT = NewVT.getVectorElementType();
1834 SmallVector<SDValue, 16> SVOps;
1835
1836 // Extract the input elements by hand.
1837 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1838 // The mask element. This indexes into the input.
1839 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1840
1841 // The input vector this mask element indexes into.
1842 unsigned Input = (unsigned)Idx / NewElts;
1843
1844 if (Input >= array_lengthof(Inputs)) {
1845 // The mask element is "undef" or indexes off the end of the input.
1846 SVOps.push_back(DAG.getUNDEF(EltVT));
1847 continue;
1848 }
1849
1850 // Turn the index into an offset from the start of the input vector.
1851 Idx -= Input * NewElts;
1852
1853 // Extract the vector element by hand.
1854 SVOps.push_back(DAG.getNode(
1855 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1856 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1857 }
1858
1859 // Construct the Lo/Hi output using a BUILD_VECTOR.
1860 Output = DAG.getBuildVector(NewVT, dl, SVOps);
1861 } else if (InputUsed[0] == -1U) {
1862 // No input vectors were used! The result is undefined.
1863 Output = DAG.getUNDEF(NewVT);
1864 } else {
1865 SDValue Op0 = Inputs[InputUsed[0]];
1866 // If only one input was used, use an undefined vector for the other.
1867 SDValue Op1 = InputUsed[1] == -1U ?
1868 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1869 // At least one input vector was used. Create a new shuffle vector.
1870 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1871 }
1872
1873 Ops.clear();
1874 }
1875 }
1876
SplitVecRes_VAARG(SDNode * N,SDValue & Lo,SDValue & Hi)1877 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
1878 EVT OVT = N->getValueType(0);
1879 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
1880 SDValue Chain = N->getOperand(0);
1881 SDValue Ptr = N->getOperand(1);
1882 SDValue SV = N->getOperand(2);
1883 SDLoc dl(N);
1884
1885 const unsigned Alignment = DAG.getDataLayout().getABITypeAlignment(
1886 NVT.getTypeForEVT(*DAG.getContext()));
1887
1888 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment);
1889 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment);
1890 Chain = Hi.getValue(1);
1891
1892 // Modified the chain - switch anything that used the old chain to use
1893 // the new one.
1894 ReplaceValueWith(SDValue(N, 1), Chain);
1895 }
1896
1897
1898 //===----------------------------------------------------------------------===//
1899 // Operand Vector Splitting
1900 //===----------------------------------------------------------------------===//
1901
1902 /// This method is called when the specified operand of the specified node is
1903 /// found to need vector splitting. At this point, all of the result types of
1904 /// the node are known to be legal, but other operands of the node may need
1905 /// legalization as well as the specified one.
SplitVectorOperand(SDNode * N,unsigned OpNo)1906 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1907 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1908 SDValue Res = SDValue();
1909
1910 // See if the target wants to custom split this node.
1911 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1912 return false;
1913
1914 if (!Res.getNode()) {
1915 switch (N->getOpcode()) {
1916 default:
1917 #ifndef NDEBUG
1918 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1919 N->dump(&DAG);
1920 dbgs() << "\n";
1921 #endif
1922 report_fatal_error("Do not know how to split this operator's "
1923 "operand!\n");
1924
1925 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1926 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1927 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1928 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1929 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1930 case ISD::TRUNCATE:
1931 Res = SplitVecOp_TruncateHelper(N);
1932 break;
1933 case ISD::STRICT_FP_ROUND:
1934 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1935 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
1936 case ISD::STORE:
1937 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1938 break;
1939 case ISD::MSTORE:
1940 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1941 break;
1942 case ISD::MSCATTER:
1943 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1944 break;
1945 case ISD::MGATHER:
1946 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1947 break;
1948 case ISD::VSELECT:
1949 Res = SplitVecOp_VSELECT(N, OpNo);
1950 break;
1951 case ISD::STRICT_SINT_TO_FP:
1952 case ISD::STRICT_UINT_TO_FP:
1953 case ISD::SINT_TO_FP:
1954 case ISD::UINT_TO_FP:
1955 if (N->getValueType(0).bitsLT(
1956 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
1957 Res = SplitVecOp_TruncateHelper(N);
1958 else
1959 Res = SplitVecOp_UnaryOp(N);
1960 break;
1961 case ISD::FP_TO_SINT:
1962 case ISD::FP_TO_UINT:
1963 case ISD::STRICT_FP_TO_SINT:
1964 case ISD::STRICT_FP_TO_UINT:
1965 case ISD::CTTZ:
1966 case ISD::CTLZ:
1967 case ISD::CTPOP:
1968 case ISD::STRICT_FP_EXTEND:
1969 case ISD::FP_EXTEND:
1970 case ISD::SIGN_EXTEND:
1971 case ISD::ZERO_EXTEND:
1972 case ISD::ANY_EXTEND:
1973 case ISD::FTRUNC:
1974 case ISD::FCANONICALIZE:
1975 Res = SplitVecOp_UnaryOp(N);
1976 break;
1977
1978 case ISD::ANY_EXTEND_VECTOR_INREG:
1979 case ISD::SIGN_EXTEND_VECTOR_INREG:
1980 case ISD::ZERO_EXTEND_VECTOR_INREG:
1981 Res = SplitVecOp_ExtVecInRegOp(N);
1982 break;
1983
1984 case ISD::VECREDUCE_FADD:
1985 case ISD::VECREDUCE_FMUL:
1986 case ISD::VECREDUCE_ADD:
1987 case ISD::VECREDUCE_MUL:
1988 case ISD::VECREDUCE_AND:
1989 case ISD::VECREDUCE_OR:
1990 case ISD::VECREDUCE_XOR:
1991 case ISD::VECREDUCE_SMAX:
1992 case ISD::VECREDUCE_SMIN:
1993 case ISD::VECREDUCE_UMAX:
1994 case ISD::VECREDUCE_UMIN:
1995 case ISD::VECREDUCE_FMAX:
1996 case ISD::VECREDUCE_FMIN:
1997 Res = SplitVecOp_VECREDUCE(N, OpNo);
1998 break;
1999 }
2000 }
2001
2002 // If the result is null, the sub-method took care of registering results etc.
2003 if (!Res.getNode()) return false;
2004
2005 // If the result is N, the sub-method updated N in place. Tell the legalizer
2006 // core about this.
2007 if (Res.getNode() == N)
2008 return true;
2009
2010 if (N->isStrictFPOpcode())
2011 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2012 "Invalid operand expansion");
2013 else
2014 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2015 "Invalid operand expansion");
2016
2017 ReplaceValueWith(SDValue(N, 0), Res);
2018 return false;
2019 }
2020
SplitVecOp_VSELECT(SDNode * N,unsigned OpNo)2021 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2022 // The only possibility for an illegal operand is the mask, since result type
2023 // legalization would have handled this node already otherwise.
2024 assert(OpNo == 0 && "Illegal operand must be mask");
2025
2026 SDValue Mask = N->getOperand(0);
2027 SDValue Src0 = N->getOperand(1);
2028 SDValue Src1 = N->getOperand(2);
2029 EVT Src0VT = Src0.getValueType();
2030 SDLoc DL(N);
2031 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2032
2033 SDValue Lo, Hi;
2034 GetSplitVector(N->getOperand(0), Lo, Hi);
2035 assert(Lo.getValueType() == Hi.getValueType() &&
2036 "Lo and Hi have differing types");
2037
2038 EVT LoOpVT, HiOpVT;
2039 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2040 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2041
2042 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2043 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2044 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2045 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2046
2047 SDValue LoSelect =
2048 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2049 SDValue HiSelect =
2050 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2051
2052 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2053 }
2054
SplitVecOp_VECREDUCE(SDNode * N,unsigned OpNo)2055 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2056 EVT ResVT = N->getValueType(0);
2057 SDValue Lo, Hi;
2058 SDLoc dl(N);
2059
2060 SDValue VecOp = N->getOperand(OpNo);
2061 EVT VecVT = VecOp.getValueType();
2062 assert(VecVT.isVector() && "Can only split reduce vector operand");
2063 GetSplitVector(VecOp, Lo, Hi);
2064 EVT LoOpVT, HiOpVT;
2065 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2066
2067 bool NoNaN = N->getFlags().hasNoNaNs();
2068 unsigned CombineOpc = 0;
2069 switch (N->getOpcode()) {
2070 case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break;
2071 case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break;
2072 case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break;
2073 case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break;
2074 case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break;
2075 case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break;
2076 case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break;
2077 case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break;
2078 case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break;
2079 case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break;
2080 case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break;
2081 case ISD::VECREDUCE_FMAX:
2082 CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXIMUM;
2083 break;
2084 case ISD::VECREDUCE_FMIN:
2085 CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINIMUM;
2086 break;
2087 default:
2088 llvm_unreachable("Unexpected reduce ISD node");
2089 }
2090
2091 // Use the appropriate scalar instruction on the split subvectors before
2092 // reducing the now partially reduced smaller vector.
2093 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2094 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2095 }
2096
SplitVecOp_UnaryOp(SDNode * N)2097 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2098 // The result has a legal vector type, but the input needs splitting.
2099 EVT ResVT = N->getValueType(0);
2100 SDValue Lo, Hi;
2101 SDLoc dl(N);
2102 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2103 EVT InVT = Lo.getValueType();
2104
2105 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2106 InVT.getVectorNumElements());
2107
2108 if (N->isStrictFPOpcode()) {
2109 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2110 { N->getOperand(0), Lo });
2111 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2112 { N->getOperand(0), Hi });
2113
2114 // Build a factor node to remember that this operation is independent
2115 // of the other one.
2116 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2117 Hi.getValue(1));
2118
2119 // Legalize the chain result - switch anything that used the old chain to
2120 // use the new one.
2121 ReplaceValueWith(SDValue(N, 1), Ch);
2122 } else {
2123 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2124 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2125 }
2126
2127 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2128 }
2129
SplitVecOp_BITCAST(SDNode * N)2130 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2131 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2132 // end up being split all the way down to individual components. Convert the
2133 // split pieces into integers and reassemble.
2134 SDValue Lo, Hi;
2135 GetSplitVector(N->getOperand(0), Lo, Hi);
2136 Lo = BitConvertToInteger(Lo);
2137 Hi = BitConvertToInteger(Hi);
2138
2139 if (DAG.getDataLayout().isBigEndian())
2140 std::swap(Lo, Hi);
2141
2142 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2143 JoinIntegers(Lo, Hi));
2144 }
2145
SplitVecOp_EXTRACT_SUBVECTOR(SDNode * N)2146 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2147 // We know that the extracted result type is legal.
2148 EVT SubVT = N->getValueType(0);
2149 SDValue Idx = N->getOperand(1);
2150 SDLoc dl(N);
2151 SDValue Lo, Hi;
2152 GetSplitVector(N->getOperand(0), Lo, Hi);
2153
2154 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2155 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2156
2157 if (IdxVal < LoElts) {
2158 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
2159 "Extracted subvector crosses vector split!");
2160 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2161 } else {
2162 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2163 DAG.getConstant(IdxVal - LoElts, dl,
2164 Idx.getValueType()));
2165 }
2166 }
2167
SplitVecOp_EXTRACT_VECTOR_ELT(SDNode * N)2168 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2169 SDValue Vec = N->getOperand(0);
2170 SDValue Idx = N->getOperand(1);
2171 EVT VecVT = Vec.getValueType();
2172
2173 if (isa<ConstantSDNode>(Idx)) {
2174 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2175
2176 SDValue Lo, Hi;
2177 GetSplitVector(Vec, Lo, Hi);
2178
2179 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2180
2181 if (IdxVal < LoElts)
2182 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2183 return SDValue(DAG.UpdateNodeOperands(N, Hi,
2184 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2185 Idx.getValueType())), 0);
2186 }
2187
2188 // See if the target wants to custom expand this node.
2189 if (CustomLowerNode(N, N->getValueType(0), true))
2190 return SDValue();
2191
2192 // Make the vector elements byte-addressable if they aren't already.
2193 SDLoc dl(N);
2194 EVT EltVT = VecVT.getVectorElementType();
2195 if (VecVT.getScalarSizeInBits() < 8) {
2196 EltVT = MVT::i8;
2197 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2198 VecVT.getVectorNumElements());
2199 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2200 }
2201
2202 // Store the vector to the stack.
2203 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
2204 auto &MF = DAG.getMachineFunction();
2205 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2206 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2207 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
2208
2209 // Load back the required element.
2210 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2211
2212 // FIXME: This is to handle i1 vectors with elements promoted to i8.
2213 // i1 vector handling needs general improvement.
2214 if (N->getValueType(0).bitsLT(EltVT)) {
2215 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2216 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
2217 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2218 }
2219
2220 return DAG.getExtLoad(
2221 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2222 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
2223 }
2224
SplitVecOp_ExtVecInRegOp(SDNode * N)2225 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2226 SDValue Lo, Hi;
2227
2228 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2229 // splitting the result has the same effect as splitting the input operand.
2230 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2231
2232 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2233 }
2234
SplitVecOp_MGATHER(MaskedGatherSDNode * MGT,unsigned OpNo)2235 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2236 unsigned OpNo) {
2237 EVT LoVT, HiVT;
2238 SDLoc dl(MGT);
2239 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2240
2241 SDValue Ch = MGT->getChain();
2242 SDValue Ptr = MGT->getBasePtr();
2243 SDValue Index = MGT->getIndex();
2244 SDValue Scale = MGT->getScale();
2245 SDValue Mask = MGT->getMask();
2246 SDValue PassThru = MGT->getPassThru();
2247 unsigned Alignment = MGT->getOriginalAlignment();
2248
2249 SDValue MaskLo, MaskHi;
2250 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2251 // Split Mask operand
2252 GetSplitVector(Mask, MaskLo, MaskHi);
2253 else
2254 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2255
2256 EVT MemoryVT = MGT->getMemoryVT();
2257 EVT LoMemVT, HiMemVT;
2258 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2259
2260 SDValue PassThruLo, PassThruHi;
2261 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2262 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2263 else
2264 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2265
2266 SDValue IndexHi, IndexLo;
2267 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2268 GetSplitVector(Index, IndexLo, IndexHi);
2269 else
2270 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2271
2272 MachineMemOperand *MMO = DAG.getMachineFunction().
2273 getMachineMemOperand(MGT->getPointerInfo(),
2274 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
2275 Alignment, MGT->getAAInfo(), MGT->getRanges());
2276
2277 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2278 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2279 OpsLo, MMO, MGT->getIndexType());
2280
2281 MMO = DAG.getMachineFunction().
2282 getMachineMemOperand(MGT->getPointerInfo(),
2283 MachineMemOperand::MOLoad, HiMemVT.getStoreSize(),
2284 Alignment, MGT->getAAInfo(),
2285 MGT->getRanges());
2286
2287 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2288 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2289 OpsHi, MMO, MGT->getIndexType());
2290
2291 // Build a factor node to remember that this load is independent of the
2292 // other one.
2293 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2294 Hi.getValue(1));
2295
2296 // Legalize the chain result - switch anything that used the old chain to
2297 // use the new one.
2298 ReplaceValueWith(SDValue(MGT, 1), Ch);
2299
2300 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2301 Hi);
2302 ReplaceValueWith(SDValue(MGT, 0), Res);
2303 return SDValue();
2304 }
2305
SplitVecOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)2306 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2307 unsigned OpNo) {
2308 assert(N->isUnindexed() && "Indexed masked store of vector?");
2309 SDValue Ch = N->getChain();
2310 SDValue Ptr = N->getBasePtr();
2311 SDValue Offset = N->getOffset();
2312 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
2313 SDValue Mask = N->getMask();
2314 SDValue Data = N->getValue();
2315 EVT MemoryVT = N->getMemoryVT();
2316 unsigned Alignment = N->getOriginalAlignment();
2317 SDLoc DL(N);
2318
2319 EVT LoMemVT, HiMemVT;
2320 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2321
2322 SDValue DataLo, DataHi;
2323 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2324 // Split Data operand
2325 GetSplitVector(Data, DataLo, DataHi);
2326 else
2327 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2328
2329 // Split Mask operand
2330 SDValue MaskLo, MaskHi;
2331 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2332 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2333 } else {
2334 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2335 GetSplitVector(Mask, MaskLo, MaskHi);
2336 else
2337 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2338 }
2339
2340 SDValue Lo, Hi;
2341 MachineMemOperand *MMO = DAG.getMachineFunction().
2342 getMachineMemOperand(N->getPointerInfo(),
2343 MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
2344 Alignment, N->getAAInfo(), N->getRanges());
2345
2346 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
2347 N->getAddressingMode(), N->isTruncatingStore(),
2348 N->isCompressingStore());
2349
2350 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2351 N->isCompressingStore());
2352 unsigned HiOffset = LoMemVT.getStoreSize();
2353
2354 MMO = DAG.getMachineFunction().getMachineMemOperand(
2355 N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore,
2356 HiMemVT.getStoreSize(), Alignment, N->getAAInfo(),
2357 N->getRanges());
2358
2359 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
2360 N->getAddressingMode(), N->isTruncatingStore(),
2361 N->isCompressingStore());
2362
2363 // Build a factor node to remember that this store is independent of the
2364 // other one.
2365 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2366 }
2367
SplitVecOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)2368 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2369 unsigned OpNo) {
2370 SDValue Ch = N->getChain();
2371 SDValue Ptr = N->getBasePtr();
2372 SDValue Mask = N->getMask();
2373 SDValue Index = N->getIndex();
2374 SDValue Scale = N->getScale();
2375 SDValue Data = N->getValue();
2376 EVT MemoryVT = N->getMemoryVT();
2377 unsigned Alignment = N->getOriginalAlignment();
2378 SDLoc DL(N);
2379
2380 // Split all operands
2381 EVT LoMemVT, HiMemVT;
2382 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2383
2384 SDValue DataLo, DataHi;
2385 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2386 // Split Data operand
2387 GetSplitVector(Data, DataLo, DataHi);
2388 else
2389 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2390
2391 // Split Mask operand
2392 SDValue MaskLo, MaskHi;
2393 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2394 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2395 } else {
2396 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2397 GetSplitVector(Mask, MaskLo, MaskHi);
2398 else
2399 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2400 }
2401
2402 SDValue IndexHi, IndexLo;
2403 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2404 GetSplitVector(Index, IndexLo, IndexHi);
2405 else
2406 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2407
2408 SDValue Lo;
2409 MachineMemOperand *MMO = DAG.getMachineFunction().
2410 getMachineMemOperand(N->getPointerInfo(),
2411 MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
2412 Alignment, N->getAAInfo(), N->getRanges());
2413
2414 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2415 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2416 DL, OpsLo, MMO, N->getIndexType());
2417
2418 MMO = DAG.getMachineFunction().
2419 getMachineMemOperand(N->getPointerInfo(),
2420 MachineMemOperand::MOStore, HiMemVT.getStoreSize(),
2421 Alignment, N->getAAInfo(), N->getRanges());
2422
2423 // The order of the Scatter operation after split is well defined. The "Hi"
2424 // part comes after the "Lo". So these two operations should be chained one
2425 // after another.
2426 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2427 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2428 DL, OpsHi, MMO, N->getIndexType());
2429 }
2430
SplitVecOp_STORE(StoreSDNode * N,unsigned OpNo)2431 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2432 assert(N->isUnindexed() && "Indexed store of vector?");
2433 assert(OpNo == 1 && "Can only split the stored value");
2434 SDLoc DL(N);
2435
2436 bool isTruncating = N->isTruncatingStore();
2437 SDValue Ch = N->getChain();
2438 SDValue Ptr = N->getBasePtr();
2439 EVT MemoryVT = N->getMemoryVT();
2440 unsigned Alignment = N->getOriginalAlignment();
2441 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2442 AAMDNodes AAInfo = N->getAAInfo();
2443 SDValue Lo, Hi;
2444 GetSplitVector(N->getOperand(1), Lo, Hi);
2445
2446 EVT LoMemVT, HiMemVT;
2447 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2448
2449 // Scalarize if the split halves are not byte-sized.
2450 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2451 return TLI.scalarizeVectorStore(N, DAG);
2452
2453 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2454
2455 if (isTruncating)
2456 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2457 Alignment, MMOFlags, AAInfo);
2458 else
2459 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2460 AAInfo);
2461
2462 // Increment the pointer to the other half.
2463 Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2464
2465 if (isTruncating)
2466 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2467 N->getPointerInfo().getWithOffset(IncrementSize),
2468 HiMemVT, Alignment, MMOFlags, AAInfo);
2469 else
2470 Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2471 N->getPointerInfo().getWithOffset(IncrementSize),
2472 Alignment, MMOFlags, AAInfo);
2473
2474 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2475 }
2476
SplitVecOp_CONCAT_VECTORS(SDNode * N)2477 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2478 SDLoc DL(N);
2479
2480 // The input operands all must have the same type, and we know the result
2481 // type is valid. Convert this to a buildvector which extracts all the
2482 // input elements.
2483 // TODO: If the input elements are power-two vectors, we could convert this to
2484 // a new CONCAT_VECTORS node with elements that are half-wide.
2485 SmallVector<SDValue, 32> Elts;
2486 EVT EltVT = N->getValueType(0).getVectorElementType();
2487 for (const SDValue &Op : N->op_values()) {
2488 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2489 i != e; ++i) {
2490 Elts.push_back(DAG.getNode(
2491 ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2492 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2493 }
2494 }
2495
2496 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2497 }
2498
SplitVecOp_TruncateHelper(SDNode * N)2499 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2500 // The result type is legal, but the input type is illegal. If splitting
2501 // ends up with the result type of each half still being legal, just
2502 // do that. If, however, that would result in an illegal result type,
2503 // we can try to get more clever with power-two vectors. Specifically,
2504 // split the input type, but also widen the result element size, then
2505 // concatenate the halves and truncate again. For example, consider a target
2506 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2507 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2508 // %inlo = v4i32 extract_subvector %in, 0
2509 // %inhi = v4i32 extract_subvector %in, 4
2510 // %lo16 = v4i16 trunc v4i32 %inlo
2511 // %hi16 = v4i16 trunc v4i32 %inhi
2512 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2513 // %res = v8i8 trunc v8i16 %in16
2514 //
2515 // Without this transform, the original truncate would end up being
2516 // scalarized, which is pretty much always a last resort.
2517 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
2518 SDValue InVec = N->getOperand(OpNo);
2519 EVT InVT = InVec->getValueType(0);
2520 EVT OutVT = N->getValueType(0);
2521 unsigned NumElements = OutVT.getVectorNumElements();
2522 bool IsFloat = OutVT.isFloatingPoint();
2523
2524 // Widening should have already made sure this is a power-two vector
2525 // if we're trying to split it at all. assert() that's true, just in case.
2526 assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2527
2528 unsigned InElementSize = InVT.getScalarSizeInBits();
2529 unsigned OutElementSize = OutVT.getScalarSizeInBits();
2530
2531 // Determine the split output VT. If its legal we can just split dirctly.
2532 EVT LoOutVT, HiOutVT;
2533 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2534 assert(LoOutVT == HiOutVT && "Unequal split?");
2535
2536 // If the input elements are only 1/2 the width of the result elements,
2537 // just use the normal splitting. Our trick only work if there's room
2538 // to split more than once.
2539 if (isTypeLegal(LoOutVT) ||
2540 InElementSize <= OutElementSize * 2)
2541 return SplitVecOp_UnaryOp(N);
2542 SDLoc DL(N);
2543
2544 // Don't touch if this will be scalarized.
2545 EVT FinalVT = InVT;
2546 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2547 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2548
2549 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2550 return SplitVecOp_UnaryOp(N);
2551
2552 // Get the split input vector.
2553 SDValue InLoVec, InHiVec;
2554 GetSplitVector(InVec, InLoVec, InHiVec);
2555
2556 // Truncate them to 1/2 the element size.
2557 EVT HalfElementVT = IsFloat ?
2558 EVT::getFloatingPointVT(InElementSize/2) :
2559 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2560 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2561 NumElements/2);
2562
2563 SDValue HalfLo;
2564 SDValue HalfHi;
2565 SDValue Chain;
2566 if (N->isStrictFPOpcode()) {
2567 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2568 {N->getOperand(0), HalfLo});
2569 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2570 {N->getOperand(0), HalfHi});
2571 // Legalize the chain result - switch anything that used the old chain to
2572 // use the new one.
2573 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
2574 HalfHi.getValue(1));
2575 } else {
2576 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2577 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2578 }
2579 // Concatenate them to get the full intermediate truncation result.
2580 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2581 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2582 HalfHi);
2583 // Now finish up by truncating all the way down to the original result
2584 // type. This should normally be something that ends up being legal directly,
2585 // but in theory if a target has very wide vectors and an annoyingly
2586 // restricted set of legal types, this split can chain to build things up.
2587
2588 if (N->isStrictFPOpcode()) {
2589 SDValue Res = DAG.getNode(
2590 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
2591 {Chain, InterVec,
2592 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
2593 // Relink the chain
2594 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2595 return Res;
2596 }
2597
2598 return IsFloat
2599 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2600 DAG.getTargetConstant(
2601 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2602 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2603 }
2604
SplitVecOp_VSETCC(SDNode * N)2605 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2606 assert(N->getValueType(0).isVector() &&
2607 N->getOperand(0).getValueType().isVector() &&
2608 "Operand types must be vectors");
2609 // The result has a legal vector type, but the input needs splitting.
2610 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2611 SDLoc DL(N);
2612 GetSplitVector(N->getOperand(0), Lo0, Hi0);
2613 GetSplitVector(N->getOperand(1), Lo1, Hi1);
2614 unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2615 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2616 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2617
2618 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2619 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2620 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2621
2622 EVT OpVT = N->getOperand(0).getValueType();
2623 ISD::NodeType ExtendCode =
2624 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
2625 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
2626 }
2627
2628
SplitVecOp_FP_ROUND(SDNode * N)2629 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2630 // The result has a legal vector type, but the input needs splitting.
2631 EVT ResVT = N->getValueType(0);
2632 SDValue Lo, Hi;
2633 SDLoc DL(N);
2634 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2635 EVT InVT = Lo.getValueType();
2636
2637 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2638 InVT.getVectorNumElements());
2639
2640 if (N->isStrictFPOpcode()) {
2641 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2642 { N->getOperand(0), Lo, N->getOperand(2) });
2643 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2644 { N->getOperand(0), Hi, N->getOperand(2) });
2645 // Legalize the chain result - switch anything that used the old chain to
2646 // use the new one.
2647 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2648 Lo.getValue(1), Hi.getValue(1));
2649 ReplaceValueWith(SDValue(N, 1), NewChain);
2650 } else {
2651 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2652 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2653 }
2654
2655 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2656 }
2657
SplitVecOp_FCOPYSIGN(SDNode * N)2658 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2659 // The result (and the first input) has a legal vector type, but the second
2660 // input needs splitting.
2661 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2662 }
2663
2664
2665 //===----------------------------------------------------------------------===//
2666 // Result Vector Widening
2667 //===----------------------------------------------------------------------===//
2668
WidenVectorResult(SDNode * N,unsigned ResNo)2669 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2670 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2671 dbgs() << "\n");
2672
2673 // See if the target wants to custom widen this node.
2674 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2675 return;
2676
2677 SDValue Res = SDValue();
2678 switch (N->getOpcode()) {
2679 default:
2680 #ifndef NDEBUG
2681 dbgs() << "WidenVectorResult #" << ResNo << ": ";
2682 N->dump(&DAG);
2683 dbgs() << "\n";
2684 #endif
2685 llvm_unreachable("Do not know how to widen the result of this operator!");
2686
2687 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2688 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2689 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2690 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2691 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2692 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2693 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2694 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
2695 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2696 case ISD::VSELECT:
2697 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2698 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2699 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2700 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2701 case ISD::VECTOR_SHUFFLE:
2702 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2703 break;
2704 case ISD::MLOAD:
2705 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2706 break;
2707 case ISD::MGATHER:
2708 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2709 break;
2710
2711 case ISD::ADD:
2712 case ISD::AND:
2713 case ISD::MUL:
2714 case ISD::MULHS:
2715 case ISD::MULHU:
2716 case ISD::OR:
2717 case ISD::SUB:
2718 case ISD::XOR:
2719 case ISD::FMINNUM:
2720 case ISD::FMAXNUM:
2721 case ISD::FMINIMUM:
2722 case ISD::FMAXIMUM:
2723 case ISD::SMIN:
2724 case ISD::SMAX:
2725 case ISD::UMIN:
2726 case ISD::UMAX:
2727 case ISD::UADDSAT:
2728 case ISD::SADDSAT:
2729 case ISD::USUBSAT:
2730 case ISD::SSUBSAT:
2731 Res = WidenVecRes_Binary(N);
2732 break;
2733
2734 case ISD::FADD:
2735 case ISD::FMUL:
2736 case ISD::FPOW:
2737 case ISD::FSUB:
2738 case ISD::FDIV:
2739 case ISD::FREM:
2740 case ISD::SDIV:
2741 case ISD::UDIV:
2742 case ISD::SREM:
2743 case ISD::UREM:
2744 Res = WidenVecRes_BinaryCanTrap(N);
2745 break;
2746
2747 case ISD::SMULFIX:
2748 case ISD::SMULFIXSAT:
2749 case ISD::UMULFIX:
2750 case ISD::UMULFIXSAT:
2751 // These are binary operations, but with an extra operand that shouldn't
2752 // be widened (the scale).
2753 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
2754 break;
2755
2756 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
2757 case ISD::STRICT_##DAGN:
2758 #include "llvm/IR/ConstrainedOps.def"
2759 Res = WidenVecRes_StrictFP(N);
2760 break;
2761
2762 case ISD::UADDO:
2763 case ISD::SADDO:
2764 case ISD::USUBO:
2765 case ISD::SSUBO:
2766 case ISD::UMULO:
2767 case ISD::SMULO:
2768 Res = WidenVecRes_OverflowOp(N, ResNo);
2769 break;
2770
2771 case ISD::FCOPYSIGN:
2772 Res = WidenVecRes_FCOPYSIGN(N);
2773 break;
2774
2775 case ISD::FPOWI:
2776 Res = WidenVecRes_POWI(N);
2777 break;
2778
2779 case ISD::SHL:
2780 case ISD::SRA:
2781 case ISD::SRL:
2782 Res = WidenVecRes_Shift(N);
2783 break;
2784
2785 case ISD::ANY_EXTEND_VECTOR_INREG:
2786 case ISD::SIGN_EXTEND_VECTOR_INREG:
2787 case ISD::ZERO_EXTEND_VECTOR_INREG:
2788 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2789 break;
2790
2791 case ISD::ANY_EXTEND:
2792 case ISD::FP_EXTEND:
2793 case ISD::FP_ROUND:
2794 case ISD::FP_TO_SINT:
2795 case ISD::FP_TO_UINT:
2796 case ISD::SIGN_EXTEND:
2797 case ISD::SINT_TO_FP:
2798 case ISD::TRUNCATE:
2799 case ISD::UINT_TO_FP:
2800 case ISD::ZERO_EXTEND:
2801 Res = WidenVecRes_Convert(N);
2802 break;
2803
2804 case ISD::FABS:
2805 case ISD::FCEIL:
2806 case ISD::FCOS:
2807 case ISD::FEXP:
2808 case ISD::FEXP2:
2809 case ISD::FFLOOR:
2810 case ISD::FLOG:
2811 case ISD::FLOG10:
2812 case ISD::FLOG2:
2813 case ISD::FNEARBYINT:
2814 case ISD::FRINT:
2815 case ISD::FROUND:
2816 case ISD::FSIN:
2817 case ISD::FSQRT:
2818 case ISD::FTRUNC: {
2819 // We're going to widen this vector op to a legal type by padding with undef
2820 // elements. If the wide vector op is eventually going to be expanded to
2821 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
2822 // libcalls on the undef elements.
2823 EVT VT = N->getValueType(0);
2824 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2825 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
2826 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
2827 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2828 break;
2829 }
2830 }
2831 // If the target has custom/legal support for the scalar FP intrinsic ops
2832 // (they are probably not destined to become libcalls), then widen those like
2833 // any other unary ops.
2834 LLVM_FALLTHROUGH;
2835
2836 case ISD::ABS:
2837 case ISD::BITREVERSE:
2838 case ISD::BSWAP:
2839 case ISD::CTLZ:
2840 case ISD::CTLZ_ZERO_UNDEF:
2841 case ISD::CTPOP:
2842 case ISD::CTTZ:
2843 case ISD::CTTZ_ZERO_UNDEF:
2844 case ISD::FNEG:
2845 case ISD::FCANONICALIZE:
2846 Res = WidenVecRes_Unary(N);
2847 break;
2848 case ISD::FMA:
2849 Res = WidenVecRes_Ternary(N);
2850 break;
2851 }
2852
2853 // If Res is null, the sub-method took care of registering the result.
2854 if (Res.getNode())
2855 SetWidenedVector(SDValue(N, ResNo), Res);
2856 }
2857
WidenVecRes_Ternary(SDNode * N)2858 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2859 // Ternary op widening.
2860 SDLoc dl(N);
2861 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2862 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2863 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2864 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2865 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2866 }
2867
WidenVecRes_Binary(SDNode * N)2868 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2869 // Binary op widening.
2870 SDLoc dl(N);
2871 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2872 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2873 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2874 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2875 }
2876
WidenVecRes_BinaryWithExtraScalarOp(SDNode * N)2877 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
2878 // Binary op widening, but with an extra operand that shouldn't be widened.
2879 SDLoc dl(N);
2880 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2881 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2882 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2883 SDValue InOp3 = N->getOperand(2);
2884 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
2885 N->getFlags());
2886 }
2887
2888 // Given a vector of operations that have been broken up to widen, see
2889 // if we can collect them together into the next widest legal VT. This
2890 // implementation is trap-safe.
CollectOpsToWiden(SelectionDAG & DAG,const TargetLowering & TLI,SmallVectorImpl<SDValue> & ConcatOps,unsigned ConcatEnd,EVT VT,EVT MaxVT,EVT WidenVT)2891 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
2892 SmallVectorImpl<SDValue> &ConcatOps,
2893 unsigned ConcatEnd, EVT VT, EVT MaxVT,
2894 EVT WidenVT) {
2895 // Check to see if we have a single operation with the widen type.
2896 if (ConcatEnd == 1) {
2897 VT = ConcatOps[0].getValueType();
2898 if (VT == WidenVT)
2899 return ConcatOps[0];
2900 }
2901
2902 SDLoc dl(ConcatOps[0]);
2903 EVT WidenEltVT = WidenVT.getVectorElementType();
2904
2905 // while (Some element of ConcatOps is not of type MaxVT) {
2906 // From the end of ConcatOps, collect elements of the same type and put
2907 // them into an op of the next larger supported type
2908 // }
2909 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2910 int Idx = ConcatEnd - 1;
2911 VT = ConcatOps[Idx--].getValueType();
2912 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2913 Idx--;
2914
2915 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
2916 EVT NextVT;
2917 do {
2918 NextSize *= 2;
2919 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2920 } while (!TLI.isTypeLegal(NextVT));
2921
2922 if (!VT.isVector()) {
2923 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2924 SDValue VecOp = DAG.getUNDEF(NextVT);
2925 unsigned NumToInsert = ConcatEnd - Idx - 1;
2926 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2927 VecOp = DAG.getNode(
2928 ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2929 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2930 }
2931 ConcatOps[Idx+1] = VecOp;
2932 ConcatEnd = Idx + 2;
2933 } else {
2934 // Vector type, create a CONCAT_VECTORS of type NextVT
2935 SDValue undefVec = DAG.getUNDEF(VT);
2936 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2937 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2938 unsigned RealVals = ConcatEnd - Idx - 1;
2939 unsigned SubConcatEnd = 0;
2940 unsigned SubConcatIdx = Idx + 1;
2941 while (SubConcatEnd < RealVals)
2942 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2943 while (SubConcatEnd < OpsToConcat)
2944 SubConcatOps[SubConcatEnd++] = undefVec;
2945 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2946 NextVT, SubConcatOps);
2947 ConcatEnd = SubConcatIdx + 1;
2948 }
2949 }
2950
2951 // Check to see if we have a single operation with the widen type.
2952 if (ConcatEnd == 1) {
2953 VT = ConcatOps[0].getValueType();
2954 if (VT == WidenVT)
2955 return ConcatOps[0];
2956 }
2957
2958 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2959 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2960 if (NumOps != ConcatEnd ) {
2961 SDValue UndefVal = DAG.getUNDEF(MaxVT);
2962 for (unsigned j = ConcatEnd; j < NumOps; ++j)
2963 ConcatOps[j] = UndefVal;
2964 }
2965 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2966 makeArrayRef(ConcatOps.data(), NumOps));
2967 }
2968
WidenVecRes_BinaryCanTrap(SDNode * N)2969 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2970 // Binary op widening for operations that can trap.
2971 unsigned Opcode = N->getOpcode();
2972 SDLoc dl(N);
2973 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2974 EVT WidenEltVT = WidenVT.getVectorElementType();
2975 EVT VT = WidenVT;
2976 unsigned NumElts = VT.getVectorNumElements();
2977 const SDNodeFlags Flags = N->getFlags();
2978 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
2979 NumElts = NumElts / 2;
2980 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2981 }
2982
2983 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
2984 // Operation doesn't trap so just widen as normal.
2985 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2986 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2987 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2988 }
2989
2990 // No legal vector version so unroll the vector operation and then widen.
2991 if (NumElts == 1)
2992 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2993
2994 // Since the operation can trap, apply operation on the original vector.
2995 EVT MaxVT = VT;
2996 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2997 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2998 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2999
3000 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3001 unsigned ConcatEnd = 0; // Current ConcatOps index.
3002 int Idx = 0; // Current Idx into input vectors.
3003
3004 // NumElts := greatest legal vector size (at most WidenVT)
3005 // while (orig. vector has unhandled elements) {
3006 // take munches of size NumElts from the beginning and add to ConcatOps
3007 // NumElts := next smaller supported vector size or 1
3008 // }
3009 while (CurNumElts != 0) {
3010 while (CurNumElts >= NumElts) {
3011 SDValue EOp1 = DAG.getNode(
3012 ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3013 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3014 SDValue EOp2 = DAG.getNode(
3015 ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3016 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3017 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3018 Idx += NumElts;
3019 CurNumElts -= NumElts;
3020 }
3021 do {
3022 NumElts = NumElts / 2;
3023 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3024 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3025
3026 if (NumElts == 1) {
3027 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3028 SDValue EOp1 = DAG.getNode(
3029 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
3030 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3031 SDValue EOp2 = DAG.getNode(
3032 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
3033 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3034 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3035 EOp1, EOp2, Flags);
3036 }
3037 CurNumElts = 0;
3038 }
3039 }
3040
3041 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3042 }
3043
WidenVecRes_StrictFP(SDNode * N)3044 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3045 switch (N->getOpcode()) {
3046 case ISD::STRICT_FSETCC:
3047 case ISD::STRICT_FSETCCS:
3048 return WidenVecRes_STRICT_FSETCC(N);
3049 case ISD::STRICT_FP_EXTEND:
3050 case ISD::STRICT_FP_ROUND:
3051 case ISD::STRICT_FP_TO_SINT:
3052 case ISD::STRICT_FP_TO_UINT:
3053 case ISD::STRICT_SINT_TO_FP:
3054 case ISD::STRICT_UINT_TO_FP:
3055 return WidenVecRes_Convert_StrictFP(N);
3056 default:
3057 break;
3058 }
3059
3060 // StrictFP op widening for operations that can trap.
3061 unsigned NumOpers = N->getNumOperands();
3062 unsigned Opcode = N->getOpcode();
3063 SDLoc dl(N);
3064 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3065 EVT WidenEltVT = WidenVT.getVectorElementType();
3066 EVT VT = WidenVT;
3067 unsigned NumElts = VT.getVectorNumElements();
3068 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3069 NumElts = NumElts / 2;
3070 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3071 }
3072
3073 // No legal vector version so unroll the vector operation and then widen.
3074 if (NumElts == 1)
3075 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3076
3077 // Since the operation can trap, apply operation on the original vector.
3078 EVT MaxVT = VT;
3079 SmallVector<SDValue, 4> InOps;
3080 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3081
3082 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3083 SmallVector<SDValue, 16> Chains;
3084 unsigned ConcatEnd = 0; // Current ConcatOps index.
3085 int Idx = 0; // Current Idx into input vectors.
3086
3087 // The Chain is the first operand.
3088 InOps.push_back(N->getOperand(0));
3089
3090 // Now process the remaining operands.
3091 for (unsigned i = 1; i < NumOpers; ++i) {
3092 SDValue Oper = N->getOperand(i);
3093
3094 if (Oper.getValueType().isVector()) {
3095 assert(Oper.getValueType() == N->getValueType(0) &&
3096 "Invalid operand type to widen!");
3097 Oper = GetWidenedVector(Oper);
3098 }
3099
3100 InOps.push_back(Oper);
3101 }
3102
3103 // NumElts := greatest legal vector size (at most WidenVT)
3104 // while (orig. vector has unhandled elements) {
3105 // take munches of size NumElts from the beginning and add to ConcatOps
3106 // NumElts := next smaller supported vector size or 1
3107 // }
3108 while (CurNumElts != 0) {
3109 while (CurNumElts >= NumElts) {
3110 SmallVector<SDValue, 4> EOps;
3111
3112 for (unsigned i = 0; i < NumOpers; ++i) {
3113 SDValue Op = InOps[i];
3114
3115 if (Op.getValueType().isVector())
3116 Op = DAG.getNode(
3117 ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3118 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3119
3120 EOps.push_back(Op);
3121 }
3122
3123 EVT OperVT[] = {VT, MVT::Other};
3124 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3125 ConcatOps[ConcatEnd++] = Oper;
3126 Chains.push_back(Oper.getValue(1));
3127 Idx += NumElts;
3128 CurNumElts -= NumElts;
3129 }
3130 do {
3131 NumElts = NumElts / 2;
3132 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3133 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3134
3135 if (NumElts == 1) {
3136 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3137 SmallVector<SDValue, 4> EOps;
3138
3139 for (unsigned i = 0; i < NumOpers; ++i) {
3140 SDValue Op = InOps[i];
3141
3142 if (Op.getValueType().isVector())
3143 Op = DAG.getNode(
3144 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3145 DAG.getConstant(Idx, dl,
3146 TLI.getVectorIdxTy(DAG.getDataLayout())));
3147
3148 EOps.push_back(Op);
3149 }
3150
3151 EVT WidenVT[] = {WidenEltVT, MVT::Other};
3152 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3153 ConcatOps[ConcatEnd++] = Oper;
3154 Chains.push_back(Oper.getValue(1));
3155 }
3156 CurNumElts = 0;
3157 }
3158 }
3159
3160 // Build a factor node to remember all the Ops that have been created.
3161 SDValue NewChain;
3162 if (Chains.size() == 1)
3163 NewChain = Chains[0];
3164 else
3165 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3166 ReplaceValueWith(SDValue(N, 1), NewChain);
3167
3168 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3169 }
3170
WidenVecRes_OverflowOp(SDNode * N,unsigned ResNo)3171 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3172 SDLoc DL(N);
3173 EVT ResVT = N->getValueType(0);
3174 EVT OvVT = N->getValueType(1);
3175 EVT WideResVT, WideOvVT;
3176 SDValue WideLHS, WideRHS;
3177
3178 // TODO: This might result in a widen/split loop.
3179 if (ResNo == 0) {
3180 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3181 WideOvVT = EVT::getVectorVT(
3182 *DAG.getContext(), OvVT.getVectorElementType(),
3183 WideResVT.getVectorNumElements());
3184
3185 WideLHS = GetWidenedVector(N->getOperand(0));
3186 WideRHS = GetWidenedVector(N->getOperand(1));
3187 } else {
3188 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3189 WideResVT = EVT::getVectorVT(
3190 *DAG.getContext(), ResVT.getVectorElementType(),
3191 WideOvVT.getVectorNumElements());
3192
3193 SDValue Zero = DAG.getConstant(
3194 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3195 WideLHS = DAG.getNode(
3196 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3197 N->getOperand(0), Zero);
3198 WideRHS = DAG.getNode(
3199 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3200 N->getOperand(1), Zero);
3201 }
3202
3203 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3204 SDNode *WideNode = DAG.getNode(
3205 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3206
3207 // Replace the other vector result not being explicitly widened here.
3208 unsigned OtherNo = 1 - ResNo;
3209 EVT OtherVT = N->getValueType(OtherNo);
3210 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3211 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3212 } else {
3213 SDValue Zero = DAG.getConstant(
3214 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3215 SDValue OtherVal = DAG.getNode(
3216 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3217 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3218 }
3219
3220 return SDValue(WideNode, ResNo);
3221 }
3222
WidenVecRes_Convert(SDNode * N)3223 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3224 SDValue InOp = N->getOperand(0);
3225 SDLoc DL(N);
3226
3227 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3228 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3229
3230 EVT InVT = InOp.getValueType();
3231 EVT InEltVT = InVT.getVectorElementType();
3232 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
3233
3234 unsigned Opcode = N->getOpcode();
3235 unsigned InVTNumElts = InVT.getVectorNumElements();
3236 const SDNodeFlags Flags = N->getFlags();
3237 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3238 InOp = GetWidenedVector(N->getOperand(0));
3239 InVT = InOp.getValueType();
3240 InVTNumElts = InVT.getVectorNumElements();
3241 if (InVTNumElts == WidenNumElts) {
3242 if (N->getNumOperands() == 1)
3243 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3244 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3245 }
3246 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3247 // If both input and result vector types are of same width, extend
3248 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3249 // accepts fewer elements in the result than in the input.
3250 if (Opcode == ISD::ANY_EXTEND)
3251 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3252 if (Opcode == ISD::SIGN_EXTEND)
3253 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3254 if (Opcode == ISD::ZERO_EXTEND)
3255 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3256 }
3257 }
3258
3259 if (TLI.isTypeLegal(InWidenVT)) {
3260 // Because the result and the input are different vector types, widening
3261 // the result could create a legal type but widening the input might make
3262 // it an illegal type that might lead to repeatedly splitting the input
3263 // and then widening it. To avoid this, we widen the input only if
3264 // it results in a legal type.
3265 if (WidenNumElts % InVTNumElts == 0) {
3266 // Widen the input and call convert on the widened input vector.
3267 unsigned NumConcat = WidenNumElts/InVTNumElts;
3268 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3269 Ops[0] = InOp;
3270 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3271 if (N->getNumOperands() == 1)
3272 return DAG.getNode(Opcode, DL, WidenVT, InVec);
3273 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3274 }
3275
3276 if (InVTNumElts % WidenNumElts == 0) {
3277 SDValue InVal = DAG.getNode(
3278 ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3279 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3280 // Extract the input and convert the shorten input vector.
3281 if (N->getNumOperands() == 1)
3282 return DAG.getNode(Opcode, DL, WidenVT, InVal);
3283 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3284 }
3285 }
3286
3287 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3288 EVT EltVT = WidenVT.getVectorElementType();
3289 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3290 // Use the original element count so we don't do more scalar opts than
3291 // necessary.
3292 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3293 for (unsigned i=0; i < MinElts; ++i) {
3294 SDValue Val = DAG.getNode(
3295 ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3296 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3297 if (N->getNumOperands() == 1)
3298 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3299 else
3300 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3301 }
3302
3303 return DAG.getBuildVector(WidenVT, DL, Ops);
3304 }
3305
WidenVecRes_Convert_StrictFP(SDNode * N)3306 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3307 SDValue InOp = N->getOperand(1);
3308 SDLoc DL(N);
3309 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3310
3311 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3312 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3313 SmallVector<EVT, 2> WidenVTs = { WidenVT, MVT::Other };
3314
3315 EVT InVT = InOp.getValueType();
3316 EVT InEltVT = InVT.getVectorElementType();
3317
3318 unsigned Opcode = N->getOpcode();
3319
3320 // FIXME: Optimizations need to be implemented here.
3321
3322 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3323 EVT EltVT = WidenVT.getVectorElementType();
3324 SmallVector<EVT, 2> EltVTs = { EltVT, MVT::Other };
3325 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3326 SmallVector<SDValue, 32> OpChains;
3327 // Use the original element count so we don't do more scalar opts than
3328 // necessary.
3329 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3330 for (unsigned i=0; i < MinElts; ++i) {
3331 NewOps[1] = DAG.getNode(
3332 ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3333 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3334 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3335 OpChains.push_back(Ops[i].getValue(1));
3336 }
3337 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3338 ReplaceValueWith(SDValue(N, 1), NewChain);
3339
3340 return DAG.getBuildVector(WidenVT, DL, Ops);
3341 }
3342
WidenVecRes_EXTEND_VECTOR_INREG(SDNode * N)3343 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3344 unsigned Opcode = N->getOpcode();
3345 SDValue InOp = N->getOperand(0);
3346 SDLoc DL(N);
3347
3348 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3349 EVT WidenSVT = WidenVT.getVectorElementType();
3350 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3351
3352 EVT InVT = InOp.getValueType();
3353 EVT InSVT = InVT.getVectorElementType();
3354 unsigned InVTNumElts = InVT.getVectorNumElements();
3355
3356 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3357 InOp = GetWidenedVector(InOp);
3358 InVT = InOp.getValueType();
3359 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3360 switch (Opcode) {
3361 case ISD::ANY_EXTEND_VECTOR_INREG:
3362 case ISD::SIGN_EXTEND_VECTOR_INREG:
3363 case ISD::ZERO_EXTEND_VECTOR_INREG:
3364 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3365 }
3366 }
3367 }
3368
3369 // Unroll, extend the scalars and rebuild the vector.
3370 SmallVector<SDValue, 16> Ops;
3371 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3372 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3373 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3374 switch (Opcode) {
3375 case ISD::ANY_EXTEND_VECTOR_INREG:
3376 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3377 break;
3378 case ISD::SIGN_EXTEND_VECTOR_INREG:
3379 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3380 break;
3381 case ISD::ZERO_EXTEND_VECTOR_INREG:
3382 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3383 break;
3384 default:
3385 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3386 }
3387 Ops.push_back(Val);
3388 }
3389
3390 while (Ops.size() != WidenNumElts)
3391 Ops.push_back(DAG.getUNDEF(WidenSVT));
3392
3393 return DAG.getBuildVector(WidenVT, DL, Ops);
3394 }
3395
WidenVecRes_FCOPYSIGN(SDNode * N)3396 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3397 // If this is an FCOPYSIGN with same input types, we can treat it as a
3398 // normal (can trap) binary op.
3399 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3400 return WidenVecRes_BinaryCanTrap(N);
3401
3402 // If the types are different, fall back to unrolling.
3403 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3404 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3405 }
3406
WidenVecRes_POWI(SDNode * N)3407 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3408 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3409 SDValue InOp = GetWidenedVector(N->getOperand(0));
3410 SDValue ShOp = N->getOperand(1);
3411 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3412 }
3413
WidenVecRes_Shift(SDNode * N)3414 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
3415 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3416 SDValue InOp = GetWidenedVector(N->getOperand(0));
3417 SDValue ShOp = N->getOperand(1);
3418
3419 EVT ShVT = ShOp.getValueType();
3420 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
3421 ShOp = GetWidenedVector(ShOp);
3422 ShVT = ShOp.getValueType();
3423 }
3424 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
3425 ShVT.getVectorElementType(),
3426 WidenVT.getVectorNumElements());
3427 if (ShVT != ShWidenVT)
3428 ShOp = ModifyToType(ShOp, ShWidenVT);
3429
3430 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3431 }
3432
WidenVecRes_Unary(SDNode * N)3433 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3434 // Unary op widening.
3435 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3436 SDValue InOp = GetWidenedVector(N->getOperand(0));
3437 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3438 }
3439
WidenVecRes_InregOp(SDNode * N)3440 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3441 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3442 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3443 cast<VTSDNode>(N->getOperand(1))->getVT()
3444 .getVectorElementType(),
3445 WidenVT.getVectorNumElements());
3446 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3447 return DAG.getNode(N->getOpcode(), SDLoc(N),
3448 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3449 }
3450
WidenVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)3451 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3452 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3453 return GetWidenedVector(WidenVec);
3454 }
3455
WidenVecRes_BITCAST(SDNode * N)3456 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3457 SDValue InOp = N->getOperand(0);
3458 EVT InVT = InOp.getValueType();
3459 EVT VT = N->getValueType(0);
3460 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3461 SDLoc dl(N);
3462
3463 switch (getTypeAction(InVT)) {
3464 case TargetLowering::TypeLegal:
3465 break;
3466 case TargetLowering::TypePromoteInteger: {
3467 // If the incoming type is a vector that is being promoted, then
3468 // we know that the elements are arranged differently and that we
3469 // must perform the conversion using a stack slot.
3470 if (InVT.isVector())
3471 break;
3472
3473 // If the InOp is promoted to the same size, convert it. Otherwise,
3474 // fall out of the switch and widen the promoted input.
3475 SDValue NInOp = GetPromotedInteger(InOp);
3476 EVT NInVT = NInOp.getValueType();
3477 if (WidenVT.bitsEq(NInVT)) {
3478 // For big endian targets we need to shift the input integer or the
3479 // interesting bits will end up at the wrong place.
3480 if (DAG.getDataLayout().isBigEndian()) {
3481 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
3482 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
3483 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
3484 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
3485 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
3486 }
3487 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
3488 }
3489 InOp = NInOp;
3490 InVT = NInVT;
3491 break;
3492 }
3493 case TargetLowering::TypeSoftenFloat:
3494 case TargetLowering::TypePromoteFloat:
3495 case TargetLowering::TypeExpandInteger:
3496 case TargetLowering::TypeExpandFloat:
3497 case TargetLowering::TypeScalarizeVector:
3498 case TargetLowering::TypeSplitVector:
3499 break;
3500 case TargetLowering::TypeWidenVector:
3501 // If the InOp is widened to the same size, convert it. Otherwise, fall
3502 // out of the switch and widen the widened input.
3503 InOp = GetWidenedVector(InOp);
3504 InVT = InOp.getValueType();
3505 if (WidenVT.bitsEq(InVT))
3506 // The input widens to the same size. Convert to the widen value.
3507 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3508 break;
3509 }
3510
3511 unsigned WidenSize = WidenVT.getSizeInBits();
3512 unsigned InSize = InVT.getSizeInBits();
3513 // x86mmx is not an acceptable vector element type, so don't try.
3514 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3515 // Determine new input vector type. The new input vector type will use
3516 // the same element type (if its a vector) or use the input type as a
3517 // vector. It is the same size as the type to widen to.
3518 EVT NewInVT;
3519 unsigned NewNumElts = WidenSize / InSize;
3520 if (InVT.isVector()) {
3521 EVT InEltVT = InVT.getVectorElementType();
3522 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3523 WidenSize / InEltVT.getSizeInBits());
3524 } else {
3525 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3526 }
3527
3528 if (TLI.isTypeLegal(NewInVT)) {
3529 SDValue NewVec;
3530 if (InVT.isVector()) {
3531 // Because the result and the input are different vector types, widening
3532 // the result could create a legal type but widening the input might make
3533 // it an illegal type that might lead to repeatedly splitting the input
3534 // and then widening it. To avoid this, we widen the input only if
3535 // it results in a legal type.
3536 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3537 Ops[0] = InOp;
3538
3539 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3540 } else {
3541 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3542 }
3543 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3544 }
3545 }
3546
3547 return CreateStackStoreLoad(InOp, WidenVT);
3548 }
3549
WidenVecRes_BUILD_VECTOR(SDNode * N)3550 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3551 SDLoc dl(N);
3552 // Build a vector with undefined for the new nodes.
3553 EVT VT = N->getValueType(0);
3554
3555 // Integer BUILD_VECTOR operands may be larger than the node's vector element
3556 // type. The UNDEFs need to have the same type as the existing operands.
3557 EVT EltVT = N->getOperand(0).getValueType();
3558 unsigned NumElts = VT.getVectorNumElements();
3559
3560 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3561 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3562
3563 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3564 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3565 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3566
3567 return DAG.getBuildVector(WidenVT, dl, NewOps);
3568 }
3569
WidenVecRes_CONCAT_VECTORS(SDNode * N)3570 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3571 EVT InVT = N->getOperand(0).getValueType();
3572 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3573 SDLoc dl(N);
3574 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3575 unsigned NumInElts = InVT.getVectorNumElements();
3576 unsigned NumOperands = N->getNumOperands();
3577
3578 bool InputWidened = false; // Indicates we need to widen the input.
3579 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3580 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
3581 // Add undef vectors to widen to correct length.
3582 unsigned NumConcat = WidenVT.getVectorNumElements() /
3583 InVT.getVectorNumElements();
3584 SDValue UndefVal = DAG.getUNDEF(InVT);
3585 SmallVector<SDValue, 16> Ops(NumConcat);
3586 for (unsigned i=0; i < NumOperands; ++i)
3587 Ops[i] = N->getOperand(i);
3588 for (unsigned i = NumOperands; i != NumConcat; ++i)
3589 Ops[i] = UndefVal;
3590 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3591 }
3592 } else {
3593 InputWidened = true;
3594 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3595 // The inputs and the result are widen to the same value.
3596 unsigned i;
3597 for (i=1; i < NumOperands; ++i)
3598 if (!N->getOperand(i).isUndef())
3599 break;
3600
3601 if (i == NumOperands)
3602 // Everything but the first operand is an UNDEF so just return the
3603 // widened first operand.
3604 return GetWidenedVector(N->getOperand(0));
3605
3606 if (NumOperands == 2) {
3607 // Replace concat of two operands with a shuffle.
3608 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3609 for (unsigned i = 0; i < NumInElts; ++i) {
3610 MaskOps[i] = i;
3611 MaskOps[i + NumInElts] = i + WidenNumElts;
3612 }
3613 return DAG.getVectorShuffle(WidenVT, dl,
3614 GetWidenedVector(N->getOperand(0)),
3615 GetWidenedVector(N->getOperand(1)),
3616 MaskOps);
3617 }
3618 }
3619 }
3620
3621 // Fall back to use extracts and build vector.
3622 EVT EltVT = WidenVT.getVectorElementType();
3623 SmallVector<SDValue, 16> Ops(WidenNumElts);
3624 unsigned Idx = 0;
3625 for (unsigned i=0; i < NumOperands; ++i) {
3626 SDValue InOp = N->getOperand(i);
3627 if (InputWidened)
3628 InOp = GetWidenedVector(InOp);
3629 for (unsigned j=0; j < NumInElts; ++j)
3630 Ops[Idx++] = DAG.getNode(
3631 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3632 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3633 }
3634 SDValue UndefVal = DAG.getUNDEF(EltVT);
3635 for (; Idx < WidenNumElts; ++Idx)
3636 Ops[Idx] = UndefVal;
3637 return DAG.getBuildVector(WidenVT, dl, Ops);
3638 }
3639
WidenVecRes_EXTRACT_SUBVECTOR(SDNode * N)3640 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3641 EVT VT = N->getValueType(0);
3642 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3643 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3644 SDValue InOp = N->getOperand(0);
3645 SDValue Idx = N->getOperand(1);
3646 SDLoc dl(N);
3647
3648 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3649 InOp = GetWidenedVector(InOp);
3650
3651 EVT InVT = InOp.getValueType();
3652
3653 // Check if we can just return the input vector after widening.
3654 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3655 if (IdxVal == 0 && InVT == WidenVT)
3656 return InOp;
3657
3658 // Check if we can extract from the vector.
3659 unsigned InNumElts = InVT.getVectorNumElements();
3660 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3661 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3662
3663 // We could try widening the input to the right length but for now, extract
3664 // the original elements, fill the rest with undefs and build a vector.
3665 SmallVector<SDValue, 16> Ops(WidenNumElts);
3666 EVT EltVT = VT.getVectorElementType();
3667 unsigned NumElts = VT.getVectorNumElements();
3668 unsigned i;
3669 for (i=0; i < NumElts; ++i)
3670 Ops[i] =
3671 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3672 DAG.getConstant(IdxVal + i, dl,
3673 TLI.getVectorIdxTy(DAG.getDataLayout())));
3674
3675 SDValue UndefVal = DAG.getUNDEF(EltVT);
3676 for (; i < WidenNumElts; ++i)
3677 Ops[i] = UndefVal;
3678 return DAG.getBuildVector(WidenVT, dl, Ops);
3679 }
3680
WidenVecRes_INSERT_VECTOR_ELT(SDNode * N)3681 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3682 SDValue InOp = GetWidenedVector(N->getOperand(0));
3683 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3684 InOp.getValueType(), InOp,
3685 N->getOperand(1), N->getOperand(2));
3686 }
3687
WidenVecRes_LOAD(SDNode * N)3688 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3689 LoadSDNode *LD = cast<LoadSDNode>(N);
3690 ISD::LoadExtType ExtType = LD->getExtensionType();
3691
3692 SDValue Result;
3693 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3694 if (ExtType != ISD::NON_EXTLOAD)
3695 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
3696 else
3697 Result = GenWidenVectorLoads(LdChain, LD);
3698
3699 // If we generate a single load, we can use that for the chain. Otherwise,
3700 // build a factor node to remember the multiple loads are independent and
3701 // chain to that.
3702 SDValue NewChain;
3703 if (LdChain.size() == 1)
3704 NewChain = LdChain[0];
3705 else
3706 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3707
3708 // Modified the chain - switch anything that used the old chain to use
3709 // the new one.
3710 ReplaceValueWith(SDValue(N, 1), NewChain);
3711
3712 return Result;
3713 }
3714
WidenVecRes_MLOAD(MaskedLoadSDNode * N)3715 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3716
3717 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3718 SDValue Mask = N->getMask();
3719 EVT MaskVT = Mask.getValueType();
3720 SDValue PassThru = GetWidenedVector(N->getPassThru());
3721 ISD::LoadExtType ExtType = N->getExtensionType();
3722 SDLoc dl(N);
3723
3724 // The mask should be widened as well
3725 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3726 MaskVT.getVectorElementType(),
3727 WidenVT.getVectorNumElements());
3728 Mask = ModifyToType(Mask, WideMaskVT, true);
3729
3730 SDValue Res = DAG.getMaskedLoad(
3731 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
3732 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
3733 ExtType, N->isExpandingLoad());
3734 // Legalize the chain result - switch anything that used the old chain to
3735 // use the new one.
3736 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3737 return Res;
3738 }
3739
WidenVecRes_MGATHER(MaskedGatherSDNode * N)3740 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3741
3742 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3743 SDValue Mask = N->getMask();
3744 EVT MaskVT = Mask.getValueType();
3745 SDValue PassThru = GetWidenedVector(N->getPassThru());
3746 SDValue Scale = N->getScale();
3747 unsigned NumElts = WideVT.getVectorNumElements();
3748 SDLoc dl(N);
3749
3750 // The mask should be widened as well
3751 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3752 MaskVT.getVectorElementType(),
3753 WideVT.getVectorNumElements());
3754 Mask = ModifyToType(Mask, WideMaskVT, true);
3755
3756 // Widen the Index operand
3757 SDValue Index = N->getIndex();
3758 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3759 Index.getValueType().getScalarType(),
3760 NumElts);
3761 Index = ModifyToType(Index, WideIndexVT);
3762 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3763 Scale };
3764 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3765 N->getMemoryVT(), dl, Ops,
3766 N->getMemOperand(), N->getIndexType());
3767
3768 // Legalize the chain result - switch anything that used the old chain to
3769 // use the new one.
3770 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3771 return Res;
3772 }
3773
WidenVecRes_SCALAR_TO_VECTOR(SDNode * N)3774 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3775 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3776 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
3777 WidenVT, N->getOperand(0));
3778 }
3779
3780 // Return true is this is a SETCC node or a strict version of it.
isSETCCOp(unsigned Opcode)3781 static inline bool isSETCCOp(unsigned Opcode) {
3782 switch (Opcode) {
3783 case ISD::SETCC:
3784 case ISD::STRICT_FSETCC:
3785 case ISD::STRICT_FSETCCS:
3786 return true;
3787 }
3788 return false;
3789 }
3790
3791 // Return true if this is a node that could have two SETCCs as operands.
isLogicalMaskOp(unsigned Opcode)3792 static inline bool isLogicalMaskOp(unsigned Opcode) {
3793 switch (Opcode) {
3794 case ISD::AND:
3795 case ISD::OR:
3796 case ISD::XOR:
3797 return true;
3798 }
3799 return false;
3800 }
3801
3802 // If N is a SETCC or a strict variant of it, return the type
3803 // of the compare operands.
getSETCCOperandType(SDValue N)3804 static inline EVT getSETCCOperandType(SDValue N) {
3805 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
3806 return N->getOperand(OpNo).getValueType();
3807 }
3808
3809 // This is used just for the assert in convertMask(). Check that this either
3810 // a SETCC or a previously handled SETCC by convertMask().
3811 #ifndef NDEBUG
isSETCCorConvertedSETCC(SDValue N)3812 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3813 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3814 N = N.getOperand(0);
3815 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3816 for (unsigned i = 1; i < N->getNumOperands(); ++i)
3817 if (!N->getOperand(i)->isUndef())
3818 return false;
3819 N = N.getOperand(0);
3820 }
3821
3822 if (N.getOpcode() == ISD::TRUNCATE)
3823 N = N.getOperand(0);
3824 else if (N.getOpcode() == ISD::SIGN_EXTEND)
3825 N = N.getOperand(0);
3826
3827 if (isLogicalMaskOp(N.getOpcode()))
3828 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3829 isSETCCorConvertedSETCC(N.getOperand(1));
3830
3831 return (isSETCCOp(N.getOpcode()) ||
3832 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
3833 }
3834 #endif
3835
3836 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3837 // to ToMaskVT if needed with vector extension or truncation.
convertMask(SDValue InMask,EVT MaskVT,EVT ToMaskVT)3838 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3839 EVT ToMaskVT) {
3840 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3841 // FIXME: This code seems to be too restrictive, we might consider
3842 // generalizing it or dropping it.
3843 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3844
3845 // Make a new Mask node, with a legal result VT.
3846 SDValue Mask;
3847 SmallVector<SDValue, 4> Ops;
3848 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
3849 Ops.push_back(InMask->getOperand(i));
3850 if (InMask->isStrictFPOpcode()) {
3851 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
3852 { MaskVT, MVT::Other }, Ops);
3853 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
3854 }
3855 else
3856 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3857
3858 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3859 // extend or truncate is needed.
3860 LLVMContext &Ctx = *DAG.getContext();
3861 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3862 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3863 if (MaskScalarBits < ToMaskScalBits) {
3864 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3865 MaskVT.getVectorNumElements());
3866 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3867 } else if (MaskScalarBits > ToMaskScalBits) {
3868 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3869 MaskVT.getVectorNumElements());
3870 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3871 }
3872
3873 assert(Mask->getValueType(0).getScalarSizeInBits() ==
3874 ToMaskVT.getScalarSizeInBits() &&
3875 "Mask should have the right element size by now.");
3876
3877 // Adjust Mask to the right number of elements.
3878 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3879 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3880 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3881 SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3882 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3883 ZeroIdx);
3884 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3885 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3886 EVT SubVT = Mask->getValueType(0);
3887 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3888 SubOps[0] = Mask;
3889 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3890 }
3891
3892 assert((Mask->getValueType(0) == ToMaskVT) &&
3893 "A mask of ToMaskVT should have been produced by now.");
3894
3895 return Mask;
3896 }
3897
3898 // This method tries to handle VSELECT and its mask by legalizing operands
3899 // (which may require widening) and if needed adjusting the mask vector type
3900 // to match that of the VSELECT. Without it, many cases end up with
3901 // scalarization of the SETCC, with many unnecessary instructions.
WidenVSELECTAndMask(SDNode * N)3902 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3903 LLVMContext &Ctx = *DAG.getContext();
3904 SDValue Cond = N->getOperand(0);
3905
3906 if (N->getOpcode() != ISD::VSELECT)
3907 return SDValue();
3908
3909 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
3910 return SDValue();
3911
3912 // If this is a splitted VSELECT that was previously already handled, do
3913 // nothing.
3914 EVT CondVT = Cond->getValueType(0);
3915 if (CondVT.getScalarSizeInBits() != 1)
3916 return SDValue();
3917
3918 EVT VSelVT = N->getValueType(0);
3919 // Only handle vector types which are a power of 2.
3920 if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3921 return SDValue();
3922
3923 // Don't touch if this will be scalarized.
3924 EVT FinalVT = VSelVT;
3925 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3926 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3927
3928 if (FinalVT.getVectorNumElements() == 1)
3929 return SDValue();
3930
3931 // If there is support for an i1 vector mask, don't touch.
3932 if (isSETCCOp(Cond.getOpcode())) {
3933 EVT SetCCOpVT = getSETCCOperandType(Cond);
3934 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3935 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3936 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3937 if (SetCCResVT.getScalarSizeInBits() == 1)
3938 return SDValue();
3939 } else if (CondVT.getScalarType() == MVT::i1) {
3940 // If there is support for an i1 vector mask (or only scalar i1 conditions),
3941 // don't touch.
3942 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3943 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3944
3945 if (CondVT.getScalarType() == MVT::i1)
3946 return SDValue();
3947 }
3948
3949 // Get the VT and operands for VSELECT, and widen if needed.
3950 SDValue VSelOp1 = N->getOperand(1);
3951 SDValue VSelOp2 = N->getOperand(2);
3952 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3953 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3954 VSelOp1 = GetWidenedVector(VSelOp1);
3955 VSelOp2 = GetWidenedVector(VSelOp2);
3956 }
3957
3958 // The mask of the VSELECT should have integer elements.
3959 EVT ToMaskVT = VSelVT;
3960 if (!ToMaskVT.getScalarType().isInteger())
3961 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3962
3963 SDValue Mask;
3964 if (isSETCCOp(Cond->getOpcode())) {
3965 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
3966 Mask = convertMask(Cond, MaskVT, ToMaskVT);
3967 } else if (isLogicalMaskOp(Cond->getOpcode()) &&
3968 isSETCCOp(Cond->getOperand(0).getOpcode()) &&
3969 isSETCCOp(Cond->getOperand(1).getOpcode())) {
3970 // Cond is (AND/OR/XOR (SETCC, SETCC))
3971 SDValue SETCC0 = Cond->getOperand(0);
3972 SDValue SETCC1 = Cond->getOperand(1);
3973 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
3974 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
3975 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3976 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3977 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3978 EVT MaskVT;
3979 // If the two SETCCs have different VTs, either extend/truncate one of
3980 // them to the other "towards" ToMaskVT, or truncate one and extend the
3981 // other to ToMaskVT.
3982 if (ScalarBits0 != ScalarBits1) {
3983 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3984 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3985 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3986 MaskVT = WideVT;
3987 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3988 MaskVT = NarrowVT;
3989 else
3990 MaskVT = ToMaskVT;
3991 } else
3992 // If the two SETCCs have the same VT, don't change it.
3993 MaskVT = VT0;
3994
3995 // Make new SETCCs and logical nodes.
3996 SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3997 SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3998 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3999
4000 // Convert the logical op for VSELECT if needed.
4001 Mask = convertMask(Cond, MaskVT, ToMaskVT);
4002 } else
4003 return SDValue();
4004
4005 return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
4006 }
4007
WidenVecRes_SELECT(SDNode * N)4008 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
4009 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4010 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4011
4012 SDValue Cond1 = N->getOperand(0);
4013 EVT CondVT = Cond1.getValueType();
4014 if (CondVT.isVector()) {
4015 if (SDValue Res = WidenVSELECTAndMask(N))
4016 return Res;
4017
4018 EVT CondEltVT = CondVT.getVectorElementType();
4019 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
4020 CondEltVT, WidenNumElts);
4021 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
4022 Cond1 = GetWidenedVector(Cond1);
4023
4024 // If we have to split the condition there is no point in widening the
4025 // select. This would result in an cycle of widening the select ->
4026 // widening the condition operand -> splitting the condition operand ->
4027 // splitting the select -> widening the select. Instead split this select
4028 // further and widen the resulting type.
4029 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
4030 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
4031 SDValue Res = ModifyToType(SplitSelect, WidenVT);
4032 return Res;
4033 }
4034
4035 if (Cond1.getValueType() != CondWidenVT)
4036 Cond1 = ModifyToType(Cond1, CondWidenVT);
4037 }
4038
4039 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4040 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4041 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4042 return DAG.getNode(N->getOpcode(), SDLoc(N),
4043 WidenVT, Cond1, InOp1, InOp2);
4044 }
4045
WidenVecRes_SELECT_CC(SDNode * N)4046 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
4047 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
4048 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
4049 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
4050 InOp1.getValueType(), N->getOperand(0),
4051 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
4052 }
4053
WidenVecRes_UNDEF(SDNode * N)4054 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
4055 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4056 return DAG.getUNDEF(WidenVT);
4057 }
4058
WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N)4059 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
4060 EVT VT = N->getValueType(0);
4061 SDLoc dl(N);
4062
4063 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4064 unsigned NumElts = VT.getVectorNumElements();
4065 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4066
4067 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4068 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4069
4070 // Adjust mask based on new input vector length.
4071 SmallVector<int, 16> NewMask;
4072 for (unsigned i = 0; i != NumElts; ++i) {
4073 int Idx = N->getMaskElt(i);
4074 if (Idx < (int)NumElts)
4075 NewMask.push_back(Idx);
4076 else
4077 NewMask.push_back(Idx - NumElts + WidenNumElts);
4078 }
4079 for (unsigned i = NumElts; i != WidenNumElts; ++i)
4080 NewMask.push_back(-1);
4081 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
4082 }
4083
WidenVecRes_SETCC(SDNode * N)4084 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4085 assert(N->getValueType(0).isVector() &&
4086 N->getOperand(0).getValueType().isVector() &&
4087 "Operands must be vectors");
4088 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4089 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4090
4091 SDValue InOp1 = N->getOperand(0);
4092 EVT InVT = InOp1.getValueType();
4093 assert(InVT.isVector() && "can not widen non-vector type");
4094 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
4095 InVT.getVectorElementType(), WidenNumElts);
4096
4097 // The input and output types often differ here, and it could be that while
4098 // we'd prefer to widen the result type, the input operands have been split.
4099 // In this case, we also need to split the result of this node as well.
4100 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
4101 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4102 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4103 return Res;
4104 }
4105
4106 // If the inputs also widen, handle them directly. Otherwise widen by hand.
4107 SDValue InOp2 = N->getOperand(1);
4108 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4109 InOp1 = GetWidenedVector(InOp1);
4110 InOp2 = GetWidenedVector(InOp2);
4111 } else {
4112 InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
4113 InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
4114 }
4115
4116 // Assume that the input and output will be widen appropriately. If not,
4117 // we will have to unroll it at some point.
4118 assert(InOp1.getValueType() == WidenInVT &&
4119 InOp2.getValueType() == WidenInVT &&
4120 "Input not widened to expected type!");
4121 (void)WidenInVT;
4122 return DAG.getNode(ISD::SETCC, SDLoc(N),
4123 WidenVT, InOp1, InOp2, N->getOperand(2));
4124 }
4125
WidenVecRes_STRICT_FSETCC(SDNode * N)4126 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
4127 assert(N->getValueType(0).isVector() &&
4128 N->getOperand(1).getValueType().isVector() &&
4129 "Operands must be vectors");
4130 EVT VT = N->getValueType(0);
4131 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4132 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4133 unsigned NumElts = VT.getVectorNumElements();
4134 EVT EltVT = VT.getVectorElementType();
4135
4136 SDLoc dl(N);
4137 SDValue Chain = N->getOperand(0);
4138 SDValue LHS = N->getOperand(1);
4139 SDValue RHS = N->getOperand(2);
4140 SDValue CC = N->getOperand(3);
4141 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
4142
4143 // Fully unroll and reassemble.
4144 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT));
4145 SmallVector<SDValue, 8> Chains(NumElts);
4146 for (unsigned i = 0; i != NumElts; ++i) {
4147 SDValue LHSElem = DAG.getNode(
4148 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
4149 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4150 SDValue RHSElem = DAG.getNode(
4151 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
4152 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4153
4154 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
4155 {Chain, LHSElem, RHSElem, CC});
4156 Chains[i] = Scalars[i].getValue(1);
4157 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
4158 DAG.getBoolConstant(true, dl, EltVT, VT),
4159 DAG.getBoolConstant(false, dl, EltVT, VT));
4160 }
4161
4162 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4163 ReplaceValueWith(SDValue(N, 1), NewChain);
4164
4165 return DAG.getBuildVector(WidenVT, dl, Scalars);
4166 }
4167
4168 //===----------------------------------------------------------------------===//
4169 // Widen Vector Operand
4170 //===----------------------------------------------------------------------===//
WidenVectorOperand(SDNode * N,unsigned OpNo)4171 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
4172 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
4173 dbgs() << "\n");
4174 SDValue Res = SDValue();
4175
4176 // See if the target wants to custom widen this node.
4177 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4178 return false;
4179
4180 switch (N->getOpcode()) {
4181 default:
4182 #ifndef NDEBUG
4183 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
4184 N->dump(&DAG);
4185 dbgs() << "\n";
4186 #endif
4187 llvm_unreachable("Do not know how to widen this operator's operand!");
4188
4189 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
4190 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
4191 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
4192 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
4193 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
4194 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
4195 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
4196 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
4197 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
4198 case ISD::STRICT_FSETCC:
4199 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
4200 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
4201 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
4202
4203 case ISD::ANY_EXTEND:
4204 case ISD::SIGN_EXTEND:
4205 case ISD::ZERO_EXTEND:
4206 Res = WidenVecOp_EXTEND(N);
4207 break;
4208
4209 case ISD::FP_EXTEND:
4210 case ISD::STRICT_FP_EXTEND:
4211 case ISD::FP_ROUND:
4212 case ISD::STRICT_FP_ROUND:
4213 case ISD::FP_TO_SINT:
4214 case ISD::STRICT_FP_TO_SINT:
4215 case ISD::FP_TO_UINT:
4216 case ISD::STRICT_FP_TO_UINT:
4217 case ISD::SINT_TO_FP:
4218 case ISD::STRICT_SINT_TO_FP:
4219 case ISD::UINT_TO_FP:
4220 case ISD::STRICT_UINT_TO_FP:
4221 case ISD::TRUNCATE:
4222 Res = WidenVecOp_Convert(N);
4223 break;
4224
4225 case ISD::VECREDUCE_FADD:
4226 case ISD::VECREDUCE_FMUL:
4227 case ISD::VECREDUCE_ADD:
4228 case ISD::VECREDUCE_MUL:
4229 case ISD::VECREDUCE_AND:
4230 case ISD::VECREDUCE_OR:
4231 case ISD::VECREDUCE_XOR:
4232 case ISD::VECREDUCE_SMAX:
4233 case ISD::VECREDUCE_SMIN:
4234 case ISD::VECREDUCE_UMAX:
4235 case ISD::VECREDUCE_UMIN:
4236 case ISD::VECREDUCE_FMAX:
4237 case ISD::VECREDUCE_FMIN:
4238 Res = WidenVecOp_VECREDUCE(N);
4239 break;
4240 }
4241
4242 // If Res is null, the sub-method took care of registering the result.
4243 if (!Res.getNode()) return false;
4244
4245 // If the result is N, the sub-method updated N in place. Tell the legalizer
4246 // core about this.
4247 if (Res.getNode() == N)
4248 return true;
4249
4250
4251 if (N->isStrictFPOpcode())
4252 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4253 "Invalid operand expansion");
4254 else
4255 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4256 "Invalid operand expansion");
4257
4258 ReplaceValueWith(SDValue(N, 0), Res);
4259 return false;
4260 }
4261
WidenVecOp_EXTEND(SDNode * N)4262 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4263 SDLoc DL(N);
4264 EVT VT = N->getValueType(0);
4265
4266 SDValue InOp = N->getOperand(0);
4267 assert(getTypeAction(InOp.getValueType()) ==
4268 TargetLowering::TypeWidenVector &&
4269 "Unexpected type action");
4270 InOp = GetWidenedVector(InOp);
4271 assert(VT.getVectorNumElements() <
4272 InOp.getValueType().getVectorNumElements() &&
4273 "Input wasn't widened!");
4274
4275 // We may need to further widen the operand until it has the same total
4276 // vector size as the result.
4277 EVT InVT = InOp.getValueType();
4278 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
4279 EVT InEltVT = InVT.getVectorElementType();
4280 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
4281 EVT FixedVT = (MVT::SimpleValueType)i;
4282 EVT FixedEltVT = FixedVT.getVectorElementType();
4283 if (TLI.isTypeLegal(FixedVT) &&
4284 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
4285 FixedEltVT == InEltVT) {
4286 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
4287 "Not enough elements in the fixed type for the operand!");
4288 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
4289 "We can't have the same type as we started with!");
4290 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
4291 InOp = DAG.getNode(
4292 ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
4293 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4294 else
4295 InOp = DAG.getNode(
4296 ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
4297 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4298 break;
4299 }
4300 }
4301 InVT = InOp.getValueType();
4302 if (InVT.getSizeInBits() != VT.getSizeInBits())
4303 // We couldn't find a legal vector type that was a widening of the input
4304 // and could be extended in-register to the result type, so we have to
4305 // scalarize.
4306 return WidenVecOp_Convert(N);
4307 }
4308
4309 // Use special DAG nodes to represent the operation of extending the
4310 // low lanes.
4311 switch (N->getOpcode()) {
4312 default:
4313 llvm_unreachable("Extend legalization on extend operation!");
4314 case ISD::ANY_EXTEND:
4315 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
4316 case ISD::SIGN_EXTEND:
4317 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
4318 case ISD::ZERO_EXTEND:
4319 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
4320 }
4321 }
4322
WidenVecOp_FCOPYSIGN(SDNode * N)4323 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4324 // The result (and first input) is legal, but the second input is illegal.
4325 // We can't do much to fix that, so just unroll and let the extracts off of
4326 // the second input be widened as needed later.
4327 return DAG.UnrollVectorOp(N);
4328 }
4329
WidenVecOp_Convert(SDNode * N)4330 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4331 // Since the result is legal and the input is illegal.
4332 EVT VT = N->getValueType(0);
4333 EVT EltVT = VT.getVectorElementType();
4334 SDLoc dl(N);
4335 unsigned NumElts = VT.getVectorNumElements();
4336 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
4337 assert(getTypeAction(InOp.getValueType()) ==
4338 TargetLowering::TypeWidenVector &&
4339 "Unexpected type action");
4340 InOp = GetWidenedVector(InOp);
4341 EVT InVT = InOp.getValueType();
4342 unsigned Opcode = N->getOpcode();
4343
4344 // See if a widened result type would be legal, if so widen the node.
4345 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
4346 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4347 InVT.getVectorNumElements());
4348 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
4349 SDValue Res;
4350 if (N->isStrictFPOpcode()) {
4351 if (Opcode == ISD::STRICT_FP_ROUND)
4352 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4353 { N->getOperand(0), InOp, N->getOperand(2) });
4354 else
4355 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4356 { N->getOperand(0), InOp });
4357 // Legalize the chain result - switch anything that used the old chain to
4358 // use the new one.
4359 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4360 } else {
4361 if (Opcode == ISD::FP_ROUND)
4362 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1));
4363 else
4364 Res = DAG.getNode(Opcode, dl, WideVT, InOp);
4365 }
4366 return DAG.getNode(
4367 ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
4368 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4369 }
4370
4371 EVT InEltVT = InVT.getVectorElementType();
4372
4373 // Unroll the convert into some scalar code and create a nasty build vector.
4374 SmallVector<SDValue, 16> Ops(NumElts);
4375 if (N->isStrictFPOpcode()) {
4376 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4377 SmallVector<SDValue, 32> OpChains;
4378 for (unsigned i=0; i < NumElts; ++i) {
4379 NewOps[1] = DAG.getNode(
4380 ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4381 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4382 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
4383 OpChains.push_back(Ops[i].getValue(1));
4384 }
4385 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4386 ReplaceValueWith(SDValue(N, 1), NewChain);
4387 } else {
4388 for (unsigned i = 0; i < NumElts; ++i)
4389 Ops[i] = DAG.getNode(
4390 Opcode, dl, EltVT,
4391 DAG.getNode(
4392 ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4393 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
4394 }
4395
4396 return DAG.getBuildVector(VT, dl, Ops);
4397 }
4398
WidenVecOp_BITCAST(SDNode * N)4399 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4400 EVT VT = N->getValueType(0);
4401 SDValue InOp = GetWidenedVector(N->getOperand(0));
4402 EVT InWidenVT = InOp.getValueType();
4403 SDLoc dl(N);
4404
4405 // Check if we can convert between two legal vector types and extract.
4406 unsigned InWidenSize = InWidenVT.getSizeInBits();
4407 unsigned Size = VT.getSizeInBits();
4408 // x86mmx is not an acceptable vector element type, so don't try.
4409 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
4410 unsigned NewNumElts = InWidenSize / Size;
4411 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4412 if (TLI.isTypeLegal(NewVT)) {
4413 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4414 return DAG.getNode(
4415 ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
4416 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4417 }
4418 }
4419
4420 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
4421 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
4422 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
4423 // having to copy via memory.
4424 if (VT.isVector()) {
4425 EVT EltVT = VT.getVectorElementType();
4426 unsigned EltSize = EltVT.getSizeInBits();
4427 if (InWidenSize % EltSize == 0) {
4428 unsigned NewNumElts = InWidenSize / EltSize;
4429 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
4430 if (TLI.isTypeLegal(NewVT)) {
4431 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4432 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
4433 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4434 }
4435 }
4436 }
4437
4438 return CreateStackStoreLoad(InOp, VT);
4439 }
4440
WidenVecOp_CONCAT_VECTORS(SDNode * N)4441 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4442 EVT VT = N->getValueType(0);
4443 EVT EltVT = VT.getVectorElementType();
4444 EVT InVT = N->getOperand(0).getValueType();
4445 SDLoc dl(N);
4446
4447 // If the widen width for this operand is the same as the width of the concat
4448 // and all but the first operand is undef, just use the widened operand.
4449 unsigned NumOperands = N->getNumOperands();
4450 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4451 unsigned i;
4452 for (i = 1; i < NumOperands; ++i)
4453 if (!N->getOperand(i).isUndef())
4454 break;
4455
4456 if (i == NumOperands)
4457 return GetWidenedVector(N->getOperand(0));
4458 }
4459
4460 // Otherwise, fall back to a nasty build vector.
4461 unsigned NumElts = VT.getVectorNumElements();
4462 SmallVector<SDValue, 16> Ops(NumElts);
4463
4464 unsigned NumInElts = InVT.getVectorNumElements();
4465
4466 unsigned Idx = 0;
4467 for (unsigned i=0; i < NumOperands; ++i) {
4468 SDValue InOp = N->getOperand(i);
4469 assert(getTypeAction(InOp.getValueType()) ==
4470 TargetLowering::TypeWidenVector &&
4471 "Unexpected type action");
4472 InOp = GetWidenedVector(InOp);
4473 for (unsigned j=0; j < NumInElts; ++j)
4474 Ops[Idx++] = DAG.getNode(
4475 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4476 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4477 }
4478 return DAG.getBuildVector(VT, dl, Ops);
4479 }
4480
WidenVecOp_EXTRACT_SUBVECTOR(SDNode * N)4481 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4482 SDValue InOp = GetWidenedVector(N->getOperand(0));
4483 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4484 N->getValueType(0), InOp, N->getOperand(1));
4485 }
4486
WidenVecOp_EXTRACT_VECTOR_ELT(SDNode * N)4487 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4488 SDValue InOp = GetWidenedVector(N->getOperand(0));
4489 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4490 N->getValueType(0), InOp, N->getOperand(1));
4491 }
4492
WidenVecOp_STORE(SDNode * N)4493 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4494 // We have to widen the value, but we want only to store the original
4495 // vector type.
4496 StoreSDNode *ST = cast<StoreSDNode>(N);
4497
4498 if (!ST->getMemoryVT().getScalarType().isByteSized())
4499 return TLI.scalarizeVectorStore(ST, DAG);
4500
4501 SmallVector<SDValue, 16> StChain;
4502 if (ST->isTruncatingStore())
4503 GenWidenVectorTruncStores(StChain, ST);
4504 else
4505 GenWidenVectorStores(StChain, ST);
4506
4507 if (StChain.size() == 1)
4508 return StChain[0];
4509 else
4510 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4511 }
4512
WidenVecOp_MSTORE(SDNode * N,unsigned OpNo)4513 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4514 assert((OpNo == 1 || OpNo == 3) &&
4515 "Can widen only data or mask operand of mstore");
4516 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
4517 SDValue Mask = MST->getMask();
4518 EVT MaskVT = Mask.getValueType();
4519 SDValue StVal = MST->getValue();
4520 SDLoc dl(N);
4521
4522 if (OpNo == 1) {
4523 // Widen the value.
4524 StVal = GetWidenedVector(StVal);
4525
4526 // The mask should be widened as well.
4527 EVT WideVT = StVal.getValueType();
4528 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4529 MaskVT.getVectorElementType(),
4530 WideVT.getVectorNumElements());
4531 Mask = ModifyToType(Mask, WideMaskVT, true);
4532 } else {
4533 // Widen the mask.
4534 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4535 Mask = ModifyToType(Mask, WideMaskVT, true);
4536
4537 EVT ValueVT = StVal.getValueType();
4538 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4539 ValueVT.getVectorElementType(),
4540 WideMaskVT.getVectorNumElements());
4541 StVal = ModifyToType(StVal, WideVT);
4542 }
4543
4544 assert(Mask.getValueType().getVectorNumElements() ==
4545 StVal.getValueType().getVectorNumElements() &&
4546 "Mask and data vectors should have the same number of elements");
4547 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4548 MST->getOffset(), Mask, MST->getMemoryVT(),
4549 MST->getMemOperand(), MST->getAddressingMode(),
4550 false, MST->isCompressingStore());
4551 }
4552
WidenVecOp_MGATHER(SDNode * N,unsigned OpNo)4553 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4554 assert(OpNo == 4 && "Can widen only the index of mgather");
4555 auto *MG = cast<MaskedGatherSDNode>(N);
4556 SDValue DataOp = MG->getPassThru();
4557 SDValue Mask = MG->getMask();
4558 SDValue Scale = MG->getScale();
4559
4560 // Just widen the index. It's allowed to have extra elements.
4561 SDValue Index = GetWidenedVector(MG->getIndex());
4562
4563 SDLoc dl(N);
4564 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4565 Scale};
4566 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4567 MG->getMemOperand(), MG->getIndexType());
4568 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4569 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4570 return SDValue();
4571 }
4572
WidenVecOp_MSCATTER(SDNode * N,unsigned OpNo)4573 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4574 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
4575 SDValue DataOp = MSC->getValue();
4576 SDValue Mask = MSC->getMask();
4577 SDValue Index = MSC->getIndex();
4578 SDValue Scale = MSC->getScale();
4579
4580 if (OpNo == 1) {
4581 DataOp = GetWidenedVector(DataOp);
4582 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
4583
4584 // Widen index.
4585 EVT IndexVT = Index.getValueType();
4586 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4587 IndexVT.getVectorElementType(), NumElts);
4588 Index = ModifyToType(Index, WideIndexVT);
4589
4590 // The mask should be widened as well.
4591 EVT MaskVT = Mask.getValueType();
4592 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4593 MaskVT.getVectorElementType(), NumElts);
4594 Mask = ModifyToType(Mask, WideMaskVT, true);
4595 } else if (OpNo == 4) {
4596 // Just widen the index. It's allowed to have extra elements.
4597 Index = GetWidenedVector(Index);
4598 } else
4599 llvm_unreachable("Can't widen this operand of mscatter");
4600
4601 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4602 Scale};
4603 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
4604 MSC->getMemoryVT(), SDLoc(N), Ops,
4605 MSC->getMemOperand(), MSC->getIndexType());
4606 }
4607
WidenVecOp_SETCC(SDNode * N)4608 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4609 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4610 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4611 SDLoc dl(N);
4612 EVT VT = N->getValueType(0);
4613
4614 // WARNING: In this code we widen the compare instruction with garbage.
4615 // This garbage may contain denormal floats which may be slow. Is this a real
4616 // concern ? Should we zero the unused lanes if this is a float compare ?
4617
4618 // Get a new SETCC node to compare the newly widened operands.
4619 // Only some of the compared elements are legal.
4620 EVT SVT = getSetCCResultType(InOp0.getValueType());
4621 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
4622 if (VT.getScalarType() == MVT::i1)
4623 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4624 SVT.getVectorNumElements());
4625
4626 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4627 SVT, InOp0, InOp1, N->getOperand(2));
4628
4629 // Extract the needed results from the result vector.
4630 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
4631 SVT.getVectorElementType(),
4632 VT.getVectorNumElements());
4633 SDValue CC = DAG.getNode(
4634 ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
4635 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4636
4637 EVT OpVT = N->getOperand(0).getValueType();
4638 ISD::NodeType ExtendCode =
4639 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
4640 return DAG.getNode(ExtendCode, dl, VT, CC);
4641 }
4642
WidenVecOp_STRICT_FSETCC(SDNode * N)4643 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
4644 SDValue Chain = N->getOperand(0);
4645 SDValue LHS = GetWidenedVector(N->getOperand(1));
4646 SDValue RHS = GetWidenedVector(N->getOperand(2));
4647 SDValue CC = N->getOperand(3);
4648 SDLoc dl(N);
4649
4650 EVT VT = N->getValueType(0);
4651 EVT EltVT = VT.getVectorElementType();
4652 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
4653 unsigned NumElts = VT.getVectorNumElements();
4654
4655 // Unroll into a build vector.
4656 SmallVector<SDValue, 8> Scalars(NumElts);
4657 SmallVector<SDValue, 8> Chains(NumElts);
4658
4659 for (unsigned i = 0; i != NumElts; ++i) {
4660 SDValue LHSElem = DAG.getNode(
4661 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
4662 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4663 SDValue RHSElem = DAG.getNode(
4664 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
4665 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4666
4667 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
4668 {Chain, LHSElem, RHSElem, CC});
4669 Chains[i] = Scalars[i].getValue(1);
4670 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
4671 DAG.getBoolConstant(true, dl, EltVT, VT),
4672 DAG.getBoolConstant(false, dl, EltVT, VT));
4673 }
4674
4675 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4676 ReplaceValueWith(SDValue(N, 1), NewChain);
4677
4678 return DAG.getBuildVector(VT, dl, Scalars);
4679 }
4680
WidenVecOp_VECREDUCE(SDNode * N)4681 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
4682 SDLoc dl(N);
4683 SDValue Op = GetWidenedVector(N->getOperand(0));
4684 EVT OrigVT = N->getOperand(0).getValueType();
4685 EVT WideVT = Op.getValueType();
4686 EVT ElemVT = OrigVT.getVectorElementType();
4687
4688 SDValue NeutralElem;
4689 switch (N->getOpcode()) {
4690 case ISD::VECREDUCE_ADD:
4691 case ISD::VECREDUCE_OR:
4692 case ISD::VECREDUCE_XOR:
4693 case ISD::VECREDUCE_UMAX:
4694 NeutralElem = DAG.getConstant(0, dl, ElemVT);
4695 break;
4696 case ISD::VECREDUCE_MUL:
4697 NeutralElem = DAG.getConstant(1, dl, ElemVT);
4698 break;
4699 case ISD::VECREDUCE_AND:
4700 case ISD::VECREDUCE_UMIN:
4701 NeutralElem = DAG.getAllOnesConstant(dl, ElemVT);
4702 break;
4703 case ISD::VECREDUCE_SMAX:
4704 NeutralElem = DAG.getConstant(
4705 APInt::getSignedMinValue(ElemVT.getSizeInBits()), dl, ElemVT);
4706 break;
4707 case ISD::VECREDUCE_SMIN:
4708 NeutralElem = DAG.getConstant(
4709 APInt::getSignedMaxValue(ElemVT.getSizeInBits()), dl, ElemVT);
4710 break;
4711 case ISD::VECREDUCE_FADD:
4712 NeutralElem = DAG.getConstantFP(0.0, dl, ElemVT);
4713 break;
4714 case ISD::VECREDUCE_FMUL:
4715 NeutralElem = DAG.getConstantFP(1.0, dl, ElemVT);
4716 break;
4717 case ISD::VECREDUCE_FMAX:
4718 NeutralElem = DAG.getConstantFP(
4719 -std::numeric_limits<double>::infinity(), dl, ElemVT);
4720 break;
4721 case ISD::VECREDUCE_FMIN:
4722 NeutralElem = DAG.getConstantFP(
4723 std::numeric_limits<double>::infinity(), dl, ElemVT);
4724 break;
4725 }
4726
4727 // Pad the vector with the neutral element.
4728 unsigned OrigElts = OrigVT.getVectorNumElements();
4729 unsigned WideElts = WideVT.getVectorNumElements();
4730 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
4731 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
4732 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4733
4734 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Op, N->getFlags());
4735 }
4736
WidenVecOp_VSELECT(SDNode * N)4737 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
4738 // This only gets called in the case that the left and right inputs and
4739 // result are of a legal odd vector type, and the condition is illegal i1 of
4740 // the same odd width that needs widening.
4741 EVT VT = N->getValueType(0);
4742 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
4743
4744 SDValue Cond = GetWidenedVector(N->getOperand(0));
4745 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
4746 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
4747 SDLoc DL(N);
4748
4749 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
4750 LeftIn, RightIn);
4751 return DAG.getNode(
4752 ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
4753 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4754 }
4755
4756 //===----------------------------------------------------------------------===//
4757 // Vector Widening Utilities
4758 //===----------------------------------------------------------------------===//
4759
4760 // Utility function to find the type to chop up a widen vector for load/store
4761 // TLI: Target lowering used to determine legal types.
4762 // Width: Width left need to load/store.
4763 // WidenVT: The widen vector type to load to/store from
4764 // Align: If 0, don't allow use of a wider type
4765 // WidenEx: If Align is not 0, the amount additional we can load/store from.
4766
FindMemType(SelectionDAG & DAG,const TargetLowering & TLI,unsigned Width,EVT WidenVT,unsigned Align=0,unsigned WidenEx=0)4767 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
4768 unsigned Width, EVT WidenVT,
4769 unsigned Align = 0, unsigned WidenEx = 0) {
4770 EVT WidenEltVT = WidenVT.getVectorElementType();
4771 const bool Scalable = WidenVT.isScalableVector();
4772 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize();
4773 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
4774 unsigned AlignInBits = Align*8;
4775
4776 // If we have one element to load/store, return it.
4777 EVT RetVT = WidenEltVT;
4778 if (Width == WidenEltWidth)
4779 return RetVT;
4780
4781 // See if there is larger legal integer than the element type to load/store.
4782 unsigned VT;
4783 // Don't bother looking for an integer type if the vector is scalable, skip
4784 // to vector types.
4785 if (!Scalable) {
4786 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
4787 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
4788 EVT MemVT((MVT::SimpleValueType) VT);
4789 unsigned MemVTWidth = MemVT.getSizeInBits();
4790 if (MemVT.getSizeInBits() <= WidenEltWidth)
4791 break;
4792 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4793 if ((Action == TargetLowering::TypeLegal ||
4794 Action == TargetLowering::TypePromoteInteger) &&
4795 (WidenWidth % MemVTWidth) == 0 &&
4796 isPowerOf2_32(WidenWidth / MemVTWidth) &&
4797 (MemVTWidth <= Width ||
4798 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4799 if (MemVTWidth == WidenWidth)
4800 return MemVT;
4801 RetVT = MemVT;
4802 break;
4803 }
4804 }
4805 }
4806
4807 // See if there is a larger vector type to load/store that has the same vector
4808 // element type and is evenly divisible with the WidenVT.
4809 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
4810 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
4811 EVT MemVT = (MVT::SimpleValueType) VT;
4812 // Skip vector MVTs which don't match the scalable property of WidenVT.
4813 if (Scalable != MemVT.isScalableVector())
4814 continue;
4815 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize();
4816 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4817 if ((Action == TargetLowering::TypeLegal ||
4818 Action == TargetLowering::TypePromoteInteger) &&
4819 WidenEltVT == MemVT.getVectorElementType() &&
4820 (WidenWidth % MemVTWidth) == 0 &&
4821 isPowerOf2_32(WidenWidth / MemVTWidth) &&
4822 (MemVTWidth <= Width ||
4823 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
4824 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
4825 return MemVT;
4826 }
4827 }
4828
4829 return RetVT;
4830 }
4831
4832 // Builds a vector type from scalar loads
4833 // VecTy: Resulting Vector type
4834 // LDOps: Load operators to build a vector type
4835 // [Start,End) the list of loads to use.
BuildVectorFromScalar(SelectionDAG & DAG,EVT VecTy,SmallVectorImpl<SDValue> & LdOps,unsigned Start,unsigned End)4836 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
4837 SmallVectorImpl<SDValue> &LdOps,
4838 unsigned Start, unsigned End) {
4839 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4840 SDLoc dl(LdOps[Start]);
4841 EVT LdTy = LdOps[Start].getValueType();
4842 unsigned Width = VecTy.getSizeInBits();
4843 unsigned NumElts = Width / LdTy.getSizeInBits();
4844 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
4845
4846 unsigned Idx = 1;
4847 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4848
4849 for (unsigned i = Start + 1; i != End; ++i) {
4850 EVT NewLdTy = LdOps[i].getValueType();
4851 if (NewLdTy != LdTy) {
4852 NumElts = Width / NewLdTy.getSizeInBits();
4853 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
4854 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
4855 // Readjust position and vector position based on new load type.
4856 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
4857 LdTy = NewLdTy;
4858 }
4859 VecOp = DAG.getNode(
4860 ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
4861 DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4862 }
4863 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
4864 }
4865
GenWidenVectorLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD)4866 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4867 LoadSDNode *LD) {
4868 // The strategy assumes that we can efficiently load power-of-two widths.
4869 // The routine chops the vector into the largest vector loads with the same
4870 // element type or scalar loads and then recombines it to the widen vector
4871 // type.
4872 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4873 unsigned WidenWidth = WidenVT.getSizeInBits();
4874 EVT LdVT = LD->getMemoryVT();
4875 SDLoc dl(LD);
4876 assert(LdVT.isVector() && WidenVT.isVector());
4877 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
4878
4879 // Load information
4880 SDValue Chain = LD->getChain();
4881 SDValue BasePtr = LD->getBasePtr();
4882 unsigned Align = LD->getAlignment();
4883 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4884 AAMDNodes AAInfo = LD->getAAInfo();
4885
4886 int LdWidth = LdVT.getSizeInBits();
4887 int WidthDiff = WidenWidth - LdWidth;
4888 unsigned LdAlign = (!LD->isSimple()) ? 0 : Align; // Allow wider loads.
4889
4890 // Find the vector type that can load from.
4891 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4892 int NewVTWidth = NewVT.getSizeInBits();
4893 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4894 Align, MMOFlags, AAInfo);
4895 LdChain.push_back(LdOp.getValue(1));
4896
4897 // Check if we can load the element with one instruction.
4898 if (LdWidth <= NewVTWidth) {
4899 if (!NewVT.isVector()) {
4900 unsigned NumElts = WidenWidth / NewVTWidth;
4901 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4902 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4903 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
4904 }
4905 if (NewVT == WidenVT)
4906 return LdOp;
4907
4908 assert(WidenWidth % NewVTWidth == 0);
4909 unsigned NumConcat = WidenWidth / NewVTWidth;
4910 SmallVector<SDValue, 16> ConcatOps(NumConcat);
4911 SDValue UndefVal = DAG.getUNDEF(NewVT);
4912 ConcatOps[0] = LdOp;
4913 for (unsigned i = 1; i != NumConcat; ++i)
4914 ConcatOps[i] = UndefVal;
4915 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
4916 }
4917
4918 // Load vector by using multiple loads from largest vector to scalar.
4919 SmallVector<SDValue, 16> LdOps;
4920 LdOps.push_back(LdOp);
4921
4922 LdWidth -= NewVTWidth;
4923 unsigned Offset = 0;
4924
4925 while (LdWidth > 0) {
4926 unsigned Increment = NewVTWidth / 8;
4927 Offset += Increment;
4928 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4929
4930 SDValue L;
4931 if (LdWidth < NewVTWidth) {
4932 // The current type we are using is too large. Find a better size.
4933 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4934 NewVTWidth = NewVT.getSizeInBits();
4935 L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4936 LD->getPointerInfo().getWithOffset(Offset),
4937 MinAlign(Align, Increment), MMOFlags, AAInfo);
4938 LdChain.push_back(L.getValue(1));
4939 if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
4940 // Later code assumes the vector loads produced will be mergeable, so we
4941 // must pad the final entry up to the previous width. Scalars are
4942 // combined separately.
4943 SmallVector<SDValue, 16> Loads;
4944 Loads.push_back(L);
4945 unsigned size = L->getValueSizeInBits(0);
4946 while (size < LdOp->getValueSizeInBits(0)) {
4947 Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
4948 size += L->getValueSizeInBits(0);
4949 }
4950 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
4951 }
4952 } else {
4953 L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4954 LD->getPointerInfo().getWithOffset(Offset),
4955 MinAlign(Align, Increment), MMOFlags, AAInfo);
4956 LdChain.push_back(L.getValue(1));
4957 }
4958
4959 LdOps.push_back(L);
4960 LdOp = L;
4961
4962 LdWidth -= NewVTWidth;
4963 }
4964
4965 // Build the vector from the load operations.
4966 unsigned End = LdOps.size();
4967 if (!LdOps[0].getValueType().isVector())
4968 // All the loads are scalar loads.
4969 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
4970
4971 // If the load contains vectors, build the vector using concat vector.
4972 // All of the vectors used to load are power-of-2, and the scalar loads can be
4973 // combined to make a power-of-2 vector.
4974 SmallVector<SDValue, 16> ConcatOps(End);
4975 int i = End - 1;
4976 int Idx = End;
4977 EVT LdTy = LdOps[i].getValueType();
4978 // First, combine the scalar loads to a vector.
4979 if (!LdTy.isVector()) {
4980 for (--i; i >= 0; --i) {
4981 LdTy = LdOps[i].getValueType();
4982 if (LdTy.isVector())
4983 break;
4984 }
4985 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
4986 }
4987 ConcatOps[--Idx] = LdOps[i];
4988 for (--i; i >= 0; --i) {
4989 EVT NewLdTy = LdOps[i].getValueType();
4990 if (NewLdTy != LdTy) {
4991 // Create a larger vector.
4992 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
4993 makeArrayRef(&ConcatOps[Idx], End - Idx));
4994 Idx = End - 1;
4995 LdTy = NewLdTy;
4996 }
4997 ConcatOps[--Idx] = LdOps[i];
4998 }
4999
5000 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
5001 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
5002 makeArrayRef(&ConcatOps[Idx], End - Idx));
5003
5004 // We need to fill the rest with undefs to build the vector.
5005 unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
5006 SmallVector<SDValue, 16> WidenOps(NumOps);
5007 SDValue UndefVal = DAG.getUNDEF(LdTy);
5008 {
5009 unsigned i = 0;
5010 for (; i != End-Idx; ++i)
5011 WidenOps[i] = ConcatOps[Idx+i];
5012 for (; i != NumOps; ++i)
5013 WidenOps[i] = UndefVal;
5014 }
5015 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
5016 }
5017
5018 SDValue
GenWidenVectorExtLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD,ISD::LoadExtType ExtType)5019 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
5020 LoadSDNode *LD,
5021 ISD::LoadExtType ExtType) {
5022 // For extension loads, it may not be more efficient to chop up the vector
5023 // and then extend it. Instead, we unroll the load and build a new vector.
5024 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
5025 EVT LdVT = LD->getMemoryVT();
5026 SDLoc dl(LD);
5027 assert(LdVT.isVector() && WidenVT.isVector());
5028
5029 // Load information
5030 SDValue Chain = LD->getChain();
5031 SDValue BasePtr = LD->getBasePtr();
5032 unsigned Align = LD->getAlignment();
5033 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
5034 AAMDNodes AAInfo = LD->getAAInfo();
5035
5036 EVT EltVT = WidenVT.getVectorElementType();
5037 EVT LdEltVT = LdVT.getVectorElementType();
5038 unsigned NumElts = LdVT.getVectorNumElements();
5039
5040 // Load each element and widen.
5041 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5042 SmallVector<SDValue, 16> Ops(WidenNumElts);
5043 unsigned Increment = LdEltVT.getSizeInBits() / 8;
5044 Ops[0] =
5045 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
5046 LdEltVT, Align, MMOFlags, AAInfo);
5047 LdChain.push_back(Ops[0].getValue(1));
5048 unsigned i = 0, Offset = Increment;
5049 for (i=1; i < NumElts; ++i, Offset += Increment) {
5050 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
5051 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
5052 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
5053 Align, MMOFlags, AAInfo);
5054 LdChain.push_back(Ops[i].getValue(1));
5055 }
5056
5057 // Fill the rest with undefs.
5058 SDValue UndefVal = DAG.getUNDEF(EltVT);
5059 for (; i != WidenNumElts; ++i)
5060 Ops[i] = UndefVal;
5061
5062 return DAG.getBuildVector(WidenVT, dl, Ops);
5063 }
5064
GenWidenVectorStores(SmallVectorImpl<SDValue> & StChain,StoreSDNode * ST)5065 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
5066 StoreSDNode *ST) {
5067 // The strategy assumes that we can efficiently store power-of-two widths.
5068 // The routine chops the vector into the largest vector stores with the same
5069 // element type or scalar stores.
5070 SDValue Chain = ST->getChain();
5071 SDValue BasePtr = ST->getBasePtr();
5072 unsigned Align = ST->getAlignment();
5073 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
5074 AAMDNodes AAInfo = ST->getAAInfo();
5075 SDValue ValOp = GetWidenedVector(ST->getValue());
5076 SDLoc dl(ST);
5077
5078 EVT StVT = ST->getMemoryVT();
5079 unsigned StWidth = StVT.getSizeInBits();
5080 EVT ValVT = ValOp.getValueType();
5081 unsigned ValWidth = ValVT.getSizeInBits();
5082 EVT ValEltVT = ValVT.getVectorElementType();
5083 unsigned ValEltWidth = ValEltVT.getSizeInBits();
5084 assert(StVT.getVectorElementType() == ValEltVT);
5085
5086 int Idx = 0; // current index to store
5087 unsigned Offset = 0; // offset from base to store
5088 while (StWidth != 0) {
5089 // Find the largest vector type we can store with.
5090 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
5091 unsigned NewVTWidth = NewVT.getSizeInBits();
5092 unsigned Increment = NewVTWidth / 8;
5093 if (NewVT.isVector()) {
5094 unsigned NumVTElts = NewVT.getVectorNumElements();
5095 do {
5096 SDValue EOp = DAG.getNode(
5097 ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
5098 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5099 StChain.push_back(DAG.getStore(
5100 Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
5101 MinAlign(Align, Offset), MMOFlags, AAInfo));
5102 StWidth -= NewVTWidth;
5103 Offset += Increment;
5104 Idx += NumVTElts;
5105
5106 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
5107 } while (StWidth != 0 && StWidth >= NewVTWidth);
5108 } else {
5109 // Cast the vector to the scalar type we can store.
5110 unsigned NumElts = ValWidth / NewVTWidth;
5111 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
5112 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
5113 // Readjust index position based on new vector type.
5114 Idx = Idx * ValEltWidth / NewVTWidth;
5115 do {
5116 SDValue EOp = DAG.getNode(
5117 ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
5118 DAG.getConstant(Idx++, dl,
5119 TLI.getVectorIdxTy(DAG.getDataLayout())));
5120 StChain.push_back(DAG.getStore(
5121 Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
5122 MinAlign(Align, Offset), MMOFlags, AAInfo));
5123 StWidth -= NewVTWidth;
5124 Offset += Increment;
5125 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
5126 } while (StWidth != 0 && StWidth >= NewVTWidth);
5127 // Restore index back to be relative to the original widen element type.
5128 Idx = Idx * NewVTWidth / ValEltWidth;
5129 }
5130 }
5131 }
5132
5133 void
GenWidenVectorTruncStores(SmallVectorImpl<SDValue> & StChain,StoreSDNode * ST)5134 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
5135 StoreSDNode *ST) {
5136 // For extension loads, it may not be more efficient to truncate the vector
5137 // and then store it. Instead, we extract each element and then store it.
5138 SDValue Chain = ST->getChain();
5139 SDValue BasePtr = ST->getBasePtr();
5140 unsigned Align = ST->getAlignment();
5141 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
5142 AAMDNodes AAInfo = ST->getAAInfo();
5143 SDValue ValOp = GetWidenedVector(ST->getValue());
5144 SDLoc dl(ST);
5145
5146 EVT StVT = ST->getMemoryVT();
5147 EVT ValVT = ValOp.getValueType();
5148
5149 // It must be true that the wide vector type is bigger than where we need to
5150 // store.
5151 assert(StVT.isVector() && ValOp.getValueType().isVector());
5152 assert(StVT.bitsLT(ValOp.getValueType()));
5153
5154 // For truncating stores, we can not play the tricks of chopping legal vector
5155 // types and bitcast it to the right type. Instead, we unroll the store.
5156 EVT StEltVT = StVT.getVectorElementType();
5157 EVT ValEltVT = ValVT.getVectorElementType();
5158 unsigned Increment = ValEltVT.getSizeInBits() / 8;
5159 unsigned NumElts = StVT.getVectorNumElements();
5160 SDValue EOp = DAG.getNode(
5161 ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
5162 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5163 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
5164 ST->getPointerInfo(), StEltVT, Align,
5165 MMOFlags, AAInfo));
5166 unsigned Offset = Increment;
5167 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
5168 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
5169 SDValue EOp = DAG.getNode(
5170 ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
5171 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5172 StChain.push_back(DAG.getTruncStore(
5173 Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
5174 StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
5175 }
5176 }
5177
5178 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
5179 /// input vector must have the same element type as NVT.
5180 /// FillWithZeroes specifies that the vector should be widened with zeroes.
ModifyToType(SDValue InOp,EVT NVT,bool FillWithZeroes)5181 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5182 bool FillWithZeroes) {
5183 // Note that InOp might have been widened so it might already have
5184 // the right width or it might need be narrowed.
5185 EVT InVT = InOp.getValueType();
5186 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
5187 "input and widen element type must match");
5188 SDLoc dl(InOp);
5189
5190 // Check if InOp already has the right width.
5191 if (InVT == NVT)
5192 return InOp;
5193
5194 unsigned InNumElts = InVT.getVectorNumElements();
5195 unsigned WidenNumElts = NVT.getVectorNumElements();
5196 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
5197 unsigned NumConcat = WidenNumElts / InNumElts;
5198 SmallVector<SDValue, 16> Ops(NumConcat);
5199 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
5200 DAG.getUNDEF(InVT);
5201 Ops[0] = InOp;
5202 for (unsigned i = 1; i != NumConcat; ++i)
5203 Ops[i] = FillVal;
5204
5205 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
5206 }
5207
5208 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
5209 return DAG.getNode(
5210 ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
5211 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5212
5213 // Fall back to extract and build.
5214 SmallVector<SDValue, 16> Ops(WidenNumElts);
5215 EVT EltVT = NVT.getVectorElementType();
5216 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
5217 unsigned Idx;
5218 for (Idx = 0; Idx < MinNumElts; ++Idx)
5219 Ops[Idx] = DAG.getNode(
5220 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5221 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5222
5223 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
5224 DAG.getUNDEF(EltVT);
5225 for ( ; Idx < WidenNumElts; ++Idx)
5226 Ops[Idx] = FillVal;
5227 return DAG.getBuildVector(NVT, dl, Ops);
5228 }
5229