1 //===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::LegalizeVectors method.
10 //
11 // The vector legalizer looks for vector operations which might need to be
12 // scalarized and legalizes them. This is a separate step from Legalize because
13 // scalarizing can introduce illegal types. For example, suppose we have an
14 // ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15 // on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16 // operation, which introduces nodes with the illegal type i64 which must be
17 // expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18 // the operation must be unrolled, which introduces nodes with the illegal
19 // type i8 which must be promoted.
20 //
21 // This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22 // or operations that happen to take a vector which are custom-lowered;
23 // the legalization for such operations never produces nodes
24 // with illegal types, so it's okay to put off legalizing them until
25 // SelectionDAG::Legalize runs.
26 //
27 //===----------------------------------------------------------------------===//
28
29 #include "llvm/ADT/APInt.h"
30 #include "llvm/ADT/DenseMap.h"
31 #include "llvm/ADT/SmallVector.h"
32 #include "llvm/CodeGen/ISDOpcodes.h"
33 #include "llvm/CodeGen/MachineMemOperand.h"
34 #include "llvm/CodeGen/SelectionDAG.h"
35 #include "llvm/CodeGen/SelectionDAGNodes.h"
36 #include "llvm/CodeGen/TargetLowering.h"
37 #include "llvm/CodeGen/ValueTypes.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/Compiler.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/MachineValueType.h"
44 #include "llvm/Support/MathExtras.h"
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <utility>
49
50 using namespace llvm;
51
52 #define DEBUG_TYPE "legalizevectorops"
53
54 namespace {
55
56 class VectorLegalizer {
57 SelectionDAG& DAG;
58 const TargetLowering &TLI;
59 bool Changed = false; // Keep track of whether anything changed
60
61 /// For nodes that are of legal width, and that have more than one use, this
62 /// map indicates what regularized operand to use. This allows us to avoid
63 /// legalizing the same thing more than once.
64 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
65
66 /// Adds a node to the translation cache.
AddLegalizedOperand(SDValue From,SDValue To)67 void AddLegalizedOperand(SDValue From, SDValue To) {
68 LegalizedNodes.insert(std::make_pair(From, To));
69 // If someone requests legalization of the new node, return itself.
70 if (From != To)
71 LegalizedNodes.insert(std::make_pair(To, To));
72 }
73
74 /// Legalizes the given node.
75 SDValue LegalizeOp(SDValue Op);
76
77 /// Assuming the node is legal, "legalize" the results.
78 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
79
80 /// Make sure Results are legal and update the translation cache.
81 SDValue RecursivelyLegalizeResults(SDValue Op,
82 MutableArrayRef<SDValue> Results);
83
84 /// Wrapper to interface LowerOperation with a vector of Results.
85 /// Returns false if the target wants to use default expansion. Otherwise
86 /// returns true. If return is true and the Results are empty, then the
87 /// target wants to keep the input node as is.
88 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
89
90 /// Implements unrolling a VSETCC.
91 SDValue UnrollVSETCC(SDNode *Node);
92
93 /// Implement expand-based legalization of vector operations.
94 ///
95 /// This is just a high-level routine to dispatch to specific code paths for
96 /// operations to legalize them.
97 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
98
99 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
100 /// FP_TO_SINT isn't legal.
101 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
102
103 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
104 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
105 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
106
107 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
108 SDValue ExpandSEXTINREG(SDNode *Node);
109
110 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
111 ///
112 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
113 /// type. The contents of the bits in the extended part of each element are
114 /// undef.
115 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
116
117 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
118 ///
119 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
120 /// type, then shifts left and arithmetic shifts right to introduce a sign
121 /// extension.
122 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
123
124 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
125 ///
126 /// Shuffles the low lanes of the operand into place and blends zeros into
127 /// the remaining lanes, finally bitcasting to the proper type.
128 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
129
130 /// Expand bswap of vectors into a shuffle if legal.
131 SDValue ExpandBSWAP(SDNode *Node);
132
133 /// Implement vselect in terms of XOR, AND, OR when blend is not
134 /// supported by the target.
135 SDValue ExpandVSELECT(SDNode *Node);
136 SDValue ExpandSELECT(SDNode *Node);
137 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
138 SDValue ExpandStore(SDNode *N);
139 SDValue ExpandFNEG(SDNode *Node);
140 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
141 void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145 SDValue ExpandFixedPointDiv(SDNode *Node);
146 SDValue ExpandStrictFPOp(SDNode *Node);
147 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148
149 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150
151 /// Implements vector promotion.
152 ///
153 /// This is essentially just bitcasting the operands to a different type and
154 /// bitcasting the result back to the original type.
155 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156
157 /// Implements [SU]INT_TO_FP vector promotion.
158 ///
159 /// This is a [zs]ext of the input operand to a larger integer type.
160 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161
162 /// Implements FP_TO_[SU]INT vector promotion of the result type.
163 ///
164 /// It is promoted to a larger integer type. The result is then
165 /// truncated back to the original type.
166 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
167
168 public:
VectorLegalizer(SelectionDAG & dag)169 VectorLegalizer(SelectionDAG& dag) :
170 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
171
172 /// Begin legalizer the vector operations in the DAG.
173 bool Run();
174 };
175
176 } // end anonymous namespace
177
Run()178 bool VectorLegalizer::Run() {
179 // Before we start legalizing vector nodes, check if there are any vectors.
180 bool HasVectors = false;
181 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
182 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
183 // Check if the values of the nodes contain vectors. We don't need to check
184 // the operands because we are going to check their values at some point.
185 for (SDNode::value_iterator J = I->value_begin(), E = I->value_end();
186 J != E; ++J)
187 HasVectors |= J->isVector();
188
189 // If we found a vector node we can start the legalization.
190 if (HasVectors)
191 break;
192 }
193
194 // If this basic block has no vectors then no need to legalize vectors.
195 if (!HasVectors)
196 return false;
197
198 // The legalize process is inherently a bottom-up recursive process (users
199 // legalize their uses before themselves). Given infinite stack space, we
200 // could just start legalizing on the root and traverse the whole graph. In
201 // practice however, this causes us to run out of stack space on large basic
202 // blocks. To avoid this problem, compute an ordering of the nodes where each
203 // node is only legalized after all of its operands are legalized.
204 DAG.AssignTopologicalOrder();
205 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
206 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
207 LegalizeOp(SDValue(&*I, 0));
208
209 // Finally, it's possible the root changed. Get the new root.
210 SDValue OldRoot = DAG.getRoot();
211 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
212 DAG.setRoot(LegalizedNodes[OldRoot]);
213
214 LegalizedNodes.clear();
215
216 // Remove dead nodes now.
217 DAG.RemoveDeadNodes();
218
219 return Changed;
220 }
221
TranslateLegalizeResults(SDValue Op,SDNode * Result)222 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
223 assert(Op->getNumValues() == Result->getNumValues() &&
224 "Unexpected number of results");
225 // Generic legalization: just pass the operand through.
226 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
227 AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
228 return SDValue(Result, Op.getResNo());
229 }
230
231 SDValue
RecursivelyLegalizeResults(SDValue Op,MutableArrayRef<SDValue> Results)232 VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
233 MutableArrayRef<SDValue> Results) {
234 assert(Results.size() == Op->getNumValues() &&
235 "Unexpected number of results");
236 // Make sure that the generated code is itself legal.
237 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
238 Results[i] = LegalizeOp(Results[i]);
239 AddLegalizedOperand(Op.getValue(i), Results[i]);
240 }
241
242 return Results[Op.getResNo()];
243 }
244
LegalizeOp(SDValue Op)245 SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
246 // Note that LegalizeOp may be reentered even from single-use nodes, which
247 // means that we always must cache transformed nodes.
248 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
249 if (I != LegalizedNodes.end()) return I->second;
250
251 // Legalize the operands
252 SmallVector<SDValue, 8> Ops;
253 for (const SDValue &Oper : Op->op_values())
254 Ops.push_back(LegalizeOp(Oper));
255
256 SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
257
258 if (Op.getOpcode() == ISD::LOAD) {
259 LoadSDNode *LD = cast<LoadSDNode>(Node);
260 ISD::LoadExtType ExtType = LD->getExtensionType();
261 if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) {
262 LLVM_DEBUG(dbgs() << "\nLegalizing extending vector load: ";
263 Node->dump(&DAG));
264 switch (TLI.getLoadExtAction(LD->getExtensionType(), LD->getValueType(0),
265 LD->getMemoryVT())) {
266 default: llvm_unreachable("This action is not supported yet!");
267 case TargetLowering::Legal:
268 return TranslateLegalizeResults(Op, Node);
269 case TargetLowering::Custom: {
270 SmallVector<SDValue, 2> ResultVals;
271 if (LowerOperationWrapper(Node, ResultVals)) {
272 if (ResultVals.empty())
273 return TranslateLegalizeResults(Op, Node);
274
275 Changed = true;
276 return RecursivelyLegalizeResults(Op, ResultVals);
277 }
278 LLVM_FALLTHROUGH;
279 }
280 case TargetLowering::Expand: {
281 Changed = true;
282 std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
283 AddLegalizedOperand(Op.getValue(0), Tmp.first);
284 AddLegalizedOperand(Op.getValue(1), Tmp.second);
285 return Op.getResNo() ? Tmp.first : Tmp.second;
286 }
287 }
288 }
289 } else if (Op.getOpcode() == ISD::STORE) {
290 StoreSDNode *ST = cast<StoreSDNode>(Node);
291 EVT StVT = ST->getMemoryVT();
292 MVT ValVT = ST->getValue().getSimpleValueType();
293 if (StVT.isVector() && ST->isTruncatingStore()) {
294 LLVM_DEBUG(dbgs() << "\nLegalizing truncating vector store: ";
295 Node->dump(&DAG));
296 switch (TLI.getTruncStoreAction(ValVT, StVT)) {
297 default: llvm_unreachable("This action is not supported yet!");
298 case TargetLowering::Legal:
299 return TranslateLegalizeResults(Op, Node);
300 case TargetLowering::Custom: {
301 SmallVector<SDValue, 1> ResultVals;
302 if (LowerOperationWrapper(Node, ResultVals)) {
303 if (ResultVals.empty())
304 return TranslateLegalizeResults(Op, Node);
305
306 Changed = true;
307 return RecursivelyLegalizeResults(Op, ResultVals);
308 }
309 LLVM_FALLTHROUGH;
310 }
311 case TargetLowering::Expand: {
312 Changed = true;
313 SDValue Chain = ExpandStore(Node);
314 AddLegalizedOperand(Op, Chain);
315 return Chain;
316 }
317 }
318 }
319 }
320
321 bool HasVectorValueOrOp = false;
322 for (auto J = Node->value_begin(), E = Node->value_end(); J != E; ++J)
323 HasVectorValueOrOp |= J->isVector();
324 for (const SDValue &Oper : Node->op_values())
325 HasVectorValueOrOp |= Oper.getValueType().isVector();
326
327 if (!HasVectorValueOrOp)
328 return TranslateLegalizeResults(Op, Node);
329
330 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
331 EVT ValVT;
332 switch (Op.getOpcode()) {
333 default:
334 return TranslateLegalizeResults(Op, Node);
335 case ISD::MERGE_VALUES:
336 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
337 // This operation lies about being legal: when it claims to be legal,
338 // it should actually be expanded.
339 if (Action == TargetLowering::Legal)
340 Action = TargetLowering::Expand;
341 break;
342 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
343 case ISD::STRICT_##DAGN:
344 #include "llvm/IR/ConstrainedOps.def"
345 ValVT = Node->getValueType(0);
346 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
347 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
348 ValVT = Node->getOperand(1).getValueType();
349 Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
350 // If we're asked to expand a strict vector floating-point operation,
351 // by default we're going to simply unroll it. That is usually the
352 // best approach, except in the case where the resulting strict (scalar)
353 // operations would themselves use the fallback mutation to non-strict.
354 // In that specific case, just do the fallback on the vector op.
355 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
356 TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
357 TargetLowering::Legal) {
358 EVT EltVT = ValVT.getVectorElementType();
359 if (TLI.getOperationAction(Node->getOpcode(), EltVT)
360 == TargetLowering::Expand &&
361 TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
362 == TargetLowering::Legal)
363 Action = TargetLowering::Legal;
364 }
365 break;
366 case ISD::ADD:
367 case ISD::SUB:
368 case ISD::MUL:
369 case ISD::MULHS:
370 case ISD::MULHU:
371 case ISD::SDIV:
372 case ISD::UDIV:
373 case ISD::SREM:
374 case ISD::UREM:
375 case ISD::SDIVREM:
376 case ISD::UDIVREM:
377 case ISD::FADD:
378 case ISD::FSUB:
379 case ISD::FMUL:
380 case ISD::FDIV:
381 case ISD::FREM:
382 case ISD::AND:
383 case ISD::OR:
384 case ISD::XOR:
385 case ISD::SHL:
386 case ISD::SRA:
387 case ISD::SRL:
388 case ISD::FSHL:
389 case ISD::FSHR:
390 case ISD::ROTL:
391 case ISD::ROTR:
392 case ISD::ABS:
393 case ISD::BSWAP:
394 case ISD::BITREVERSE:
395 case ISD::CTLZ:
396 case ISD::CTTZ:
397 case ISD::CTLZ_ZERO_UNDEF:
398 case ISD::CTTZ_ZERO_UNDEF:
399 case ISD::CTPOP:
400 case ISD::SELECT:
401 case ISD::VSELECT:
402 case ISD::SELECT_CC:
403 case ISD::SETCC:
404 case ISD::ZERO_EXTEND:
405 case ISD::ANY_EXTEND:
406 case ISD::TRUNCATE:
407 case ISD::SIGN_EXTEND:
408 case ISD::FP_TO_SINT:
409 case ISD::FP_TO_UINT:
410 case ISD::FNEG:
411 case ISD::FABS:
412 case ISD::FMINNUM:
413 case ISD::FMAXNUM:
414 case ISD::FMINNUM_IEEE:
415 case ISD::FMAXNUM_IEEE:
416 case ISD::FMINIMUM:
417 case ISD::FMAXIMUM:
418 case ISD::FCOPYSIGN:
419 case ISD::FSQRT:
420 case ISD::FSIN:
421 case ISD::FCOS:
422 case ISD::FPOWI:
423 case ISD::FPOW:
424 case ISD::FLOG:
425 case ISD::FLOG2:
426 case ISD::FLOG10:
427 case ISD::FEXP:
428 case ISD::FEXP2:
429 case ISD::FCEIL:
430 case ISD::FTRUNC:
431 case ISD::FRINT:
432 case ISD::FNEARBYINT:
433 case ISD::FROUND:
434 case ISD::FFLOOR:
435 case ISD::FP_ROUND:
436 case ISD::FP_EXTEND:
437 case ISD::FMA:
438 case ISD::SIGN_EXTEND_INREG:
439 case ISD::ANY_EXTEND_VECTOR_INREG:
440 case ISD::SIGN_EXTEND_VECTOR_INREG:
441 case ISD::ZERO_EXTEND_VECTOR_INREG:
442 case ISD::SMIN:
443 case ISD::SMAX:
444 case ISD::UMIN:
445 case ISD::UMAX:
446 case ISD::SMUL_LOHI:
447 case ISD::UMUL_LOHI:
448 case ISD::SADDO:
449 case ISD::UADDO:
450 case ISD::SSUBO:
451 case ISD::USUBO:
452 case ISD::SMULO:
453 case ISD::UMULO:
454 case ISD::FCANONICALIZE:
455 case ISD::SADDSAT:
456 case ISD::UADDSAT:
457 case ISD::SSUBSAT:
458 case ISD::USUBSAT:
459 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
460 break;
461 case ISD::SMULFIX:
462 case ISD::SMULFIXSAT:
463 case ISD::UMULFIX:
464 case ISD::UMULFIXSAT:
465 case ISD::SDIVFIX:
466 case ISD::UDIVFIX: {
467 unsigned Scale = Node->getConstantOperandVal(2);
468 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
469 Node->getValueType(0), Scale);
470 break;
471 }
472 case ISD::SINT_TO_FP:
473 case ISD::UINT_TO_FP:
474 case ISD::VECREDUCE_ADD:
475 case ISD::VECREDUCE_MUL:
476 case ISD::VECREDUCE_AND:
477 case ISD::VECREDUCE_OR:
478 case ISD::VECREDUCE_XOR:
479 case ISD::VECREDUCE_SMAX:
480 case ISD::VECREDUCE_SMIN:
481 case ISD::VECREDUCE_UMAX:
482 case ISD::VECREDUCE_UMIN:
483 case ISD::VECREDUCE_FADD:
484 case ISD::VECREDUCE_FMUL:
485 case ISD::VECREDUCE_FMAX:
486 case ISD::VECREDUCE_FMIN:
487 Action = TLI.getOperationAction(Node->getOpcode(),
488 Node->getOperand(0).getValueType());
489 break;
490 }
491
492 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
493
494 SmallVector<SDValue, 8> ResultVals;
495 switch (Action) {
496 default: llvm_unreachable("This action is not supported yet!");
497 case TargetLowering::Promote:
498 LLVM_DEBUG(dbgs() << "Promoting\n");
499 Promote(Node, ResultVals);
500 assert(!ResultVals.empty() && "No results for promotion?");
501 break;
502 case TargetLowering::Legal:
503 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
504 break;
505 case TargetLowering::Custom:
506 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
507 if (LowerOperationWrapper(Node, ResultVals))
508 break;
509 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
510 LLVM_FALLTHROUGH;
511 case TargetLowering::Expand:
512 LLVM_DEBUG(dbgs() << "Expanding\n");
513 Expand(Node, ResultVals);
514 break;
515 }
516
517 if (ResultVals.empty())
518 return TranslateLegalizeResults(Op, Node);
519
520 Changed = true;
521 return RecursivelyLegalizeResults(Op, ResultVals);
522 }
523
524 // FIME: This is very similar to the X86 override of
525 // TargetLowering::LowerOperationWrapper. Can we merge them somehow?
LowerOperationWrapper(SDNode * Node,SmallVectorImpl<SDValue> & Results)526 bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
527 SmallVectorImpl<SDValue> &Results) {
528 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
529
530 if (!Res.getNode())
531 return false;
532
533 if (Res == SDValue(Node, 0))
534 return true;
535
536 // If the original node has one result, take the return value from
537 // LowerOperation as is. It might not be result number 0.
538 if (Node->getNumValues() == 1) {
539 Results.push_back(Res);
540 return true;
541 }
542
543 // If the original node has multiple results, then the return node should
544 // have the same number of results.
545 assert((Node->getNumValues() == Res->getNumValues()) &&
546 "Lowering returned the wrong number of results!");
547
548 // Places new result values base on N result number.
549 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
550 Results.push_back(Res.getValue(I));
551
552 return true;
553 }
554
Promote(SDNode * Node,SmallVectorImpl<SDValue> & Results)555 void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
556 // For a few operations there is a specific concept for promotion based on
557 // the operand's type.
558 switch (Node->getOpcode()) {
559 case ISD::SINT_TO_FP:
560 case ISD::UINT_TO_FP:
561 case ISD::STRICT_SINT_TO_FP:
562 case ISD::STRICT_UINT_TO_FP:
563 // "Promote" the operation by extending the operand.
564 PromoteINT_TO_FP(Node, Results);
565 return;
566 case ISD::FP_TO_UINT:
567 case ISD::FP_TO_SINT:
568 case ISD::STRICT_FP_TO_UINT:
569 case ISD::STRICT_FP_TO_SINT:
570 // Promote the operation by extending the operand.
571 PromoteFP_TO_INT(Node, Results);
572 return;
573 case ISD::FP_ROUND:
574 case ISD::FP_EXTEND:
575 // These operations are used to do promotion so they can't be promoted
576 // themselves.
577 llvm_unreachable("Don't know how to promote this operation!");
578 }
579
580 // There are currently two cases of vector promotion:
581 // 1) Bitcasting a vector of integers to a different type to a vector of the
582 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
583 // 2) Extending a vector of floats to a vector of the same number of larger
584 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
585 assert(Node->getNumValues() == 1 &&
586 "Can't promote a vector with multiple results!");
587 MVT VT = Node->getSimpleValueType(0);
588 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
589 SDLoc dl(Node);
590 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
591
592 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
593 if (Node->getOperand(j).getValueType().isVector())
594 if (Node->getOperand(j)
595 .getValueType()
596 .getVectorElementType()
597 .isFloatingPoint() &&
598 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
599 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
600 else
601 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
602 else
603 Operands[j] = Node->getOperand(j);
604 }
605
606 SDValue Res =
607 DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
608
609 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
610 (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
611 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
612 Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res, DAG.getIntPtrConstant(0, dl));
613 else
614 Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
615
616 Results.push_back(Res);
617 }
618
PromoteINT_TO_FP(SDNode * Node,SmallVectorImpl<SDValue> & Results)619 void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
620 SmallVectorImpl<SDValue> &Results) {
621 // INT_TO_FP operations may require the input operand be promoted even
622 // when the type is otherwise legal.
623 bool IsStrict = Node->isStrictFPOpcode();
624 MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
625 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
626 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
627 "Vectors have different number of elements!");
628
629 SDLoc dl(Node);
630 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
631
632 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
633 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
634 ? ISD::ZERO_EXTEND
635 : ISD::SIGN_EXTEND;
636 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
637 if (Node->getOperand(j).getValueType().isVector())
638 Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
639 else
640 Operands[j] = Node->getOperand(j);
641 }
642
643 if (IsStrict) {
644 SDValue Res = DAG.getNode(Node->getOpcode(), dl,
645 {Node->getValueType(0), MVT::Other}, Operands);
646 Results.push_back(Res);
647 Results.push_back(Res.getValue(1));
648 return;
649 }
650
651 SDValue Res =
652 DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
653 Results.push_back(Res);
654 }
655
656 // For FP_TO_INT we promote the result type to a vector type with wider
657 // elements and then truncate the result. This is different from the default
658 // PromoteVector which uses bitcast to promote thus assumning that the
659 // promoted vector type has the same overall size.
PromoteFP_TO_INT(SDNode * Node,SmallVectorImpl<SDValue> & Results)660 void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
661 SmallVectorImpl<SDValue> &Results) {
662 MVT VT = Node->getSimpleValueType(0);
663 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
664 bool IsStrict = Node->isStrictFPOpcode();
665 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
666 "Vectors have different number of elements!");
667
668 unsigned NewOpc = Node->getOpcode();
669 // Change FP_TO_UINT to FP_TO_SINT if possible.
670 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
671 if (NewOpc == ISD::FP_TO_UINT &&
672 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
673 NewOpc = ISD::FP_TO_SINT;
674
675 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
676 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
677 NewOpc = ISD::STRICT_FP_TO_SINT;
678
679 SDLoc dl(Node);
680 SDValue Promoted, Chain;
681 if (IsStrict) {
682 Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
683 {Node->getOperand(0), Node->getOperand(1)});
684 Chain = Promoted.getValue(1);
685 } else
686 Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
687
688 // Assert that the converted value fits in the original type. If it doesn't
689 // (eg: because the value being converted is too big), then the result of the
690 // original operation was undefined anyway, so the assert is still correct.
691 if (Node->getOpcode() == ISD::FP_TO_UINT ||
692 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
693 NewOpc = ISD::AssertZext;
694 else
695 NewOpc = ISD::AssertSext;
696
697 Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
698 DAG.getValueType(VT.getScalarType()));
699 Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
700 Results.push_back(Promoted);
701 if (IsStrict)
702 Results.push_back(Chain);
703 }
704
ExpandLoad(SDNode * N)705 std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
706 LoadSDNode *LD = cast<LoadSDNode>(N);
707
708 EVT SrcVT = LD->getMemoryVT();
709 EVT SrcEltVT = SrcVT.getScalarType();
710 unsigned NumElem = SrcVT.getVectorNumElements();
711
712 SDValue NewChain;
713 SDValue Value;
714 if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.isByteSized()) {
715 SDLoc dl(N);
716
717 SmallVector<SDValue, 8> Vals;
718 SmallVector<SDValue, 8> LoadChains;
719
720 EVT DstEltVT = LD->getValueType(0).getScalarType();
721 SDValue Chain = LD->getChain();
722 SDValue BasePTR = LD->getBasePtr();
723 ISD::LoadExtType ExtType = LD->getExtensionType();
724
725 // When elements in a vector is not byte-addressable, we cannot directly
726 // load each element by advancing pointer, which could only address bytes.
727 // Instead, we load all significant words, mask bits off, and concatenate
728 // them to form each element. Finally, they are extended to destination
729 // scalar type to build the destination vector.
730 EVT WideVT = TLI.getPointerTy(DAG.getDataLayout());
731
732 assert(WideVT.isRound() &&
733 "Could not handle the sophisticated case when the widest integer is"
734 " not power of 2.");
735 assert(WideVT.bitsGE(SrcEltVT) &&
736 "Type is not legalized?");
737
738 unsigned WideBytes = WideVT.getStoreSize();
739 unsigned Offset = 0;
740 unsigned RemainingBytes = SrcVT.getStoreSize();
741 SmallVector<SDValue, 8> LoadVals;
742 while (RemainingBytes > 0) {
743 SDValue ScalarLoad;
744 unsigned LoadBytes = WideBytes;
745
746 if (RemainingBytes >= LoadBytes) {
747 ScalarLoad =
748 DAG.getLoad(WideVT, dl, Chain, BasePTR,
749 LD->getPointerInfo().getWithOffset(Offset),
750 MinAlign(LD->getAlignment(), Offset),
751 LD->getMemOperand()->getFlags(), LD->getAAInfo());
752 } else {
753 EVT LoadVT = WideVT;
754 while (RemainingBytes < LoadBytes) {
755 LoadBytes >>= 1; // Reduce the load size by half.
756 LoadVT = EVT::getIntegerVT(*DAG.getContext(), LoadBytes << 3);
757 }
758 ScalarLoad =
759 DAG.getExtLoad(ISD::EXTLOAD, dl, WideVT, Chain, BasePTR,
760 LD->getPointerInfo().getWithOffset(Offset), LoadVT,
761 MinAlign(LD->getAlignment(), Offset),
762 LD->getMemOperand()->getFlags(), LD->getAAInfo());
763 }
764
765 RemainingBytes -= LoadBytes;
766 Offset += LoadBytes;
767
768 BasePTR = DAG.getObjectPtrOffset(dl, BasePTR, LoadBytes);
769
770 LoadVals.push_back(ScalarLoad.getValue(0));
771 LoadChains.push_back(ScalarLoad.getValue(1));
772 }
773
774 unsigned BitOffset = 0;
775 unsigned WideIdx = 0;
776 unsigned WideBits = WideVT.getSizeInBits();
777
778 // Extract bits, pack and extend/trunc them into destination type.
779 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
780 SDValue SrcEltBitMask = DAG.getConstant(
781 APInt::getLowBitsSet(WideBits, SrcEltBits), dl, WideVT);
782
783 for (unsigned Idx = 0; Idx != NumElem; ++Idx) {
784 assert(BitOffset < WideBits && "Unexpected offset!");
785
786 SDValue ShAmt = DAG.getConstant(
787 BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
788 SDValue Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt);
789
790 BitOffset += SrcEltBits;
791 if (BitOffset >= WideBits) {
792 WideIdx++;
793 BitOffset -= WideBits;
794 if (BitOffset > 0) {
795 ShAmt = DAG.getConstant(
796 SrcEltBits - BitOffset, dl,
797 TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
798 SDValue Hi =
799 DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt);
800 Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi);
801 }
802 }
803
804 Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask);
805
806 switch (ExtType) {
807 default: llvm_unreachable("Unknown extended-load op!");
808 case ISD::EXTLOAD:
809 Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT);
810 break;
811 case ISD::ZEXTLOAD:
812 Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT);
813 break;
814 case ISD::SEXTLOAD:
815 ShAmt =
816 DAG.getConstant(WideBits - SrcEltBits, dl,
817 TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
818 Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt);
819 Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt);
820 Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT);
821 break;
822 }
823 Vals.push_back(Lo);
824 }
825
826 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
827 Value = DAG.getBuildVector(N->getValueType(0), dl, Vals);
828 } else {
829 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
830 }
831
832 return std::make_pair(Value, NewChain);
833 }
834
ExpandStore(SDNode * N)835 SDValue VectorLegalizer::ExpandStore(SDNode *N) {
836 StoreSDNode *ST = cast<StoreSDNode>(N);
837 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
838 return TF;
839 }
840
Expand(SDNode * Node,SmallVectorImpl<SDValue> & Results)841 void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
842 SDValue Tmp;
843 switch (Node->getOpcode()) {
844 case ISD::MERGE_VALUES:
845 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
846 Results.push_back(Node->getOperand(i));
847 return;
848 case ISD::SIGN_EXTEND_INREG:
849 Results.push_back(ExpandSEXTINREG(Node));
850 return;
851 case ISD::ANY_EXTEND_VECTOR_INREG:
852 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
853 return;
854 case ISD::SIGN_EXTEND_VECTOR_INREG:
855 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
856 return;
857 case ISD::ZERO_EXTEND_VECTOR_INREG:
858 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
859 return;
860 case ISD::BSWAP:
861 Results.push_back(ExpandBSWAP(Node));
862 return;
863 case ISD::VSELECT:
864 Results.push_back(ExpandVSELECT(Node));
865 return;
866 case ISD::SELECT:
867 Results.push_back(ExpandSELECT(Node));
868 return;
869 case ISD::FP_TO_UINT:
870 ExpandFP_TO_UINT(Node, Results);
871 return;
872 case ISD::UINT_TO_FP:
873 ExpandUINT_TO_FLOAT(Node, Results);
874 return;
875 case ISD::FNEG:
876 Results.push_back(ExpandFNEG(Node));
877 return;
878 case ISD::FSUB:
879 ExpandFSUB(Node, Results);
880 return;
881 case ISD::SETCC:
882 Results.push_back(UnrollVSETCC(Node));
883 return;
884 case ISD::ABS:
885 if (TLI.expandABS(Node, Tmp, DAG)) {
886 Results.push_back(Tmp);
887 return;
888 }
889 break;
890 case ISD::BITREVERSE:
891 ExpandBITREVERSE(Node, Results);
892 return;
893 case ISD::CTPOP:
894 if (TLI.expandCTPOP(Node, Tmp, DAG)) {
895 Results.push_back(Tmp);
896 return;
897 }
898 break;
899 case ISD::CTLZ:
900 case ISD::CTLZ_ZERO_UNDEF:
901 if (TLI.expandCTLZ(Node, Tmp, DAG)) {
902 Results.push_back(Tmp);
903 return;
904 }
905 break;
906 case ISD::CTTZ:
907 case ISD::CTTZ_ZERO_UNDEF:
908 if (TLI.expandCTTZ(Node, Tmp, DAG)) {
909 Results.push_back(Tmp);
910 return;
911 }
912 break;
913 case ISD::FSHL:
914 case ISD::FSHR:
915 if (TLI.expandFunnelShift(Node, Tmp, DAG)) {
916 Results.push_back(Tmp);
917 return;
918 }
919 break;
920 case ISD::ROTL:
921 case ISD::ROTR:
922 if (TLI.expandROT(Node, Tmp, DAG)) {
923 Results.push_back(Tmp);
924 return;
925 }
926 break;
927 case ISD::FMINNUM:
928 case ISD::FMAXNUM:
929 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
930 Results.push_back(Expanded);
931 return;
932 }
933 break;
934 case ISD::UADDO:
935 case ISD::USUBO:
936 ExpandUADDSUBO(Node, Results);
937 return;
938 case ISD::SADDO:
939 case ISD::SSUBO:
940 ExpandSADDSUBO(Node, Results);
941 return;
942 case ISD::UMULO:
943 case ISD::SMULO:
944 ExpandMULO(Node, Results);
945 return;
946 case ISD::USUBSAT:
947 case ISD::SSUBSAT:
948 case ISD::UADDSAT:
949 case ISD::SADDSAT:
950 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
951 Results.push_back(Expanded);
952 return;
953 }
954 break;
955 case ISD::SMULFIX:
956 case ISD::UMULFIX:
957 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
958 Results.push_back(Expanded);
959 return;
960 }
961 break;
962 case ISD::SMULFIXSAT:
963 case ISD::UMULFIXSAT:
964 // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
965 // why. Maybe it results in worse codegen compared to the unroll for some
966 // targets? This should probably be investigated. And if we still prefer to
967 // unroll an explanation could be helpful.
968 break;
969 case ISD::SDIVFIX:
970 case ISD::UDIVFIX:
971 Results.push_back(ExpandFixedPointDiv(Node));
972 return;
973 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
974 case ISD::STRICT_##DAGN:
975 #include "llvm/IR/ConstrainedOps.def"
976 ExpandStrictFPOp(Node, Results);
977 return;
978 case ISD::VECREDUCE_ADD:
979 case ISD::VECREDUCE_MUL:
980 case ISD::VECREDUCE_AND:
981 case ISD::VECREDUCE_OR:
982 case ISD::VECREDUCE_XOR:
983 case ISD::VECREDUCE_SMAX:
984 case ISD::VECREDUCE_SMIN:
985 case ISD::VECREDUCE_UMAX:
986 case ISD::VECREDUCE_UMIN:
987 case ISD::VECREDUCE_FADD:
988 case ISD::VECREDUCE_FMUL:
989 case ISD::VECREDUCE_FMAX:
990 case ISD::VECREDUCE_FMIN:
991 Results.push_back(TLI.expandVecReduce(Node, DAG));
992 return;
993 }
994
995 Results.push_back(DAG.UnrollVectorOp(Node));
996 }
997
ExpandSELECT(SDNode * Node)998 SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
999 // Lower a select instruction where the condition is a scalar and the
1000 // operands are vectors. Lower this select to VSELECT and implement it
1001 // using XOR AND OR. The selector bit is broadcasted.
1002 EVT VT = Node->getValueType(0);
1003 SDLoc DL(Node);
1004
1005 SDValue Mask = Node->getOperand(0);
1006 SDValue Op1 = Node->getOperand(1);
1007 SDValue Op2 = Node->getOperand(2);
1008
1009 assert(VT.isVector() && !Mask.getValueType().isVector()
1010 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
1011
1012 // If we can't even use the basic vector operations of
1013 // AND,OR,XOR, we will have to scalarize the op.
1014 // Notice that the operation may be 'promoted' which means that it is
1015 // 'bitcasted' to another type which is handled.
1016 // Also, we need to be able to construct a splat vector using BUILD_VECTOR.
1017 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1018 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1019 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
1020 TLI.getOperationAction(ISD::BUILD_VECTOR, VT) == TargetLowering::Expand)
1021 return DAG.UnrollVectorOp(Node);
1022
1023 // Generate a mask operand.
1024 EVT MaskTy = VT.changeVectorElementTypeToInteger();
1025
1026 // What is the size of each element in the vector mask.
1027 EVT BitTy = MaskTy.getScalarType();
1028
1029 Mask = DAG.getSelect(DL, BitTy, Mask,
1030 DAG.getConstant(APInt::getAllOnesValue(BitTy.getSizeInBits()), DL,
1031 BitTy),
1032 DAG.getConstant(0, DL, BitTy));
1033
1034 // Broadcast the mask so that the entire vector is all-one or all zero.
1035 Mask = DAG.getSplatBuildVector(MaskTy, DL, Mask);
1036
1037 // Bitcast the operands to be the same type as the mask.
1038 // This is needed when we select between FP types because
1039 // the mask is a vector of integers.
1040 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
1041 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
1042
1043 SDValue AllOnes = DAG.getConstant(
1044 APInt::getAllOnesValue(BitTy.getSizeInBits()), DL, MaskTy);
1045 SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes);
1046
1047 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
1048 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
1049 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
1050 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1051 }
1052
ExpandSEXTINREG(SDNode * Node)1053 SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
1054 EVT VT = Node->getValueType(0);
1055
1056 // Make sure that the SRA and SHL instructions are available.
1057 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
1058 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
1059 return DAG.UnrollVectorOp(Node);
1060
1061 SDLoc DL(Node);
1062 EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
1063
1064 unsigned BW = VT.getScalarSizeInBits();
1065 unsigned OrigBW = OrigTy.getScalarSizeInBits();
1066 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
1067
1068 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
1069 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
1070 }
1071
1072 // Generically expand a vector anyext in register to a shuffle of the relevant
1073 // lanes into the appropriate locations, with other lanes left undef.
ExpandANY_EXTEND_VECTOR_INREG(SDNode * Node)1074 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
1075 SDLoc DL(Node);
1076 EVT VT = Node->getValueType(0);
1077 int NumElements = VT.getVectorNumElements();
1078 SDValue Src = Node->getOperand(0);
1079 EVT SrcVT = Src.getValueType();
1080 int NumSrcElements = SrcVT.getVectorNumElements();
1081
1082 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1083 // into a larger vector type.
1084 if (SrcVT.bitsLE(VT)) {
1085 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1086 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1087 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1088 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1089 NumSrcElements);
1090 Src = DAG.getNode(
1091 ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT), Src,
1092 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
1093 }
1094
1095 // Build a base mask of undef shuffles.
1096 SmallVector<int, 16> ShuffleMask;
1097 ShuffleMask.resize(NumSrcElements, -1);
1098
1099 // Place the extended lanes into the correct locations.
1100 int ExtLaneScale = NumSrcElements / NumElements;
1101 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1102 for (int i = 0; i < NumElements; ++i)
1103 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1104
1105 return DAG.getNode(
1106 ISD::BITCAST, DL, VT,
1107 DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1108 }
1109
ExpandSIGN_EXTEND_VECTOR_INREG(SDNode * Node)1110 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1111 SDLoc DL(Node);
1112 EVT VT = Node->getValueType(0);
1113 SDValue Src = Node->getOperand(0);
1114 EVT SrcVT = Src.getValueType();
1115
1116 // First build an any-extend node which can be legalized above when we
1117 // recurse through it.
1118 SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1119
1120 // Now we need sign extend. Do this by shifting the elements. Even if these
1121 // aren't legal operations, they have a better chance of being legalized
1122 // without full scalarization than the sign extension does.
1123 unsigned EltWidth = VT.getScalarSizeInBits();
1124 unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1125 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1126 return DAG.getNode(ISD::SRA, DL, VT,
1127 DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1128 ShiftAmount);
1129 }
1130
1131 // Generically expand a vector zext in register to a shuffle of the relevant
1132 // lanes into the appropriate locations, a blend of zero into the high bits,
1133 // and a bitcast to the wider element type.
ExpandZERO_EXTEND_VECTOR_INREG(SDNode * Node)1134 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1135 SDLoc DL(Node);
1136 EVT VT = Node->getValueType(0);
1137 int NumElements = VT.getVectorNumElements();
1138 SDValue Src = Node->getOperand(0);
1139 EVT SrcVT = Src.getValueType();
1140 int NumSrcElements = SrcVT.getVectorNumElements();
1141
1142 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1143 // into a larger vector type.
1144 if (SrcVT.bitsLE(VT)) {
1145 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1146 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1147 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1148 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1149 NumSrcElements);
1150 Src = DAG.getNode(
1151 ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT), Src,
1152 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
1153 }
1154
1155 // Build up a zero vector to blend into this one.
1156 SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1157
1158 // Shuffle the incoming lanes into the correct position, and pull all other
1159 // lanes from the zero vector.
1160 SmallVector<int, 16> ShuffleMask;
1161 ShuffleMask.reserve(NumSrcElements);
1162 for (int i = 0; i < NumSrcElements; ++i)
1163 ShuffleMask.push_back(i);
1164
1165 int ExtLaneScale = NumSrcElements / NumElements;
1166 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1167 for (int i = 0; i < NumElements; ++i)
1168 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1169
1170 return DAG.getNode(ISD::BITCAST, DL, VT,
1171 DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1172 }
1173
createBSWAPShuffleMask(EVT VT,SmallVectorImpl<int> & ShuffleMask)1174 static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1175 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1176 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1177 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1178 ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1179 }
1180
ExpandBSWAP(SDNode * Node)1181 SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1182 EVT VT = Node->getValueType(0);
1183
1184 // Generate a byte wise shuffle mask for the BSWAP.
1185 SmallVector<int, 16> ShuffleMask;
1186 createBSWAPShuffleMask(VT, ShuffleMask);
1187 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1188
1189 // Only emit a shuffle if the mask is legal.
1190 if (!TLI.isShuffleMaskLegal(ShuffleMask, ByteVT))
1191 return DAG.UnrollVectorOp(Node);
1192
1193 SDLoc DL(Node);
1194 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1195 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1196 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1197 }
1198
ExpandBITREVERSE(SDNode * Node,SmallVectorImpl<SDValue> & Results)1199 void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1200 SmallVectorImpl<SDValue> &Results) {
1201 EVT VT = Node->getValueType(0);
1202
1203 // If we have the scalar operation, it's probably cheaper to unroll it.
1204 if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1205 SDValue Tmp = DAG.UnrollVectorOp(Node);
1206 Results.push_back(Tmp);
1207 return;
1208 }
1209
1210 // If the vector element width is a whole number of bytes, test if its legal
1211 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1212 // vector. This greatly reduces the number of bit shifts necessary.
1213 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1214 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1215 SmallVector<int, 16> BSWAPMask;
1216 createBSWAPShuffleMask(VT, BSWAPMask);
1217
1218 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1219 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1220 (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1221 (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1222 TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1223 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1224 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1225 SDLoc DL(Node);
1226 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1227 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1228 BSWAPMask);
1229 Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1230 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1231 Results.push_back(Op);
1232 return;
1233 }
1234 }
1235
1236 // If we have the appropriate vector bit operations, it is better to use them
1237 // than unrolling and expanding each component.
1238 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1239 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1240 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1241 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1242 // Let LegalizeDAG handle this later.
1243 return;
1244
1245 // Otherwise unroll.
1246 SDValue Tmp = DAG.UnrollVectorOp(Node);
1247 Results.push_back(Tmp);
1248 }
1249
ExpandVSELECT(SDNode * Node)1250 SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1251 // Implement VSELECT in terms of XOR, AND, OR
1252 // on platforms which do not support blend natively.
1253 SDLoc DL(Node);
1254
1255 SDValue Mask = Node->getOperand(0);
1256 SDValue Op1 = Node->getOperand(1);
1257 SDValue Op2 = Node->getOperand(2);
1258
1259 EVT VT = Mask.getValueType();
1260
1261 // If we can't even use the basic vector operations of
1262 // AND,OR,XOR, we will have to scalarize the op.
1263 // Notice that the operation may be 'promoted' which means that it is
1264 // 'bitcasted' to another type which is handled.
1265 // This operation also isn't safe with AND, OR, XOR when the boolean
1266 // type is 0/1 as we need an all ones vector constant to mask with.
1267 // FIXME: Sign extend 1 to all ones if thats legal on the target.
1268 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1269 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1270 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
1271 TLI.getBooleanContents(Op1.getValueType()) !=
1272 TargetLowering::ZeroOrNegativeOneBooleanContent)
1273 return DAG.UnrollVectorOp(Node);
1274
1275 // If the mask and the type are different sizes, unroll the vector op. This
1276 // can occur when getSetCCResultType returns something that is different in
1277 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1278 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1279 return DAG.UnrollVectorOp(Node);
1280
1281 // Bitcast the operands to be the same type as the mask.
1282 // This is needed when we select between FP types because
1283 // the mask is a vector of integers.
1284 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1285 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1286
1287 SDValue AllOnes = DAG.getConstant(
1288 APInt::getAllOnesValue(VT.getScalarSizeInBits()), DL, VT);
1289 SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes);
1290
1291 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1292 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1293 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1294 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1295 }
1296
ExpandFP_TO_UINT(SDNode * Node,SmallVectorImpl<SDValue> & Results)1297 void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1298 SmallVectorImpl<SDValue> &Results) {
1299 // Attempt to expand using TargetLowering.
1300 SDValue Result, Chain;
1301 if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1302 Results.push_back(Result);
1303 if (Node->isStrictFPOpcode())
1304 Results.push_back(Chain);
1305 return;
1306 }
1307
1308 // Otherwise go ahead and unroll.
1309 if (Node->isStrictFPOpcode()) {
1310 UnrollStrictFPOp(Node, Results);
1311 return;
1312 }
1313
1314 Results.push_back(DAG.UnrollVectorOp(Node));
1315 }
1316
ExpandUINT_TO_FLOAT(SDNode * Node,SmallVectorImpl<SDValue> & Results)1317 void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1318 SmallVectorImpl<SDValue> &Results) {
1319 bool IsStrict = Node->isStrictFPOpcode();
1320 unsigned OpNo = IsStrict ? 1 : 0;
1321 SDValue Src = Node->getOperand(OpNo);
1322 EVT VT = Src.getValueType();
1323 SDLoc DL(Node);
1324
1325 // Attempt to expand using TargetLowering.
1326 SDValue Result;
1327 SDValue Chain;
1328 if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1329 Results.push_back(Result);
1330 if (IsStrict)
1331 Results.push_back(Chain);
1332 return;
1333 }
1334
1335 // Make sure that the SINT_TO_FP and SRL instructions are available.
1336 if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1337 TargetLowering::Expand) ||
1338 (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1339 TargetLowering::Expand)) ||
1340 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1341 if (IsStrict) {
1342 UnrollStrictFPOp(Node, Results);
1343 return;
1344 }
1345
1346 Results.push_back(DAG.UnrollVectorOp(Node));
1347 return;
1348 }
1349
1350 unsigned BW = VT.getScalarSizeInBits();
1351 assert((BW == 64 || BW == 32) &&
1352 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1353
1354 SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1355
1356 // Constants to clear the upper part of the word.
1357 // Notice that we can also use SHL+SHR, but using a constant is slightly
1358 // faster on x86.
1359 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1360 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1361
1362 // Two to the power of half-word-size.
1363 SDValue TWOHW =
1364 DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1365
1366 // Clear upper part of LO, lower HI
1367 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1368 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1369
1370 if (IsStrict) {
1371 // Convert hi and lo to floats
1372 // Convert the hi part back to the upper values
1373 // TODO: Can any fast-math-flags be set on these nodes?
1374 SDValue fHI = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL,
1375 {Node->getValueType(0), MVT::Other},
1376 {Node->getOperand(0), HI});
1377 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1378 {fHI.getValue(1), fHI, TWOHW});
1379 SDValue fLO = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL,
1380 {Node->getValueType(0), MVT::Other},
1381 {Node->getOperand(0), LO});
1382
1383 SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1),
1384 fLO.getValue(1));
1385
1386 // Add the two halves
1387 SDValue Result =
1388 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1389 {TF, fHI, fLO});
1390
1391 Results.push_back(Result);
1392 Results.push_back(Result.getValue(1));
1393 return;
1394 }
1395
1396 // Convert hi and lo to floats
1397 // Convert the hi part back to the upper values
1398 // TODO: Can any fast-math-flags be set on these nodes?
1399 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1400 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1401 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1402
1403 // Add the two halves
1404 Results.push_back(
1405 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1406 }
1407
ExpandFNEG(SDNode * Node)1408 SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1409 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1410 SDLoc DL(Node);
1411 SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1412 // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1413 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1414 Node->getOperand(0));
1415 }
1416 return DAG.UnrollVectorOp(Node);
1417 }
1418
ExpandFSUB(SDNode * Node,SmallVectorImpl<SDValue> & Results)1419 void VectorLegalizer::ExpandFSUB(SDNode *Node,
1420 SmallVectorImpl<SDValue> &Results) {
1421 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1422 // we can defer this to operation legalization where it will be lowered as
1423 // a+(-b).
1424 EVT VT = Node->getValueType(0);
1425 if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1426 TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1427 return; // Defer to LegalizeDAG
1428
1429 SDValue Tmp = DAG.UnrollVectorOp(Node);
1430 Results.push_back(Tmp);
1431 }
1432
ExpandUADDSUBO(SDNode * Node,SmallVectorImpl<SDValue> & Results)1433 void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1434 SmallVectorImpl<SDValue> &Results) {
1435 SDValue Result, Overflow;
1436 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1437 Results.push_back(Result);
1438 Results.push_back(Overflow);
1439 }
1440
ExpandSADDSUBO(SDNode * Node,SmallVectorImpl<SDValue> & Results)1441 void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1442 SmallVectorImpl<SDValue> &Results) {
1443 SDValue Result, Overflow;
1444 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1445 Results.push_back(Result);
1446 Results.push_back(Overflow);
1447 }
1448
ExpandMULO(SDNode * Node,SmallVectorImpl<SDValue> & Results)1449 void VectorLegalizer::ExpandMULO(SDNode *Node,
1450 SmallVectorImpl<SDValue> &Results) {
1451 SDValue Result, Overflow;
1452 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1453 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1454
1455 Results.push_back(Result);
1456 Results.push_back(Overflow);
1457 }
1458
ExpandFixedPointDiv(SDNode * Node)1459 SDValue VectorLegalizer::ExpandFixedPointDiv(SDNode *Node) {
1460 SDNode *N = Node;
1461 if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1462 N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1463 return Expanded;
1464 return DAG.UnrollVectorOp(N);
1465 }
1466
ExpandStrictFPOp(SDNode * Node,SmallVectorImpl<SDValue> & Results)1467 void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1468 SmallVectorImpl<SDValue> &Results) {
1469 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1470 ExpandUINT_TO_FLOAT(Node, Results);
1471 return;
1472 }
1473 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1474 ExpandFP_TO_UINT(Node, Results);
1475 return;
1476 }
1477
1478 UnrollStrictFPOp(Node, Results);
1479 }
1480
UnrollStrictFPOp(SDNode * Node,SmallVectorImpl<SDValue> & Results)1481 void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1482 SmallVectorImpl<SDValue> &Results) {
1483 EVT VT = Node->getValueType(0);
1484 EVT EltVT = VT.getVectorElementType();
1485 unsigned NumElems = VT.getVectorNumElements();
1486 unsigned NumOpers = Node->getNumOperands();
1487 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1488
1489 EVT TmpEltVT = EltVT;
1490 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1491 Node->getOpcode() == ISD::STRICT_FSETCCS)
1492 TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1493 *DAG.getContext(), TmpEltVT);
1494
1495 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1496 SDValue Chain = Node->getOperand(0);
1497 SDLoc dl(Node);
1498
1499 SmallVector<SDValue, 32> OpValues;
1500 SmallVector<SDValue, 32> OpChains;
1501 for (unsigned i = 0; i < NumElems; ++i) {
1502 SmallVector<SDValue, 4> Opers;
1503 SDValue Idx = DAG.getConstant(i, dl,
1504 TLI.getVectorIdxTy(DAG.getDataLayout()));
1505
1506 // The Chain is the first operand.
1507 Opers.push_back(Chain);
1508
1509 // Now process the remaining operands.
1510 for (unsigned j = 1; j < NumOpers; ++j) {
1511 SDValue Oper = Node->getOperand(j);
1512 EVT OperVT = Oper.getValueType();
1513
1514 if (OperVT.isVector())
1515 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1516 OperVT.getVectorElementType(), Oper, Idx);
1517
1518 Opers.push_back(Oper);
1519 }
1520
1521 SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1522 SDValue ScalarResult = ScalarOp.getValue(0);
1523 SDValue ScalarChain = ScalarOp.getValue(1);
1524
1525 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1526 Node->getOpcode() == ISD::STRICT_FSETCCS)
1527 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1528 DAG.getConstant(APInt::getAllOnesValue
1529 (EltVT.getSizeInBits()), dl, EltVT),
1530 DAG.getConstant(0, dl, EltVT));
1531
1532 OpValues.push_back(ScalarResult);
1533 OpChains.push_back(ScalarChain);
1534 }
1535
1536 SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1537 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1538
1539 Results.push_back(Result);
1540 Results.push_back(NewChain);
1541 }
1542
UnrollVSETCC(SDNode * Node)1543 SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
1544 EVT VT = Node->getValueType(0);
1545 unsigned NumElems = VT.getVectorNumElements();
1546 EVT EltVT = VT.getVectorElementType();
1547 SDValue LHS = Node->getOperand(0);
1548 SDValue RHS = Node->getOperand(1);
1549 SDValue CC = Node->getOperand(2);
1550 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
1551 SDLoc dl(Node);
1552 SmallVector<SDValue, 8> Ops(NumElems);
1553 for (unsigned i = 0; i < NumElems; ++i) {
1554 SDValue LHSElem = DAG.getNode(
1555 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
1556 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
1557 SDValue RHSElem = DAG.getNode(
1558 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
1559 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
1560 Ops[i] = DAG.getNode(ISD::SETCC, dl,
1561 TLI.getSetCCResultType(DAG.getDataLayout(),
1562 *DAG.getContext(), TmpEltVT),
1563 LHSElem, RHSElem, CC);
1564 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i],
1565 DAG.getConstant(APInt::getAllOnesValue
1566 (EltVT.getSizeInBits()), dl, EltVT),
1567 DAG.getConstant(0, dl, EltVT));
1568 }
1569 return DAG.getBuildVector(VT, dl, Ops);
1570 }
1571
LegalizeVectors()1572 bool SelectionDAG::LegalizeVectors() {
1573 return VectorLegalizer(*this).Run();
1574 }
1575