1 //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the SelectionDAG class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/CodeGen/SelectionDAG.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/APSInt.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/BitVector.h"
20 #include "llvm/ADT/DenseSet.h"
21 #include "llvm/ADT/FoldingSet.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallPtrSet.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/Analysis/AliasAnalysis.h"
27 #include "llvm/Analysis/MemoryLocation.h"
28 #include "llvm/Analysis/ValueTracking.h"
29 #include "llvm/Analysis/VectorUtils.h"
30 #include "llvm/BinaryFormat/Dwarf.h"
31 #include "llvm/CodeGen/Analysis.h"
32 #include "llvm/CodeGen/FunctionLoweringInfo.h"
33 #include "llvm/CodeGen/ISDOpcodes.h"
34 #include "llvm/CodeGen/MachineBasicBlock.h"
35 #include "llvm/CodeGen/MachineConstantPool.h"
36 #include "llvm/CodeGen/MachineFrameInfo.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineMemOperand.h"
39 #include "llvm/CodeGen/MachineValueType.h"
40 #include "llvm/CodeGen/RuntimeLibcalls.h"
41 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
42 #include "llvm/CodeGen/SelectionDAGNodes.h"
43 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
44 #include "llvm/CodeGen/TargetFrameLowering.h"
45 #include "llvm/CodeGen/TargetLowering.h"
46 #include "llvm/CodeGen/TargetRegisterInfo.h"
47 #include "llvm/CodeGen/TargetSubtargetInfo.h"
48 #include "llvm/CodeGen/ValueTypes.h"
49 #include "llvm/IR/Constant.h"
50 #include "llvm/IR/ConstantRange.h"
51 #include "llvm/IR/Constants.h"
52 #include "llvm/IR/DataLayout.h"
53 #include "llvm/IR/DebugInfoMetadata.h"
54 #include "llvm/IR/DebugLoc.h"
55 #include "llvm/IR/DerivedTypes.h"
56 #include "llvm/IR/Function.h"
57 #include "llvm/IR/GlobalValue.h"
58 #include "llvm/IR/Metadata.h"
59 #include "llvm/IR/Type.h"
60 #include "llvm/Support/Casting.h"
61 #include "llvm/Support/CodeGen.h"
62 #include "llvm/Support/Compiler.h"
63 #include "llvm/Support/Debug.h"
64 #include "llvm/Support/ErrorHandling.h"
65 #include "llvm/Support/KnownBits.h"
66 #include "llvm/Support/MathExtras.h"
67 #include "llvm/Support/Mutex.h"
68 #include "llvm/Support/raw_ostream.h"
69 #include "llvm/Target/TargetMachine.h"
70 #include "llvm/Target/TargetOptions.h"
71 #include "llvm/TargetParser/Triple.h"
72 #include "llvm/Transforms/Utils/SizeOpts.h"
73 #include <algorithm>
74 #include <cassert>
75 #include <cstdint>
76 #include <cstdlib>
77 #include <limits>
78 #include <set>
79 #include <string>
80 #include <utility>
81 #include <vector>
82
83 using namespace llvm;
84
85 /// makeVTList - Return an instance of the SDVTList struct initialized with the
86 /// specified members.
makeVTList(const EVT * VTs,unsigned NumVTs)87 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
88 SDVTList Res = {VTs, NumVTs};
89 return Res;
90 }
91
92 // Default null implementations of the callbacks.
NodeDeleted(SDNode *,SDNode *)93 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
NodeUpdated(SDNode *)94 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
NodeInserted(SDNode *)95 void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
96
anchor()97 void SelectionDAG::DAGNodeDeletedListener::anchor() {}
anchor()98 void SelectionDAG::DAGNodeInsertedListener::anchor() {}
99
100 #define DEBUG_TYPE "selectiondag"
101
102 static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
103 cl::Hidden, cl::init(true),
104 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
105
106 static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
107 cl::desc("Number limit for gluing ld/st of memcpy."),
108 cl::Hidden, cl::init(0));
109
NewSDValueDbgMsg(SDValue V,StringRef Msg,SelectionDAG * G)110 static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
111 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
112 }
113
114 //===----------------------------------------------------------------------===//
115 // ConstantFPSDNode Class
116 //===----------------------------------------------------------------------===//
117
118 /// isExactlyValue - We don't rely on operator== working on double values, as
119 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
120 /// As such, this method can be used to do an exact bit-for-bit comparison of
121 /// two floating point values.
isExactlyValue(const APFloat & V) const122 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
123 return getValueAPF().bitwiseIsEqual(V);
124 }
125
isValueValidForType(EVT VT,const APFloat & Val)126 bool ConstantFPSDNode::isValueValidForType(EVT VT,
127 const APFloat& Val) {
128 assert(VT.isFloatingPoint() && "Can only convert between FP types");
129
130 // convert modifies in place, so make a copy.
131 APFloat Val2 = APFloat(Val);
132 bool losesInfo;
133 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
134 APFloat::rmNearestTiesToEven,
135 &losesInfo);
136 return !losesInfo;
137 }
138
139 //===----------------------------------------------------------------------===//
140 // ISD Namespace
141 //===----------------------------------------------------------------------===//
142
isConstantSplatVector(const SDNode * N,APInt & SplatVal)143 bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
144 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
145 unsigned EltSize =
146 N->getValueType(0).getVectorElementType().getSizeInBits();
147 if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
148 SplatVal = Op0->getAPIntValue().trunc(EltSize);
149 return true;
150 }
151 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
152 SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(EltSize);
153 return true;
154 }
155 }
156
157 auto *BV = dyn_cast<BuildVectorSDNode>(N);
158 if (!BV)
159 return false;
160
161 APInt SplatUndef;
162 unsigned SplatBitSize;
163 bool HasUndefs;
164 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
165 // Endianness does not matter here. We are checking for a splat given the
166 // element size of the vector, and if we find such a splat for little endian
167 // layout, then that should be valid also for big endian (as the full vector
168 // size is known to be a multiple of the element size).
169 const bool IsBigEndian = false;
170 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
171 EltSize, IsBigEndian) &&
172 EltSize == SplatBitSize;
173 }
174
175 // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
176 // specializations of the more general isConstantSplatVector()?
177
isConstantSplatVectorAllOnes(const SDNode * N,bool BuildVectorOnly)178 bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
179 // Look through a bit convert.
180 while (N->getOpcode() == ISD::BITCAST)
181 N = N->getOperand(0).getNode();
182
183 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
184 APInt SplatVal;
185 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
186 }
187
188 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
189
190 unsigned i = 0, e = N->getNumOperands();
191
192 // Skip over all of the undef values.
193 while (i != e && N->getOperand(i).isUndef())
194 ++i;
195
196 // Do not accept an all-undef vector.
197 if (i == e) return false;
198
199 // Do not accept build_vectors that aren't all constants or which have non-~0
200 // elements. We have to be a bit careful here, as the type of the constant
201 // may not be the same as the type of the vector elements due to type
202 // legalization (the elements are promoted to a legal type for the target and
203 // a vector of a type may be legal when the base element type is not).
204 // We only want to check enough bits to cover the vector elements, because
205 // we care if the resultant vector is all ones, not whether the individual
206 // constants are.
207 SDValue NotZero = N->getOperand(i);
208 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
209 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
210 if (CN->getAPIntValue().countr_one() < EltSize)
211 return false;
212 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
213 if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
214 return false;
215 } else
216 return false;
217
218 // Okay, we have at least one ~0 value, check to see if the rest match or are
219 // undefs. Even with the above element type twiddling, this should be OK, as
220 // the same type legalization should have applied to all the elements.
221 for (++i; i != e; ++i)
222 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
223 return false;
224 return true;
225 }
226
isConstantSplatVectorAllZeros(const SDNode * N,bool BuildVectorOnly)227 bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
228 // Look through a bit convert.
229 while (N->getOpcode() == ISD::BITCAST)
230 N = N->getOperand(0).getNode();
231
232 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
233 APInt SplatVal;
234 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
235 }
236
237 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
238
239 bool IsAllUndef = true;
240 for (const SDValue &Op : N->op_values()) {
241 if (Op.isUndef())
242 continue;
243 IsAllUndef = false;
244 // Do not accept build_vectors that aren't all constants or which have non-0
245 // elements. We have to be a bit careful here, as the type of the constant
246 // may not be the same as the type of the vector elements due to type
247 // legalization (the elements are promoted to a legal type for the target
248 // and a vector of a type may be legal when the base element type is not).
249 // We only want to check enough bits to cover the vector elements, because
250 // we care if the resultant vector is all zeros, not whether the individual
251 // constants are.
252 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
253 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
254 if (CN->getAPIntValue().countr_zero() < EltSize)
255 return false;
256 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
257 if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
258 return false;
259 } else
260 return false;
261 }
262
263 // Do not accept an all-undef vector.
264 if (IsAllUndef)
265 return false;
266 return true;
267 }
268
isBuildVectorAllOnes(const SDNode * N)269 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
270 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
271 }
272
isBuildVectorAllZeros(const SDNode * N)273 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
274 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
275 }
276
isBuildVectorOfConstantSDNodes(const SDNode * N)277 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
278 if (N->getOpcode() != ISD::BUILD_VECTOR)
279 return false;
280
281 for (const SDValue &Op : N->op_values()) {
282 if (Op.isUndef())
283 continue;
284 if (!isa<ConstantSDNode>(Op))
285 return false;
286 }
287 return true;
288 }
289
isBuildVectorOfConstantFPSDNodes(const SDNode * N)290 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
291 if (N->getOpcode() != ISD::BUILD_VECTOR)
292 return false;
293
294 for (const SDValue &Op : N->op_values()) {
295 if (Op.isUndef())
296 continue;
297 if (!isa<ConstantFPSDNode>(Op))
298 return false;
299 }
300 return true;
301 }
302
isVectorShrinkable(const SDNode * N,unsigned NewEltSize,bool Signed)303 bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
304 bool Signed) {
305 assert(N->getValueType(0).isVector() && "Expected a vector!");
306
307 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
308 if (EltSize <= NewEltSize)
309 return false;
310
311 if (N->getOpcode() == ISD::ZERO_EXTEND) {
312 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
313 NewEltSize) &&
314 !Signed;
315 }
316 if (N->getOpcode() == ISD::SIGN_EXTEND) {
317 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
318 NewEltSize) &&
319 Signed;
320 }
321 if (N->getOpcode() != ISD::BUILD_VECTOR)
322 return false;
323
324 for (const SDValue &Op : N->op_values()) {
325 if (Op.isUndef())
326 continue;
327 if (!isa<ConstantSDNode>(Op))
328 return false;
329
330 APInt C = Op->getAsAPIntVal().trunc(EltSize);
331 if (Signed && C.trunc(NewEltSize).sext(EltSize) != C)
332 return false;
333 if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C)
334 return false;
335 }
336
337 return true;
338 }
339
allOperandsUndef(const SDNode * N)340 bool ISD::allOperandsUndef(const SDNode *N) {
341 // Return false if the node has no operands.
342 // This is "logically inconsistent" with the definition of "all" but
343 // is probably the desired behavior.
344 if (N->getNumOperands() == 0)
345 return false;
346 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
347 }
348
isFreezeUndef(const SDNode * N)349 bool ISD::isFreezeUndef(const SDNode *N) {
350 return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef();
351 }
352
353 template <typename ConstNodeType>
matchUnaryPredicateImpl(SDValue Op,std::function<bool (ConstNodeType *)> Match,bool AllowUndefs)354 bool ISD::matchUnaryPredicateImpl(SDValue Op,
355 std::function<bool(ConstNodeType *)> Match,
356 bool AllowUndefs) {
357 // FIXME: Add support for scalar UNDEF cases?
358 if (auto *C = dyn_cast<ConstNodeType>(Op))
359 return Match(C);
360
361 // FIXME: Add support for vector UNDEF cases?
362 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
363 ISD::SPLAT_VECTOR != Op.getOpcode())
364 return false;
365
366 EVT SVT = Op.getValueType().getScalarType();
367 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
368 if (AllowUndefs && Op.getOperand(i).isUndef()) {
369 if (!Match(nullptr))
370 return false;
371 continue;
372 }
373
374 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
375 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
376 return false;
377 }
378 return true;
379 }
380 // Build used template types.
381 template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
382 SDValue, std::function<bool(ConstantSDNode *)>, bool);
383 template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
384 SDValue, std::function<bool(ConstantFPSDNode *)>, bool);
385
matchBinaryPredicate(SDValue LHS,SDValue RHS,std::function<bool (ConstantSDNode *,ConstantSDNode *)> Match,bool AllowUndefs,bool AllowTypeMismatch)386 bool ISD::matchBinaryPredicate(
387 SDValue LHS, SDValue RHS,
388 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
389 bool AllowUndefs, bool AllowTypeMismatch) {
390 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
391 return false;
392
393 // TODO: Add support for scalar UNDEF cases?
394 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
395 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
396 return Match(LHSCst, RHSCst);
397
398 // TODO: Add support for vector UNDEF cases?
399 if (LHS.getOpcode() != RHS.getOpcode() ||
400 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
401 LHS.getOpcode() != ISD::SPLAT_VECTOR))
402 return false;
403
404 EVT SVT = LHS.getValueType().getScalarType();
405 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
406 SDValue LHSOp = LHS.getOperand(i);
407 SDValue RHSOp = RHS.getOperand(i);
408 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
409 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
410 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
411 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
412 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
413 return false;
414 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
415 LHSOp.getValueType() != RHSOp.getValueType()))
416 return false;
417 if (!Match(LHSCst, RHSCst))
418 return false;
419 }
420 return true;
421 }
422
getVecReduceBaseOpcode(unsigned VecReduceOpcode)423 ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
424 switch (VecReduceOpcode) {
425 default:
426 llvm_unreachable("Expected VECREDUCE opcode");
427 case ISD::VECREDUCE_FADD:
428 case ISD::VECREDUCE_SEQ_FADD:
429 case ISD::VP_REDUCE_FADD:
430 case ISD::VP_REDUCE_SEQ_FADD:
431 return ISD::FADD;
432 case ISD::VECREDUCE_FMUL:
433 case ISD::VECREDUCE_SEQ_FMUL:
434 case ISD::VP_REDUCE_FMUL:
435 case ISD::VP_REDUCE_SEQ_FMUL:
436 return ISD::FMUL;
437 case ISD::VECREDUCE_ADD:
438 case ISD::VP_REDUCE_ADD:
439 return ISD::ADD;
440 case ISD::VECREDUCE_MUL:
441 case ISD::VP_REDUCE_MUL:
442 return ISD::MUL;
443 case ISD::VECREDUCE_AND:
444 case ISD::VP_REDUCE_AND:
445 return ISD::AND;
446 case ISD::VECREDUCE_OR:
447 case ISD::VP_REDUCE_OR:
448 return ISD::OR;
449 case ISD::VECREDUCE_XOR:
450 case ISD::VP_REDUCE_XOR:
451 return ISD::XOR;
452 case ISD::VECREDUCE_SMAX:
453 case ISD::VP_REDUCE_SMAX:
454 return ISD::SMAX;
455 case ISD::VECREDUCE_SMIN:
456 case ISD::VP_REDUCE_SMIN:
457 return ISD::SMIN;
458 case ISD::VECREDUCE_UMAX:
459 case ISD::VP_REDUCE_UMAX:
460 return ISD::UMAX;
461 case ISD::VECREDUCE_UMIN:
462 case ISD::VP_REDUCE_UMIN:
463 return ISD::UMIN;
464 case ISD::VECREDUCE_FMAX:
465 case ISD::VP_REDUCE_FMAX:
466 return ISD::FMAXNUM;
467 case ISD::VECREDUCE_FMIN:
468 case ISD::VP_REDUCE_FMIN:
469 return ISD::FMINNUM;
470 case ISD::VECREDUCE_FMAXIMUM:
471 return ISD::FMAXIMUM;
472 case ISD::VECREDUCE_FMINIMUM:
473 return ISD::FMINIMUM;
474 }
475 }
476
isVPOpcode(unsigned Opcode)477 bool ISD::isVPOpcode(unsigned Opcode) {
478 switch (Opcode) {
479 default:
480 return false;
481 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
482 case ISD::VPSD: \
483 return true;
484 #include "llvm/IR/VPIntrinsics.def"
485 }
486 }
487
isVPBinaryOp(unsigned Opcode)488 bool ISD::isVPBinaryOp(unsigned Opcode) {
489 switch (Opcode) {
490 default:
491 break;
492 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
493 #define VP_PROPERTY_BINARYOP return true;
494 #define END_REGISTER_VP_SDNODE(VPSD) break;
495 #include "llvm/IR/VPIntrinsics.def"
496 }
497 return false;
498 }
499
isVPReduction(unsigned Opcode)500 bool ISD::isVPReduction(unsigned Opcode) {
501 switch (Opcode) {
502 default:
503 break;
504 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
505 #define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
506 #define END_REGISTER_VP_SDNODE(VPSD) break;
507 #include "llvm/IR/VPIntrinsics.def"
508 }
509 return false;
510 }
511
512 /// The operand position of the vector mask.
getVPMaskIdx(unsigned Opcode)513 std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
514 switch (Opcode) {
515 default:
516 return std::nullopt;
517 #define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
518 case ISD::VPSD: \
519 return MASKPOS;
520 #include "llvm/IR/VPIntrinsics.def"
521 }
522 }
523
524 /// The operand position of the explicit vector length parameter.
getVPExplicitVectorLengthIdx(unsigned Opcode)525 std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
526 switch (Opcode) {
527 default:
528 return std::nullopt;
529 #define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
530 case ISD::VPSD: \
531 return EVLPOS;
532 #include "llvm/IR/VPIntrinsics.def"
533 }
534 }
535
getBaseOpcodeForVP(unsigned VPOpcode,bool hasFPExcept)536 std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
537 bool hasFPExcept) {
538 // FIXME: Return strict opcodes in case of fp exceptions.
539 switch (VPOpcode) {
540 default:
541 return std::nullopt;
542 #define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
543 #define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
544 #define END_REGISTER_VP_SDNODE(VPOPC) break;
545 #include "llvm/IR/VPIntrinsics.def"
546 }
547 return std::nullopt;
548 }
549
getVPForBaseOpcode(unsigned Opcode)550 unsigned ISD::getVPForBaseOpcode(unsigned Opcode) {
551 switch (Opcode) {
552 default:
553 llvm_unreachable("can not translate this Opcode to VP.");
554 #define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
555 #define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
556 #define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
557 #include "llvm/IR/VPIntrinsics.def"
558 }
559 }
560
getExtForLoadExtType(bool IsFP,ISD::LoadExtType ExtType)561 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
562 switch (ExtType) {
563 case ISD::EXTLOAD:
564 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
565 case ISD::SEXTLOAD:
566 return ISD::SIGN_EXTEND;
567 case ISD::ZEXTLOAD:
568 return ISD::ZERO_EXTEND;
569 default:
570 break;
571 }
572
573 llvm_unreachable("Invalid LoadExtType");
574 }
575
getSetCCSwappedOperands(ISD::CondCode Operation)576 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
577 // To perform this operation, we just need to swap the L and G bits of the
578 // operation.
579 unsigned OldL = (Operation >> 2) & 1;
580 unsigned OldG = (Operation >> 1) & 1;
581 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
582 (OldL << 1) | // New G bit
583 (OldG << 2)); // New L bit.
584 }
585
getSetCCInverseImpl(ISD::CondCode Op,bool isIntegerLike)586 static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
587 unsigned Operation = Op;
588 if (isIntegerLike)
589 Operation ^= 7; // Flip L, G, E bits, but not U.
590 else
591 Operation ^= 15; // Flip all of the condition bits.
592
593 if (Operation > ISD::SETTRUE2)
594 Operation &= ~8; // Don't let N and U bits get set.
595
596 return ISD::CondCode(Operation);
597 }
598
getSetCCInverse(ISD::CondCode Op,EVT Type)599 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
600 return getSetCCInverseImpl(Op, Type.isInteger());
601 }
602
getSetCCInverse(ISD::CondCode Op,bool isIntegerLike)603 ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
604 bool isIntegerLike) {
605 return getSetCCInverseImpl(Op, isIntegerLike);
606 }
607
608 /// For an integer comparison, return 1 if the comparison is a signed operation
609 /// and 2 if the result is an unsigned comparison. Return zero if the operation
610 /// does not depend on the sign of the input (setne and seteq).
isSignedOp(ISD::CondCode Opcode)611 static int isSignedOp(ISD::CondCode Opcode) {
612 switch (Opcode) {
613 default: llvm_unreachable("Illegal integer setcc operation!");
614 case ISD::SETEQ:
615 case ISD::SETNE: return 0;
616 case ISD::SETLT:
617 case ISD::SETLE:
618 case ISD::SETGT:
619 case ISD::SETGE: return 1;
620 case ISD::SETULT:
621 case ISD::SETULE:
622 case ISD::SETUGT:
623 case ISD::SETUGE: return 2;
624 }
625 }
626
getSetCCOrOperation(ISD::CondCode Op1,ISD::CondCode Op2,EVT Type)627 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
628 EVT Type) {
629 bool IsInteger = Type.isInteger();
630 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
631 // Cannot fold a signed integer setcc with an unsigned integer setcc.
632 return ISD::SETCC_INVALID;
633
634 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
635
636 // If the N and U bits get set, then the resultant comparison DOES suddenly
637 // care about orderedness, and it is true when ordered.
638 if (Op > ISD::SETTRUE2)
639 Op &= ~16; // Clear the U bit if the N bit is set.
640
641 // Canonicalize illegal integer setcc's.
642 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
643 Op = ISD::SETNE;
644
645 return ISD::CondCode(Op);
646 }
647
getSetCCAndOperation(ISD::CondCode Op1,ISD::CondCode Op2,EVT Type)648 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
649 EVT Type) {
650 bool IsInteger = Type.isInteger();
651 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
652 // Cannot fold a signed setcc with an unsigned setcc.
653 return ISD::SETCC_INVALID;
654
655 // Combine all of the condition bits.
656 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
657
658 // Canonicalize illegal integer setcc's.
659 if (IsInteger) {
660 switch (Result) {
661 default: break;
662 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
663 case ISD::SETOEQ: // SETEQ & SETU[LG]E
664 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
665 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
666 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
667 }
668 }
669
670 return Result;
671 }
672
673 //===----------------------------------------------------------------------===//
674 // SDNode Profile Support
675 //===----------------------------------------------------------------------===//
676
677 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
AddNodeIDOpcode(FoldingSetNodeID & ID,unsigned OpC)678 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
679 ID.AddInteger(OpC);
680 }
681
682 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
683 /// solely with their pointer.
AddNodeIDValueTypes(FoldingSetNodeID & ID,SDVTList VTList)684 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
685 ID.AddPointer(VTList.VTs);
686 }
687
688 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
AddNodeIDOperands(FoldingSetNodeID & ID,ArrayRef<SDValue> Ops)689 static void AddNodeIDOperands(FoldingSetNodeID &ID,
690 ArrayRef<SDValue> Ops) {
691 for (const auto &Op : Ops) {
692 ID.AddPointer(Op.getNode());
693 ID.AddInteger(Op.getResNo());
694 }
695 }
696
697 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
AddNodeIDOperands(FoldingSetNodeID & ID,ArrayRef<SDUse> Ops)698 static void AddNodeIDOperands(FoldingSetNodeID &ID,
699 ArrayRef<SDUse> Ops) {
700 for (const auto &Op : Ops) {
701 ID.AddPointer(Op.getNode());
702 ID.AddInteger(Op.getResNo());
703 }
704 }
705
AddNodeIDNode(FoldingSetNodeID & ID,unsigned OpC,SDVTList VTList,ArrayRef<SDValue> OpList)706 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
707 SDVTList VTList, ArrayRef<SDValue> OpList) {
708 AddNodeIDOpcode(ID, OpC);
709 AddNodeIDValueTypes(ID, VTList);
710 AddNodeIDOperands(ID, OpList);
711 }
712
713 /// If this is an SDNode with special info, add this info to the NodeID data.
AddNodeIDCustom(FoldingSetNodeID & ID,const SDNode * N)714 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
715 switch (N->getOpcode()) {
716 case ISD::TargetExternalSymbol:
717 case ISD::ExternalSymbol:
718 case ISD::MCSymbol:
719 llvm_unreachable("Should only be used on nodes with operands");
720 default: break; // Normal nodes don't need extra info.
721 case ISD::TargetConstant:
722 case ISD::Constant: {
723 const ConstantSDNode *C = cast<ConstantSDNode>(N);
724 ID.AddPointer(C->getConstantIntValue());
725 ID.AddBoolean(C->isOpaque());
726 break;
727 }
728 case ISD::TargetConstantFP:
729 case ISD::ConstantFP:
730 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
731 break;
732 case ISD::TargetGlobalAddress:
733 case ISD::GlobalAddress:
734 case ISD::TargetGlobalTLSAddress:
735 case ISD::GlobalTLSAddress: {
736 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
737 ID.AddPointer(GA->getGlobal());
738 ID.AddInteger(GA->getOffset());
739 ID.AddInteger(GA->getTargetFlags());
740 break;
741 }
742 case ISD::BasicBlock:
743 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
744 break;
745 case ISD::Register:
746 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
747 break;
748 case ISD::RegisterMask:
749 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
750 break;
751 case ISD::SRCVALUE:
752 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
753 break;
754 case ISD::FrameIndex:
755 case ISD::TargetFrameIndex:
756 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
757 break;
758 case ISD::LIFETIME_START:
759 case ISD::LIFETIME_END:
760 if (cast<LifetimeSDNode>(N)->hasOffset()) {
761 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
762 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
763 }
764 break;
765 case ISD::PSEUDO_PROBE:
766 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
767 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
768 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
769 break;
770 case ISD::JumpTable:
771 case ISD::TargetJumpTable:
772 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
773 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
774 break;
775 case ISD::ConstantPool:
776 case ISD::TargetConstantPool: {
777 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
778 ID.AddInteger(CP->getAlign().value());
779 ID.AddInteger(CP->getOffset());
780 if (CP->isMachineConstantPoolEntry())
781 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
782 else
783 ID.AddPointer(CP->getConstVal());
784 ID.AddInteger(CP->getTargetFlags());
785 break;
786 }
787 case ISD::TargetIndex: {
788 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
789 ID.AddInteger(TI->getIndex());
790 ID.AddInteger(TI->getOffset());
791 ID.AddInteger(TI->getTargetFlags());
792 break;
793 }
794 case ISD::LOAD: {
795 const LoadSDNode *LD = cast<LoadSDNode>(N);
796 ID.AddInteger(LD->getMemoryVT().getRawBits());
797 ID.AddInteger(LD->getRawSubclassData());
798 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
799 ID.AddInteger(LD->getMemOperand()->getFlags());
800 break;
801 }
802 case ISD::STORE: {
803 const StoreSDNode *ST = cast<StoreSDNode>(N);
804 ID.AddInteger(ST->getMemoryVT().getRawBits());
805 ID.AddInteger(ST->getRawSubclassData());
806 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
807 ID.AddInteger(ST->getMemOperand()->getFlags());
808 break;
809 }
810 case ISD::VP_LOAD: {
811 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(N);
812 ID.AddInteger(ELD->getMemoryVT().getRawBits());
813 ID.AddInteger(ELD->getRawSubclassData());
814 ID.AddInteger(ELD->getPointerInfo().getAddrSpace());
815 ID.AddInteger(ELD->getMemOperand()->getFlags());
816 break;
817 }
818 case ISD::VP_STORE: {
819 const VPStoreSDNode *EST = cast<VPStoreSDNode>(N);
820 ID.AddInteger(EST->getMemoryVT().getRawBits());
821 ID.AddInteger(EST->getRawSubclassData());
822 ID.AddInteger(EST->getPointerInfo().getAddrSpace());
823 ID.AddInteger(EST->getMemOperand()->getFlags());
824 break;
825 }
826 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
827 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(N);
828 ID.AddInteger(SLD->getMemoryVT().getRawBits());
829 ID.AddInteger(SLD->getRawSubclassData());
830 ID.AddInteger(SLD->getPointerInfo().getAddrSpace());
831 break;
832 }
833 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
834 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N);
835 ID.AddInteger(SST->getMemoryVT().getRawBits());
836 ID.AddInteger(SST->getRawSubclassData());
837 ID.AddInteger(SST->getPointerInfo().getAddrSpace());
838 break;
839 }
840 case ISD::VP_GATHER: {
841 const VPGatherSDNode *EG = cast<VPGatherSDNode>(N);
842 ID.AddInteger(EG->getMemoryVT().getRawBits());
843 ID.AddInteger(EG->getRawSubclassData());
844 ID.AddInteger(EG->getPointerInfo().getAddrSpace());
845 ID.AddInteger(EG->getMemOperand()->getFlags());
846 break;
847 }
848 case ISD::VP_SCATTER: {
849 const VPScatterSDNode *ES = cast<VPScatterSDNode>(N);
850 ID.AddInteger(ES->getMemoryVT().getRawBits());
851 ID.AddInteger(ES->getRawSubclassData());
852 ID.AddInteger(ES->getPointerInfo().getAddrSpace());
853 ID.AddInteger(ES->getMemOperand()->getFlags());
854 break;
855 }
856 case ISD::MLOAD: {
857 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
858 ID.AddInteger(MLD->getMemoryVT().getRawBits());
859 ID.AddInteger(MLD->getRawSubclassData());
860 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
861 ID.AddInteger(MLD->getMemOperand()->getFlags());
862 break;
863 }
864 case ISD::MSTORE: {
865 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
866 ID.AddInteger(MST->getMemoryVT().getRawBits());
867 ID.AddInteger(MST->getRawSubclassData());
868 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
869 ID.AddInteger(MST->getMemOperand()->getFlags());
870 break;
871 }
872 case ISD::MGATHER: {
873 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
874 ID.AddInteger(MG->getMemoryVT().getRawBits());
875 ID.AddInteger(MG->getRawSubclassData());
876 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
877 ID.AddInteger(MG->getMemOperand()->getFlags());
878 break;
879 }
880 case ISD::MSCATTER: {
881 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
882 ID.AddInteger(MS->getMemoryVT().getRawBits());
883 ID.AddInteger(MS->getRawSubclassData());
884 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
885 ID.AddInteger(MS->getMemOperand()->getFlags());
886 break;
887 }
888 case ISD::ATOMIC_CMP_SWAP:
889 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
890 case ISD::ATOMIC_SWAP:
891 case ISD::ATOMIC_LOAD_ADD:
892 case ISD::ATOMIC_LOAD_SUB:
893 case ISD::ATOMIC_LOAD_AND:
894 case ISD::ATOMIC_LOAD_CLR:
895 case ISD::ATOMIC_LOAD_OR:
896 case ISD::ATOMIC_LOAD_XOR:
897 case ISD::ATOMIC_LOAD_NAND:
898 case ISD::ATOMIC_LOAD_MIN:
899 case ISD::ATOMIC_LOAD_MAX:
900 case ISD::ATOMIC_LOAD_UMIN:
901 case ISD::ATOMIC_LOAD_UMAX:
902 case ISD::ATOMIC_LOAD:
903 case ISD::ATOMIC_STORE: {
904 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
905 ID.AddInteger(AT->getMemoryVT().getRawBits());
906 ID.AddInteger(AT->getRawSubclassData());
907 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
908 ID.AddInteger(AT->getMemOperand()->getFlags());
909 break;
910 }
911 case ISD::VECTOR_SHUFFLE: {
912 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
913 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
914 i != e; ++i)
915 ID.AddInteger(SVN->getMaskElt(i));
916 break;
917 }
918 case ISD::TargetBlockAddress:
919 case ISD::BlockAddress: {
920 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
921 ID.AddPointer(BA->getBlockAddress());
922 ID.AddInteger(BA->getOffset());
923 ID.AddInteger(BA->getTargetFlags());
924 break;
925 }
926 case ISD::AssertAlign:
927 ID.AddInteger(cast<AssertAlignSDNode>(N)->getAlign().value());
928 break;
929 case ISD::PREFETCH:
930 case ISD::INTRINSIC_VOID:
931 case ISD::INTRINSIC_W_CHAIN:
932 // Handled by MemIntrinsicSDNode check after the switch.
933 break;
934 } // end switch (N->getOpcode())
935
936 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
937 // to check.
938 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(N)) {
939 ID.AddInteger(MN->getRawSubclassData());
940 ID.AddInteger(MN->getPointerInfo().getAddrSpace());
941 ID.AddInteger(MN->getMemOperand()->getFlags());
942 ID.AddInteger(MN->getMemoryVT().getRawBits());
943 }
944 }
945
946 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
947 /// data.
AddNodeIDNode(FoldingSetNodeID & ID,const SDNode * N)948 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
949 AddNodeIDOpcode(ID, N->getOpcode());
950 // Add the return value info.
951 AddNodeIDValueTypes(ID, N->getVTList());
952 // Add the operand info.
953 AddNodeIDOperands(ID, N->ops());
954
955 // Handle SDNode leafs with special info.
956 AddNodeIDCustom(ID, N);
957 }
958
959 //===----------------------------------------------------------------------===//
960 // SelectionDAG Class
961 //===----------------------------------------------------------------------===//
962
963 /// doNotCSE - Return true if CSE should not be performed for this node.
doNotCSE(SDNode * N)964 static bool doNotCSE(SDNode *N) {
965 if (N->getValueType(0) == MVT::Glue)
966 return true; // Never CSE anything that produces a glue result.
967
968 switch (N->getOpcode()) {
969 default: break;
970 case ISD::HANDLENODE:
971 case ISD::EH_LABEL:
972 return true; // Never CSE these nodes.
973 }
974
975 // Check that remaining values produced are not flags.
976 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
977 if (N->getValueType(i) == MVT::Glue)
978 return true; // Never CSE anything that produces a glue result.
979
980 return false;
981 }
982
983 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
984 /// SelectionDAG.
RemoveDeadNodes()985 void SelectionDAG::RemoveDeadNodes() {
986 // Create a dummy node (which is not added to allnodes), that adds a reference
987 // to the root node, preventing it from being deleted.
988 HandleSDNode Dummy(getRoot());
989
990 SmallVector<SDNode*, 128> DeadNodes;
991
992 // Add all obviously-dead nodes to the DeadNodes worklist.
993 for (SDNode &Node : allnodes())
994 if (Node.use_empty())
995 DeadNodes.push_back(&Node);
996
997 RemoveDeadNodes(DeadNodes);
998
999 // If the root changed (e.g. it was a dead load, update the root).
1000 setRoot(Dummy.getValue());
1001 }
1002
1003 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
1004 /// given list, and any nodes that become unreachable as a result.
RemoveDeadNodes(SmallVectorImpl<SDNode * > & DeadNodes)1005 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1006
1007 // Process the worklist, deleting the nodes and adding their uses to the
1008 // worklist.
1009 while (!DeadNodes.empty()) {
1010 SDNode *N = DeadNodes.pop_back_val();
1011 // Skip to next node if we've already managed to delete the node. This could
1012 // happen if replacing a node causes a node previously added to the node to
1013 // be deleted.
1014 if (N->getOpcode() == ISD::DELETED_NODE)
1015 continue;
1016
1017 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1018 DUL->NodeDeleted(N, nullptr);
1019
1020 // Take the node out of the appropriate CSE map.
1021 RemoveNodeFromCSEMaps(N);
1022
1023 // Next, brutally remove the operand list. This is safe to do, as there are
1024 // no cycles in the graph.
1025 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1026 SDUse &Use = *I++;
1027 SDNode *Operand = Use.getNode();
1028 Use.set(SDValue());
1029
1030 // Now that we removed this operand, see if there are no uses of it left.
1031 if (Operand->use_empty())
1032 DeadNodes.push_back(Operand);
1033 }
1034
1035 DeallocateNode(N);
1036 }
1037 }
1038
RemoveDeadNode(SDNode * N)1039 void SelectionDAG::RemoveDeadNode(SDNode *N){
1040 SmallVector<SDNode*, 16> DeadNodes(1, N);
1041
1042 // Create a dummy node that adds a reference to the root node, preventing
1043 // it from being deleted. (This matters if the root is an operand of the
1044 // dead node.)
1045 HandleSDNode Dummy(getRoot());
1046
1047 RemoveDeadNodes(DeadNodes);
1048 }
1049
DeleteNode(SDNode * N)1050 void SelectionDAG::DeleteNode(SDNode *N) {
1051 // First take this out of the appropriate CSE map.
1052 RemoveNodeFromCSEMaps(N);
1053
1054 // Finally, remove uses due to operands of this node, remove from the
1055 // AllNodes list, and delete the node.
1056 DeleteNodeNotInCSEMaps(N);
1057 }
1058
DeleteNodeNotInCSEMaps(SDNode * N)1059 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1060 assert(N->getIterator() != AllNodes.begin() &&
1061 "Cannot delete the entry node!");
1062 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1063
1064 // Drop all of the operands and decrement used node's use counts.
1065 N->DropOperands();
1066
1067 DeallocateNode(N);
1068 }
1069
add(SDDbgValue * V,bool isParameter)1070 void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1071 assert(!(V->isVariadic() && isParameter));
1072 if (isParameter)
1073 ByvalParmDbgValues.push_back(V);
1074 else
1075 DbgValues.push_back(V);
1076 for (const SDNode *Node : V->getSDNodes())
1077 if (Node)
1078 DbgValMap[Node].push_back(V);
1079 }
1080
erase(const SDNode * Node)1081 void SDDbgInfo::erase(const SDNode *Node) {
1082 DbgValMapType::iterator I = DbgValMap.find(Node);
1083 if (I == DbgValMap.end())
1084 return;
1085 for (auto &Val: I->second)
1086 Val->setIsInvalidated();
1087 DbgValMap.erase(I);
1088 }
1089
DeallocateNode(SDNode * N)1090 void SelectionDAG::DeallocateNode(SDNode *N) {
1091 // If we have operands, deallocate them.
1092 removeOperands(N);
1093
1094 NodeAllocator.Deallocate(AllNodes.remove(N));
1095
1096 // Set the opcode to DELETED_NODE to help catch bugs when node
1097 // memory is reallocated.
1098 // FIXME: There are places in SDag that have grown a dependency on the opcode
1099 // value in the released node.
1100 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1101 N->NodeType = ISD::DELETED_NODE;
1102
1103 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1104 // them and forget about that node.
1105 DbgInfo->erase(N);
1106
1107 // Invalidate extra info.
1108 SDEI.erase(N);
1109 }
1110
1111 #ifndef NDEBUG
1112 /// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
VerifySDNode(SDNode * N)1113 static void VerifySDNode(SDNode *N) {
1114 switch (N->getOpcode()) {
1115 default:
1116 break;
1117 case ISD::BUILD_PAIR: {
1118 EVT VT = N->getValueType(0);
1119 assert(N->getNumValues() == 1 && "Too many results!");
1120 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1121 "Wrong return type!");
1122 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1123 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1124 "Mismatched operand types!");
1125 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1126 "Wrong operand type!");
1127 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1128 "Wrong return type size");
1129 break;
1130 }
1131 case ISD::BUILD_VECTOR: {
1132 assert(N->getNumValues() == 1 && "Too many results!");
1133 assert(N->getValueType(0).isVector() && "Wrong return type!");
1134 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1135 "Wrong number of operands!");
1136 EVT EltVT = N->getValueType(0).getVectorElementType();
1137 for (const SDUse &Op : N->ops()) {
1138 assert((Op.getValueType() == EltVT ||
1139 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1140 EltVT.bitsLE(Op.getValueType()))) &&
1141 "Wrong operand type!");
1142 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1143 "Operands must all have the same type");
1144 }
1145 break;
1146 }
1147 }
1148 }
1149 #endif // NDEBUG
1150
1151 /// Insert a newly allocated node into the DAG.
1152 ///
1153 /// Handles insertion into the all nodes list and CSE map, as well as
1154 /// verification and other common operations when a new node is allocated.
InsertNode(SDNode * N)1155 void SelectionDAG::InsertNode(SDNode *N) {
1156 AllNodes.push_back(N);
1157 #ifndef NDEBUG
1158 N->PersistentId = NextPersistentId++;
1159 VerifySDNode(N);
1160 #endif
1161 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1162 DUL->NodeInserted(N);
1163 }
1164
1165 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1166 /// correspond to it. This is useful when we're about to delete or repurpose
1167 /// the node. We don't want future request for structurally identical nodes
1168 /// to return N anymore.
RemoveNodeFromCSEMaps(SDNode * N)1169 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1170 bool Erased = false;
1171 switch (N->getOpcode()) {
1172 case ISD::HANDLENODE: return false; // noop.
1173 case ISD::CONDCODE:
1174 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1175 "Cond code doesn't exist!");
1176 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
1177 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
1178 break;
1179 case ISD::ExternalSymbol:
1180 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
1181 break;
1182 case ISD::TargetExternalSymbol: {
1183 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
1184 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1185 ESN->getSymbol(), ESN->getTargetFlags()));
1186 break;
1187 }
1188 case ISD::MCSymbol: {
1189 auto *MCSN = cast<MCSymbolSDNode>(N);
1190 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1191 break;
1192 }
1193 case ISD::VALUETYPE: {
1194 EVT VT = cast<VTSDNode>(N)->getVT();
1195 if (VT.isExtended()) {
1196 Erased = ExtendedValueTypeNodes.erase(VT);
1197 } else {
1198 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1199 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1200 }
1201 break;
1202 }
1203 default:
1204 // Remove it from the CSE Map.
1205 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1206 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1207 Erased = CSEMap.RemoveNode(N);
1208 break;
1209 }
1210 #ifndef NDEBUG
1211 // Verify that the node was actually in one of the CSE maps, unless it has a
1212 // glue result (which cannot be CSE'd) or is one of the special cases that are
1213 // not subject to CSE.
1214 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1215 !N->isMachineOpcode() && !doNotCSE(N)) {
1216 N->dump(this);
1217 dbgs() << "\n";
1218 llvm_unreachable("Node is not in map!");
1219 }
1220 #endif
1221 return Erased;
1222 }
1223
1224 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1225 /// maps and modified in place. Add it back to the CSE maps, unless an identical
1226 /// node already exists, in which case transfer all its users to the existing
1227 /// node. This transfer can potentially trigger recursive merging.
1228 void
AddModifiedNodeToCSEMaps(SDNode * N)1229 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1230 // For node types that aren't CSE'd, just act as if no identical node
1231 // already exists.
1232 if (!doNotCSE(N)) {
1233 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1234 if (Existing != N) {
1235 // If there was already an existing matching node, use ReplaceAllUsesWith
1236 // to replace the dead one with the existing one. This can cause
1237 // recursive merging of other unrelated nodes down the line.
1238 ReplaceAllUsesWith(N, Existing);
1239
1240 // N is now dead. Inform the listeners and delete it.
1241 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1242 DUL->NodeDeleted(N, Existing);
1243 DeleteNodeNotInCSEMaps(N);
1244 return;
1245 }
1246 }
1247
1248 // If the node doesn't already exist, we updated it. Inform listeners.
1249 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1250 DUL->NodeUpdated(N);
1251 }
1252
1253 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1254 /// were replaced with those specified. If this node is never memoized,
1255 /// return null, otherwise return a pointer to the slot it would take. If a
1256 /// node already exists with these operands, the slot will be non-null.
FindModifiedNodeSlot(SDNode * N,SDValue Op,void * & InsertPos)1257 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1258 void *&InsertPos) {
1259 if (doNotCSE(N))
1260 return nullptr;
1261
1262 SDValue Ops[] = { Op };
1263 FoldingSetNodeID ID;
1264 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1265 AddNodeIDCustom(ID, N);
1266 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1267 if (Node)
1268 Node->intersectFlagsWith(N->getFlags());
1269 return Node;
1270 }
1271
1272 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1273 /// were replaced with those specified. If this node is never memoized,
1274 /// return null, otherwise return a pointer to the slot it would take. If a
1275 /// node already exists with these operands, the slot will be non-null.
FindModifiedNodeSlot(SDNode * N,SDValue Op1,SDValue Op2,void * & InsertPos)1276 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1277 SDValue Op1, SDValue Op2,
1278 void *&InsertPos) {
1279 if (doNotCSE(N))
1280 return nullptr;
1281
1282 SDValue Ops[] = { Op1, Op2 };
1283 FoldingSetNodeID ID;
1284 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1285 AddNodeIDCustom(ID, N);
1286 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1287 if (Node)
1288 Node->intersectFlagsWith(N->getFlags());
1289 return Node;
1290 }
1291
1292 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1293 /// were replaced with those specified. If this node is never memoized,
1294 /// return null, otherwise return a pointer to the slot it would take. If a
1295 /// node already exists with these operands, the slot will be non-null.
FindModifiedNodeSlot(SDNode * N,ArrayRef<SDValue> Ops,void * & InsertPos)1296 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1297 void *&InsertPos) {
1298 if (doNotCSE(N))
1299 return nullptr;
1300
1301 FoldingSetNodeID ID;
1302 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1303 AddNodeIDCustom(ID, N);
1304 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1305 if (Node)
1306 Node->intersectFlagsWith(N->getFlags());
1307 return Node;
1308 }
1309
getEVTAlign(EVT VT) const1310 Align SelectionDAG::getEVTAlign(EVT VT) const {
1311 Type *Ty = VT == MVT::iPTR ? PointerType::get(*getContext(), 0)
1312 : VT.getTypeForEVT(*getContext());
1313
1314 return getDataLayout().getABITypeAlign(Ty);
1315 }
1316
1317 // EntryNode could meaningfully have debug info if we can find it...
SelectionDAG(const TargetMachine & tm,CodeGenOptLevel OL)1318 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1319 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1320 getVTList(MVT::Other, MVT::Glue)),
1321 Root(getEntryNode()) {
1322 InsertNode(&EntryNode);
1323 DbgInfo = new SDDbgInfo();
1324 }
1325
init(MachineFunction & NewMF,OptimizationRemarkEmitter & NewORE,Pass * PassPtr,const TargetLibraryInfo * LibraryInfo,UniformityInfo * NewUA,ProfileSummaryInfo * PSIin,BlockFrequencyInfo * BFIin,FunctionVarLocs const * VarLocs)1326 void SelectionDAG::init(MachineFunction &NewMF,
1327 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1328 const TargetLibraryInfo *LibraryInfo,
1329 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1330 BlockFrequencyInfo *BFIin,
1331 FunctionVarLocs const *VarLocs) {
1332 MF = &NewMF;
1333 SDAGISelPass = PassPtr;
1334 ORE = &NewORE;
1335 TLI = getSubtarget().getTargetLowering();
1336 TSI = getSubtarget().getSelectionDAGInfo();
1337 LibInfo = LibraryInfo;
1338 Context = &MF->getFunction().getContext();
1339 UA = NewUA;
1340 PSI = PSIin;
1341 BFI = BFIin;
1342 FnVarLocs = VarLocs;
1343 }
1344
~SelectionDAG()1345 SelectionDAG::~SelectionDAG() {
1346 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1347 allnodes_clear();
1348 OperandRecycler.clear(OperandAllocator);
1349 delete DbgInfo;
1350 }
1351
shouldOptForSize() const1352 bool SelectionDAG::shouldOptForSize() const {
1353 return MF->getFunction().hasOptSize() ||
1354 llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
1355 }
1356
allnodes_clear()1357 void SelectionDAG::allnodes_clear() {
1358 assert(&*AllNodes.begin() == &EntryNode);
1359 AllNodes.remove(AllNodes.begin());
1360 while (!AllNodes.empty())
1361 DeallocateNode(&AllNodes.front());
1362 #ifndef NDEBUG
1363 NextPersistentId = 0;
1364 #endif
1365 }
1366
FindNodeOrInsertPos(const FoldingSetNodeID & ID,void * & InsertPos)1367 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1368 void *&InsertPos) {
1369 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1370 if (N) {
1371 switch (N->getOpcode()) {
1372 default: break;
1373 case ISD::Constant:
1374 case ISD::ConstantFP:
1375 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1376 "debug location. Use another overload.");
1377 }
1378 }
1379 return N;
1380 }
1381
FindNodeOrInsertPos(const FoldingSetNodeID & ID,const SDLoc & DL,void * & InsertPos)1382 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1383 const SDLoc &DL, void *&InsertPos) {
1384 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1385 if (N) {
1386 switch (N->getOpcode()) {
1387 case ISD::Constant:
1388 case ISD::ConstantFP:
1389 // Erase debug location from the node if the node is used at several
1390 // different places. Do not propagate one location to all uses as it
1391 // will cause a worse single stepping debugging experience.
1392 if (N->getDebugLoc() != DL.getDebugLoc())
1393 N->setDebugLoc(DebugLoc());
1394 break;
1395 default:
1396 // When the node's point of use is located earlier in the instruction
1397 // sequence than its prior point of use, update its debug info to the
1398 // earlier location.
1399 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1400 N->setDebugLoc(DL.getDebugLoc());
1401 break;
1402 }
1403 }
1404 return N;
1405 }
1406
clear()1407 void SelectionDAG::clear() {
1408 allnodes_clear();
1409 OperandRecycler.clear(OperandAllocator);
1410 OperandAllocator.Reset();
1411 CSEMap.clear();
1412
1413 ExtendedValueTypeNodes.clear();
1414 ExternalSymbols.clear();
1415 TargetExternalSymbols.clear();
1416 MCSymbols.clear();
1417 SDEI.clear();
1418 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1419 static_cast<CondCodeSDNode*>(nullptr));
1420 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1421 static_cast<SDNode*>(nullptr));
1422
1423 EntryNode.UseList = nullptr;
1424 InsertNode(&EntryNode);
1425 Root = getEntryNode();
1426 DbgInfo->clear();
1427 }
1428
getFPExtendOrRound(SDValue Op,const SDLoc & DL,EVT VT)1429 SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1430 return VT.bitsGT(Op.getValueType())
1431 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1432 : getNode(ISD::FP_ROUND, DL, VT, Op,
1433 getIntPtrConstant(0, DL, /*isTarget=*/true));
1434 }
1435
1436 std::pair<SDValue, SDValue>
getStrictFPExtendOrRound(SDValue Op,SDValue Chain,const SDLoc & DL,EVT VT)1437 SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1438 const SDLoc &DL, EVT VT) {
1439 assert(!VT.bitsEq(Op.getValueType()) &&
1440 "Strict no-op FP extend/round not allowed.");
1441 SDValue Res =
1442 VT.bitsGT(Op.getValueType())
1443 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1444 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1445 {Chain, Op, getIntPtrConstant(0, DL)});
1446
1447 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1448 }
1449
getAnyExtOrTrunc(SDValue Op,const SDLoc & DL,EVT VT)1450 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1451 return VT.bitsGT(Op.getValueType()) ?
1452 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1453 getNode(ISD::TRUNCATE, DL, VT, Op);
1454 }
1455
getSExtOrTrunc(SDValue Op,const SDLoc & DL,EVT VT)1456 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1457 return VT.bitsGT(Op.getValueType()) ?
1458 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1459 getNode(ISD::TRUNCATE, DL, VT, Op);
1460 }
1461
getZExtOrTrunc(SDValue Op,const SDLoc & DL,EVT VT)1462 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1463 return VT.bitsGT(Op.getValueType()) ?
1464 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1465 getNode(ISD::TRUNCATE, DL, VT, Op);
1466 }
1467
getBitcastedAnyExtOrTrunc(SDValue Op,const SDLoc & DL,EVT VT)1468 SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1469 EVT VT) {
1470 assert(!VT.isVector());
1471 auto Type = Op.getValueType();
1472 SDValue DestOp;
1473 if (Type == VT)
1474 return Op;
1475 auto Size = Op.getValueSizeInBits();
1476 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1477 if (DestOp.getValueType() == VT)
1478 return DestOp;
1479
1480 return getAnyExtOrTrunc(DestOp, DL, VT);
1481 }
1482
getBitcastedSExtOrTrunc(SDValue Op,const SDLoc & DL,EVT VT)1483 SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1484 EVT VT) {
1485 assert(!VT.isVector());
1486 auto Type = Op.getValueType();
1487 SDValue DestOp;
1488 if (Type == VT)
1489 return Op;
1490 auto Size = Op.getValueSizeInBits();
1491 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1492 if (DestOp.getValueType() == VT)
1493 return DestOp;
1494
1495 return getSExtOrTrunc(DestOp, DL, VT);
1496 }
1497
getBitcastedZExtOrTrunc(SDValue Op,const SDLoc & DL,EVT VT)1498 SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1499 EVT VT) {
1500 assert(!VT.isVector());
1501 auto Type = Op.getValueType();
1502 SDValue DestOp;
1503 if (Type == VT)
1504 return Op;
1505 auto Size = Op.getValueSizeInBits();
1506 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1507 if (DestOp.getValueType() == VT)
1508 return DestOp;
1509
1510 return getZExtOrTrunc(DestOp, DL, VT);
1511 }
1512
getBoolExtOrTrunc(SDValue Op,const SDLoc & SL,EVT VT,EVT OpVT)1513 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1514 EVT OpVT) {
1515 if (VT.bitsLE(Op.getValueType()))
1516 return getNode(ISD::TRUNCATE, SL, VT, Op);
1517
1518 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1519 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1520 }
1521
getZeroExtendInReg(SDValue Op,const SDLoc & DL,EVT VT)1522 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1523 EVT OpVT = Op.getValueType();
1524 assert(VT.isInteger() && OpVT.isInteger() &&
1525 "Cannot getZeroExtendInReg FP types");
1526 assert(VT.isVector() == OpVT.isVector() &&
1527 "getZeroExtendInReg type should be vector iff the operand "
1528 "type is vector!");
1529 assert((!VT.isVector() ||
1530 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1531 "Vector element counts must match in getZeroExtendInReg");
1532 assert(VT.bitsLE(OpVT) && "Not extending!");
1533 if (OpVT == VT)
1534 return Op;
1535 APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(),
1536 VT.getScalarSizeInBits());
1537 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1538 }
1539
getPtrExtOrTrunc(SDValue Op,const SDLoc & DL,EVT VT)1540 SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1541 // Only unsigned pointer semantics are supported right now. In the future this
1542 // might delegate to TLI to check pointer signedness.
1543 return getZExtOrTrunc(Op, DL, VT);
1544 }
1545
getPtrExtendInReg(SDValue Op,const SDLoc & DL,EVT VT)1546 SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1547 // Only unsigned pointer semantics are supported right now. In the future this
1548 // might delegate to TLI to check pointer signedness.
1549 return getZeroExtendInReg(Op, DL, VT);
1550 }
1551
getNegative(SDValue Val,const SDLoc & DL,EVT VT)1552 SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1553 return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val);
1554 }
1555
1556 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
getNOT(const SDLoc & DL,SDValue Val,EVT VT)1557 SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1558 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT));
1559 }
1560
getLogicalNOT(const SDLoc & DL,SDValue Val,EVT VT)1561 SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1562 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1563 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1564 }
1565
getVPLogicalNOT(const SDLoc & DL,SDValue Val,SDValue Mask,SDValue EVL,EVT VT)1566 SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1567 SDValue Mask, SDValue EVL, EVT VT) {
1568 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1569 return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL);
1570 }
1571
getVPPtrExtOrTrunc(const SDLoc & DL,EVT VT,SDValue Op,SDValue Mask,SDValue EVL)1572 SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1573 SDValue Mask, SDValue EVL) {
1574 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1575 }
1576
getVPZExtOrTrunc(const SDLoc & DL,EVT VT,SDValue Op,SDValue Mask,SDValue EVL)1577 SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1578 SDValue Mask, SDValue EVL) {
1579 if (VT.bitsGT(Op.getValueType()))
1580 return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL);
1581 if (VT.bitsLT(Op.getValueType()))
1582 return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL);
1583 return Op;
1584 }
1585
getBoolConstant(bool V,const SDLoc & DL,EVT VT,EVT OpVT)1586 SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1587 EVT OpVT) {
1588 if (!V)
1589 return getConstant(0, DL, VT);
1590
1591 switch (TLI->getBooleanContents(OpVT)) {
1592 case TargetLowering::ZeroOrOneBooleanContent:
1593 case TargetLowering::UndefinedBooleanContent:
1594 return getConstant(1, DL, VT);
1595 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1596 return getAllOnesConstant(DL, VT);
1597 }
1598 llvm_unreachable("Unexpected boolean content enum!");
1599 }
1600
getConstant(uint64_t Val,const SDLoc & DL,EVT VT,bool isT,bool isO)1601 SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1602 bool isT, bool isO) {
1603 EVT EltVT = VT.getScalarType();
1604 assert((EltVT.getSizeInBits() >= 64 ||
1605 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1606 "getConstant with a uint64_t value that doesn't fit in the type!");
1607 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1608 }
1609
getConstant(const APInt & Val,const SDLoc & DL,EVT VT,bool isT,bool isO)1610 SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1611 bool isT, bool isO) {
1612 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1613 }
1614
getConstant(const ConstantInt & Val,const SDLoc & DL,EVT VT,bool isT,bool isO)1615 SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1616 EVT VT, bool isT, bool isO) {
1617 assert(VT.isInteger() && "Cannot create FP integer constant!");
1618
1619 EVT EltVT = VT.getScalarType();
1620 const ConstantInt *Elt = &Val;
1621
1622 // In some cases the vector type is legal but the element type is illegal and
1623 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1624 // inserted value (the type does not need to match the vector element type).
1625 // Any extra bits introduced will be truncated away.
1626 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1627 TargetLowering::TypePromoteInteger) {
1628 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1629 APInt NewVal;
1630 if (TLI->isSExtCheaperThanZExt(VT.getScalarType(), EltVT))
1631 NewVal = Elt->getValue().sextOrTrunc(EltVT.getSizeInBits());
1632 else
1633 NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1634 Elt = ConstantInt::get(*getContext(), NewVal);
1635 }
1636 // In other cases the element type is illegal and needs to be expanded, for
1637 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1638 // the value into n parts and use a vector type with n-times the elements.
1639 // Then bitcast to the type requested.
1640 // Legalizing constants too early makes the DAGCombiner's job harder so we
1641 // only legalize if the DAG tells us we must produce legal types.
1642 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1643 TLI->getTypeAction(*getContext(), EltVT) ==
1644 TargetLowering::TypeExpandInteger) {
1645 const APInt &NewVal = Elt->getValue();
1646 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1647 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1648
1649 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1650 if (VT.isScalableVector() ||
1651 TLI->isOperationLegal(ISD::SPLAT_VECTOR, VT)) {
1652 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1653 "Can only handle an even split!");
1654 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1655
1656 SmallVector<SDValue, 2> ScalarParts;
1657 for (unsigned i = 0; i != Parts; ++i)
1658 ScalarParts.push_back(getConstant(
1659 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1660 ViaEltVT, isT, isO));
1661
1662 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1663 }
1664
1665 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1666 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1667
1668 // Check the temporary vector is the correct size. If this fails then
1669 // getTypeToTransformTo() probably returned a type whose size (in bits)
1670 // isn't a power-of-2 factor of the requested type size.
1671 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1672
1673 SmallVector<SDValue, 2> EltParts;
1674 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1675 EltParts.push_back(getConstant(
1676 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1677 ViaEltVT, isT, isO));
1678
1679 // EltParts is currently in little endian order. If we actually want
1680 // big-endian order then reverse it now.
1681 if (getDataLayout().isBigEndian())
1682 std::reverse(EltParts.begin(), EltParts.end());
1683
1684 // The elements must be reversed when the element order is different
1685 // to the endianness of the elements (because the BITCAST is itself a
1686 // vector shuffle in this situation). However, we do not need any code to
1687 // perform this reversal because getConstant() is producing a vector
1688 // splat.
1689 // This situation occurs in MIPS MSA.
1690
1691 SmallVector<SDValue, 8> Ops;
1692 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1693 llvm::append_range(Ops, EltParts);
1694
1695 SDValue V =
1696 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1697 return V;
1698 }
1699
1700 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1701 "APInt size does not match type size!");
1702 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1703 FoldingSetNodeID ID;
1704 AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt);
1705 ID.AddPointer(Elt);
1706 ID.AddBoolean(isO);
1707 void *IP = nullptr;
1708 SDNode *N = nullptr;
1709 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1710 if (!VT.isVector())
1711 return SDValue(N, 0);
1712
1713 if (!N) {
1714 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1715 CSEMap.InsertNode(N, IP);
1716 InsertNode(N);
1717 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1718 }
1719
1720 SDValue Result(N, 0);
1721 if (VT.isVector())
1722 Result = getSplat(VT, DL, Result);
1723 return Result;
1724 }
1725
getIntPtrConstant(uint64_t Val,const SDLoc & DL,bool isTarget)1726 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1727 bool isTarget) {
1728 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1729 }
1730
getShiftAmountConstant(uint64_t Val,EVT VT,const SDLoc & DL,bool LegalTypes)1731 SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1732 const SDLoc &DL, bool LegalTypes) {
1733 assert(VT.isInteger() && "Shift amount is not an integer type!");
1734 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1735 return getConstant(Val, DL, ShiftVT);
1736 }
1737
getVectorIdxConstant(uint64_t Val,const SDLoc & DL,bool isTarget)1738 SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1739 bool isTarget) {
1740 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1741 }
1742
getConstantFP(const APFloat & V,const SDLoc & DL,EVT VT,bool isTarget)1743 SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1744 bool isTarget) {
1745 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1746 }
1747
getConstantFP(const ConstantFP & V,const SDLoc & DL,EVT VT,bool isTarget)1748 SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1749 EVT VT, bool isTarget) {
1750 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1751
1752 EVT EltVT = VT.getScalarType();
1753
1754 // Do the map lookup using the actual bit pattern for the floating point
1755 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1756 // we don't have issues with SNANs.
1757 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1758 FoldingSetNodeID ID;
1759 AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt);
1760 ID.AddPointer(&V);
1761 void *IP = nullptr;
1762 SDNode *N = nullptr;
1763 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1764 if (!VT.isVector())
1765 return SDValue(N, 0);
1766
1767 if (!N) {
1768 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1769 CSEMap.InsertNode(N, IP);
1770 InsertNode(N);
1771 }
1772
1773 SDValue Result(N, 0);
1774 if (VT.isVector())
1775 Result = getSplat(VT, DL, Result);
1776 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1777 return Result;
1778 }
1779
getConstantFP(double Val,const SDLoc & DL,EVT VT,bool isTarget)1780 SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1781 bool isTarget) {
1782 EVT EltVT = VT.getScalarType();
1783 if (EltVT == MVT::f32)
1784 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1785 if (EltVT == MVT::f64)
1786 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1787 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1788 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1789 bool Ignored;
1790 APFloat APF = APFloat(Val);
1791 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1792 &Ignored);
1793 return getConstantFP(APF, DL, VT, isTarget);
1794 }
1795 llvm_unreachable("Unsupported type in getConstantFP");
1796 }
1797
getGlobalAddress(const GlobalValue * GV,const SDLoc & DL,EVT VT,int64_t Offset,bool isTargetGA,unsigned TargetFlags)1798 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1799 EVT VT, int64_t Offset, bool isTargetGA,
1800 unsigned TargetFlags) {
1801 assert((TargetFlags == 0 || isTargetGA) &&
1802 "Cannot set target flags on target-independent globals");
1803
1804 // Truncate (with sign-extension) the offset value to the pointer size.
1805 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1806 if (BitWidth < 64)
1807 Offset = SignExtend64(Offset, BitWidth);
1808
1809 unsigned Opc;
1810 if (GV->isThreadLocal())
1811 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1812 else
1813 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1814
1815 FoldingSetNodeID ID;
1816 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1817 ID.AddPointer(GV);
1818 ID.AddInteger(Offset);
1819 ID.AddInteger(TargetFlags);
1820 void *IP = nullptr;
1821 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1822 return SDValue(E, 0);
1823
1824 auto *N = newSDNode<GlobalAddressSDNode>(
1825 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1826 CSEMap.InsertNode(N, IP);
1827 InsertNode(N);
1828 return SDValue(N, 0);
1829 }
1830
getFrameIndex(int FI,EVT VT,bool isTarget)1831 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1832 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1833 FoldingSetNodeID ID;
1834 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1835 ID.AddInteger(FI);
1836 void *IP = nullptr;
1837 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1838 return SDValue(E, 0);
1839
1840 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1841 CSEMap.InsertNode(N, IP);
1842 InsertNode(N);
1843 return SDValue(N, 0);
1844 }
1845
getJumpTable(int JTI,EVT VT,bool isTarget,unsigned TargetFlags)1846 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1847 unsigned TargetFlags) {
1848 assert((TargetFlags == 0 || isTarget) &&
1849 "Cannot set target flags on target-independent jump tables");
1850 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1851 FoldingSetNodeID ID;
1852 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1853 ID.AddInteger(JTI);
1854 ID.AddInteger(TargetFlags);
1855 void *IP = nullptr;
1856 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1857 return SDValue(E, 0);
1858
1859 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1860 CSEMap.InsertNode(N, IP);
1861 InsertNode(N);
1862 return SDValue(N, 0);
1863 }
1864
getJumpTableDebugInfo(int JTI,SDValue Chain,const SDLoc & DL)1865 SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1866 const SDLoc &DL) {
1867 EVT PTy = getTargetLoweringInfo().getPointerTy(getDataLayout());
1868 return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain,
1869 getTargetConstant(static_cast<uint64_t>(JTI), DL, PTy, true));
1870 }
1871
getConstantPool(const Constant * C,EVT VT,MaybeAlign Alignment,int Offset,bool isTarget,unsigned TargetFlags)1872 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1873 MaybeAlign Alignment, int Offset,
1874 bool isTarget, unsigned TargetFlags) {
1875 assert((TargetFlags == 0 || isTarget) &&
1876 "Cannot set target flags on target-independent globals");
1877 if (!Alignment)
1878 Alignment = shouldOptForSize()
1879 ? getDataLayout().getABITypeAlign(C->getType())
1880 : getDataLayout().getPrefTypeAlign(C->getType());
1881 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1882 FoldingSetNodeID ID;
1883 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1884 ID.AddInteger(Alignment->value());
1885 ID.AddInteger(Offset);
1886 ID.AddPointer(C);
1887 ID.AddInteger(TargetFlags);
1888 void *IP = nullptr;
1889 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1890 return SDValue(E, 0);
1891
1892 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1893 TargetFlags);
1894 CSEMap.InsertNode(N, IP);
1895 InsertNode(N);
1896 SDValue V = SDValue(N, 0);
1897 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1898 return V;
1899 }
1900
getConstantPool(MachineConstantPoolValue * C,EVT VT,MaybeAlign Alignment,int Offset,bool isTarget,unsigned TargetFlags)1901 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1902 MaybeAlign Alignment, int Offset,
1903 bool isTarget, unsigned TargetFlags) {
1904 assert((TargetFlags == 0 || isTarget) &&
1905 "Cannot set target flags on target-independent globals");
1906 if (!Alignment)
1907 Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1908 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1909 FoldingSetNodeID ID;
1910 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1911 ID.AddInteger(Alignment->value());
1912 ID.AddInteger(Offset);
1913 C->addSelectionDAGCSEId(ID);
1914 ID.AddInteger(TargetFlags);
1915 void *IP = nullptr;
1916 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1917 return SDValue(E, 0);
1918
1919 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1920 TargetFlags);
1921 CSEMap.InsertNode(N, IP);
1922 InsertNode(N);
1923 return SDValue(N, 0);
1924 }
1925
getBasicBlock(MachineBasicBlock * MBB)1926 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1927 FoldingSetNodeID ID;
1928 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt);
1929 ID.AddPointer(MBB);
1930 void *IP = nullptr;
1931 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1932 return SDValue(E, 0);
1933
1934 auto *N = newSDNode<BasicBlockSDNode>(MBB);
1935 CSEMap.InsertNode(N, IP);
1936 InsertNode(N);
1937 return SDValue(N, 0);
1938 }
1939
getValueType(EVT VT)1940 SDValue SelectionDAG::getValueType(EVT VT) {
1941 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1942 ValueTypeNodes.size())
1943 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1944
1945 SDNode *&N = VT.isExtended() ?
1946 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1947
1948 if (N) return SDValue(N, 0);
1949 N = newSDNode<VTSDNode>(VT);
1950 InsertNode(N);
1951 return SDValue(N, 0);
1952 }
1953
getExternalSymbol(const char * Sym,EVT VT)1954 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1955 SDNode *&N = ExternalSymbols[Sym];
1956 if (N) return SDValue(N, 0);
1957 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1958 InsertNode(N);
1959 return SDValue(N, 0);
1960 }
1961
getMCSymbol(MCSymbol * Sym,EVT VT)1962 SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1963 SDNode *&N = MCSymbols[Sym];
1964 if (N)
1965 return SDValue(N, 0);
1966 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1967 InsertNode(N);
1968 return SDValue(N, 0);
1969 }
1970
getTargetExternalSymbol(const char * Sym,EVT VT,unsigned TargetFlags)1971 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1972 unsigned TargetFlags) {
1973 SDNode *&N =
1974 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1975 if (N) return SDValue(N, 0);
1976 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1977 InsertNode(N);
1978 return SDValue(N, 0);
1979 }
1980
getCondCode(ISD::CondCode Cond)1981 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1982 if ((unsigned)Cond >= CondCodeNodes.size())
1983 CondCodeNodes.resize(Cond+1);
1984
1985 if (!CondCodeNodes[Cond]) {
1986 auto *N = newSDNode<CondCodeSDNode>(Cond);
1987 CondCodeNodes[Cond] = N;
1988 InsertNode(N);
1989 }
1990
1991 return SDValue(CondCodeNodes[Cond], 0);
1992 }
1993
getVScale(const SDLoc & DL,EVT VT,APInt MulImm,bool ConstantFold)1994 SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm,
1995 bool ConstantFold) {
1996 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
1997 "APInt size does not match type size!");
1998
1999 if (MulImm == 0)
2000 return getConstant(0, DL, VT);
2001
2002 if (ConstantFold) {
2003 const MachineFunction &MF = getMachineFunction();
2004 const Function &F = MF.getFunction();
2005 ConstantRange CR = getVScaleRange(&F, 64);
2006 if (const APInt *C = CR.getSingleElement())
2007 return getConstant(MulImm * C->getZExtValue(), DL, VT);
2008 }
2009
2010 return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
2011 }
2012
getElementCount(const SDLoc & DL,EVT VT,ElementCount EC,bool ConstantFold)2013 SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC,
2014 bool ConstantFold) {
2015 if (EC.isScalable())
2016 return getVScale(DL, VT,
2017 APInt(VT.getSizeInBits(), EC.getKnownMinValue()));
2018
2019 return getConstant(EC.getKnownMinValue(), DL, VT);
2020 }
2021
getStepVector(const SDLoc & DL,EVT ResVT)2022 SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2023 APInt One(ResVT.getScalarSizeInBits(), 1);
2024 return getStepVector(DL, ResVT, One);
2025 }
2026
getStepVector(const SDLoc & DL,EVT ResVT,APInt StepVal)2027 SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
2028 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2029 if (ResVT.isScalableVector())
2030 return getNode(
2031 ISD::STEP_VECTOR, DL, ResVT,
2032 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
2033
2034 SmallVector<SDValue, 16> OpsStepConstants;
2035 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2036 OpsStepConstants.push_back(
2037 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
2038 return getBuildVector(ResVT, DL, OpsStepConstants);
2039 }
2040
2041 /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2042 /// point at N1 to point at N2 and indices that point at N2 to point at N1.
commuteShuffle(SDValue & N1,SDValue & N2,MutableArrayRef<int> M)2043 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2044 std::swap(N1, N2);
2045 ShuffleVectorSDNode::commuteMask(M);
2046 }
2047
getVectorShuffle(EVT VT,const SDLoc & dl,SDValue N1,SDValue N2,ArrayRef<int> Mask)2048 SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2049 SDValue N2, ArrayRef<int> Mask) {
2050 assert(VT.getVectorNumElements() == Mask.size() &&
2051 "Must have the same number of vector elements as mask elements!");
2052 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2053 "Invalid VECTOR_SHUFFLE");
2054
2055 // Canonicalize shuffle undef, undef -> undef
2056 if (N1.isUndef() && N2.isUndef())
2057 return getUNDEF(VT);
2058
2059 // Validate that all indices in Mask are within the range of the elements
2060 // input to the shuffle.
2061 int NElts = Mask.size();
2062 assert(llvm::all_of(Mask,
2063 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2064 "Index out of range");
2065
2066 // Copy the mask so we can do any needed cleanup.
2067 SmallVector<int, 8> MaskVec(Mask);
2068
2069 // Canonicalize shuffle v, v -> v, undef
2070 if (N1 == N2) {
2071 N2 = getUNDEF(VT);
2072 for (int i = 0; i != NElts; ++i)
2073 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2074 }
2075
2076 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2077 if (N1.isUndef())
2078 commuteShuffle(N1, N2, MaskVec);
2079
2080 if (TLI->hasVectorBlend()) {
2081 // If shuffling a splat, try to blend the splat instead. We do this here so
2082 // that even when this arises during lowering we don't have to re-handle it.
2083 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2084 BitVector UndefElements;
2085 SDValue Splat = BV->getSplatValue(&UndefElements);
2086 if (!Splat)
2087 return;
2088
2089 for (int i = 0; i < NElts; ++i) {
2090 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2091 continue;
2092
2093 // If this input comes from undef, mark it as such.
2094 if (UndefElements[MaskVec[i] - Offset]) {
2095 MaskVec[i] = -1;
2096 continue;
2097 }
2098
2099 // If we can blend a non-undef lane, use that instead.
2100 if (!UndefElements[i])
2101 MaskVec[i] = i + Offset;
2102 }
2103 };
2104 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2105 BlendSplat(N1BV, 0);
2106 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2107 BlendSplat(N2BV, NElts);
2108 }
2109
2110 // Canonicalize all index into lhs, -> shuffle lhs, undef
2111 // Canonicalize all index into rhs, -> shuffle rhs, undef
2112 bool AllLHS = true, AllRHS = true;
2113 bool N2Undef = N2.isUndef();
2114 for (int i = 0; i != NElts; ++i) {
2115 if (MaskVec[i] >= NElts) {
2116 if (N2Undef)
2117 MaskVec[i] = -1;
2118 else
2119 AllLHS = false;
2120 } else if (MaskVec[i] >= 0) {
2121 AllRHS = false;
2122 }
2123 }
2124 if (AllLHS && AllRHS)
2125 return getUNDEF(VT);
2126 if (AllLHS && !N2Undef)
2127 N2 = getUNDEF(VT);
2128 if (AllRHS) {
2129 N1 = getUNDEF(VT);
2130 commuteShuffle(N1, N2, MaskVec);
2131 }
2132 // Reset our undef status after accounting for the mask.
2133 N2Undef = N2.isUndef();
2134 // Re-check whether both sides ended up undef.
2135 if (N1.isUndef() && N2Undef)
2136 return getUNDEF(VT);
2137
2138 // If Identity shuffle return that node.
2139 bool Identity = true, AllSame = true;
2140 for (int i = 0; i != NElts; ++i) {
2141 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2142 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2143 }
2144 if (Identity && NElts)
2145 return N1;
2146
2147 // Shuffling a constant splat doesn't change the result.
2148 if (N2Undef) {
2149 SDValue V = N1;
2150
2151 // Look through any bitcasts. We check that these don't change the number
2152 // (and size) of elements and just changes their types.
2153 while (V.getOpcode() == ISD::BITCAST)
2154 V = V->getOperand(0);
2155
2156 // A splat should always show up as a build vector node.
2157 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2158 BitVector UndefElements;
2159 SDValue Splat = BV->getSplatValue(&UndefElements);
2160 // If this is a splat of an undef, shuffling it is also undef.
2161 if (Splat && Splat.isUndef())
2162 return getUNDEF(VT);
2163
2164 bool SameNumElts =
2165 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2166
2167 // We only have a splat which can skip shuffles if there is a splatted
2168 // value and no undef lanes rearranged by the shuffle.
2169 if (Splat && UndefElements.none()) {
2170 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2171 // number of elements match or the value splatted is a zero constant.
2172 if (SameNumElts || isNullConstant(Splat))
2173 return N1;
2174 }
2175
2176 // If the shuffle itself creates a splat, build the vector directly.
2177 if (AllSame && SameNumElts) {
2178 EVT BuildVT = BV->getValueType(0);
2179 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
2180 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
2181
2182 // We may have jumped through bitcasts, so the type of the
2183 // BUILD_VECTOR may not match the type of the shuffle.
2184 if (BuildVT != VT)
2185 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
2186 return NewBV;
2187 }
2188 }
2189 }
2190
2191 FoldingSetNodeID ID;
2192 SDValue Ops[2] = { N1, N2 };
2193 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
2194 for (int i = 0; i != NElts; ++i)
2195 ID.AddInteger(MaskVec[i]);
2196
2197 void* IP = nullptr;
2198 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2199 return SDValue(E, 0);
2200
2201 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2202 // SDNode doesn't have access to it. This memory will be "leaked" when
2203 // the node is deallocated, but recovered when the NodeAllocator is released.
2204 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
2205 llvm::copy(MaskVec, MaskAlloc);
2206
2207 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
2208 dl.getDebugLoc(), MaskAlloc);
2209 createOperands(N, Ops);
2210
2211 CSEMap.InsertNode(N, IP);
2212 InsertNode(N);
2213 SDValue V = SDValue(N, 0);
2214 NewSDValueDbgMsg(V, "Creating new node: ", this);
2215 return V;
2216 }
2217
getCommutedVectorShuffle(const ShuffleVectorSDNode & SV)2218 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2219 EVT VT = SV.getValueType(0);
2220 SmallVector<int, 8> MaskVec(SV.getMask());
2221 ShuffleVectorSDNode::commuteMask(MaskVec);
2222
2223 SDValue Op0 = SV.getOperand(0);
2224 SDValue Op1 = SV.getOperand(1);
2225 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
2226 }
2227
getRegister(unsigned RegNo,EVT VT)2228 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
2229 FoldingSetNodeID ID;
2230 AddNodeIDNode(ID, ISD::Register, getVTList(VT), std::nullopt);
2231 ID.AddInteger(RegNo);
2232 void *IP = nullptr;
2233 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2234 return SDValue(E, 0);
2235
2236 auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
2237 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2238 CSEMap.InsertNode(N, IP);
2239 InsertNode(N);
2240 return SDValue(N, 0);
2241 }
2242
getRegisterMask(const uint32_t * RegMask)2243 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2244 FoldingSetNodeID ID;
2245 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt);
2246 ID.AddPointer(RegMask);
2247 void *IP = nullptr;
2248 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2249 return SDValue(E, 0);
2250
2251 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
2252 CSEMap.InsertNode(N, IP);
2253 InsertNode(N);
2254 return SDValue(N, 0);
2255 }
2256
getEHLabel(const SDLoc & dl,SDValue Root,MCSymbol * Label)2257 SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2258 MCSymbol *Label) {
2259 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
2260 }
2261
getLabelNode(unsigned Opcode,const SDLoc & dl,SDValue Root,MCSymbol * Label)2262 SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2263 SDValue Root, MCSymbol *Label) {
2264 FoldingSetNodeID ID;
2265 SDValue Ops[] = { Root };
2266 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
2267 ID.AddPointer(Label);
2268 void *IP = nullptr;
2269 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2270 return SDValue(E, 0);
2271
2272 auto *N =
2273 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2274 createOperands(N, Ops);
2275
2276 CSEMap.InsertNode(N, IP);
2277 InsertNode(N);
2278 return SDValue(N, 0);
2279 }
2280
getBlockAddress(const BlockAddress * BA,EVT VT,int64_t Offset,bool isTarget,unsigned TargetFlags)2281 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2282 int64_t Offset, bool isTarget,
2283 unsigned TargetFlags) {
2284 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2285
2286 FoldingSetNodeID ID;
2287 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
2288 ID.AddPointer(BA);
2289 ID.AddInteger(Offset);
2290 ID.AddInteger(TargetFlags);
2291 void *IP = nullptr;
2292 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2293 return SDValue(E, 0);
2294
2295 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
2296 CSEMap.InsertNode(N, IP);
2297 InsertNode(N);
2298 return SDValue(N, 0);
2299 }
2300
getSrcValue(const Value * V)2301 SDValue SelectionDAG::getSrcValue(const Value *V) {
2302 FoldingSetNodeID ID;
2303 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt);
2304 ID.AddPointer(V);
2305
2306 void *IP = nullptr;
2307 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2308 return SDValue(E, 0);
2309
2310 auto *N = newSDNode<SrcValueSDNode>(V);
2311 CSEMap.InsertNode(N, IP);
2312 InsertNode(N);
2313 return SDValue(N, 0);
2314 }
2315
getMDNode(const MDNode * MD)2316 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2317 FoldingSetNodeID ID;
2318 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt);
2319 ID.AddPointer(MD);
2320
2321 void *IP = nullptr;
2322 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2323 return SDValue(E, 0);
2324
2325 auto *N = newSDNode<MDNodeSDNode>(MD);
2326 CSEMap.InsertNode(N, IP);
2327 InsertNode(N);
2328 return SDValue(N, 0);
2329 }
2330
getBitcast(EVT VT,SDValue V)2331 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2332 if (VT == V.getValueType())
2333 return V;
2334
2335 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2336 }
2337
getAddrSpaceCast(const SDLoc & dl,EVT VT,SDValue Ptr,unsigned SrcAS,unsigned DestAS)2338 SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2339 unsigned SrcAS, unsigned DestAS) {
2340 SDValue Ops[] = {Ptr};
2341 FoldingSetNodeID ID;
2342 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
2343 ID.AddInteger(SrcAS);
2344 ID.AddInteger(DestAS);
2345
2346 void *IP = nullptr;
2347 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2348 return SDValue(E, 0);
2349
2350 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2351 VT, SrcAS, DestAS);
2352 createOperands(N, Ops);
2353
2354 CSEMap.InsertNode(N, IP);
2355 InsertNode(N);
2356 return SDValue(N, 0);
2357 }
2358
getFreeze(SDValue V)2359 SDValue SelectionDAG::getFreeze(SDValue V) {
2360 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2361 }
2362
2363 /// getShiftAmountOperand - Return the specified value casted to
2364 /// the target's desired shift amount type.
getShiftAmountOperand(EVT LHSTy,SDValue Op)2365 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2366 EVT OpTy = Op.getValueType();
2367 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2368 if (OpTy == ShTy || OpTy.isVector()) return Op;
2369
2370 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2371 }
2372
expandVAArg(SDNode * Node)2373 SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2374 SDLoc dl(Node);
2375 const TargetLowering &TLI = getTargetLoweringInfo();
2376 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2377 EVT VT = Node->getValueType(0);
2378 SDValue Tmp1 = Node->getOperand(0);
2379 SDValue Tmp2 = Node->getOperand(1);
2380 const MaybeAlign MA(Node->getConstantOperandVal(3));
2381
2382 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
2383 Tmp2, MachinePointerInfo(V));
2384 SDValue VAList = VAListLoad;
2385
2386 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2387 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2388 getConstant(MA->value() - 1, dl, VAList.getValueType()));
2389
2390 VAList =
2391 getNode(ISD::AND, dl, VAList.getValueType(), VAList,
2392 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2393 }
2394
2395 // Increment the pointer, VAList, to the next vaarg
2396 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2397 getConstant(getDataLayout().getTypeAllocSize(
2398 VT.getTypeForEVT(*getContext())),
2399 dl, VAList.getValueType()));
2400 // Store the incremented VAList to the legalized pointer
2401 Tmp1 =
2402 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2403 // Load the actual argument out of the pointer VAList
2404 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2405 }
2406
expandVACopy(SDNode * Node)2407 SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2408 SDLoc dl(Node);
2409 const TargetLowering &TLI = getTargetLoweringInfo();
2410 // This defaults to loading a pointer from the input and storing it to the
2411 // output, returning the chain.
2412 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2413 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2414 SDValue Tmp1 =
2415 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2416 Node->getOperand(2), MachinePointerInfo(VS));
2417 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2418 MachinePointerInfo(VD));
2419 }
2420
getReducedAlign(EVT VT,bool UseABI)2421 Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2422 const DataLayout &DL = getDataLayout();
2423 Type *Ty = VT.getTypeForEVT(*getContext());
2424 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2425
2426 if (TLI->isTypeLegal(VT) || !VT.isVector())
2427 return RedAlign;
2428
2429 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2430 const Align StackAlign = TFI->getStackAlign();
2431
2432 // See if we can choose a smaller ABI alignment in cases where it's an
2433 // illegal vector type that will get broken down.
2434 if (RedAlign > StackAlign) {
2435 EVT IntermediateVT;
2436 MVT RegisterVT;
2437 unsigned NumIntermediates;
2438 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2439 NumIntermediates, RegisterVT);
2440 Ty = IntermediateVT.getTypeForEVT(*getContext());
2441 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2442 if (RedAlign2 < RedAlign)
2443 RedAlign = RedAlign2;
2444 }
2445
2446 return RedAlign;
2447 }
2448
CreateStackTemporary(TypeSize Bytes,Align Alignment)2449 SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2450 MachineFrameInfo &MFI = MF->getFrameInfo();
2451 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2452 int StackID = 0;
2453 if (Bytes.isScalable())
2454 StackID = TFI->getStackIDForScalableVectors();
2455 // The stack id gives an indication of whether the object is scalable or
2456 // not, so it's safe to pass in the minimum size here.
2457 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment,
2458 false, nullptr, StackID);
2459 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2460 }
2461
CreateStackTemporary(EVT VT,unsigned minAlign)2462 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2463 Type *Ty = VT.getTypeForEVT(*getContext());
2464 Align StackAlign =
2465 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2466 return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2467 }
2468
CreateStackTemporary(EVT VT1,EVT VT2)2469 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2470 TypeSize VT1Size = VT1.getStoreSize();
2471 TypeSize VT2Size = VT2.getStoreSize();
2472 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2473 "Don't know how to choose the maximum size when creating a stack "
2474 "temporary");
2475 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2476 ? VT1Size
2477 : VT2Size;
2478
2479 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2480 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2481 const DataLayout &DL = getDataLayout();
2482 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2483 return CreateStackTemporary(Bytes, Align);
2484 }
2485
FoldSetCC(EVT VT,SDValue N1,SDValue N2,ISD::CondCode Cond,const SDLoc & dl)2486 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2487 ISD::CondCode Cond, const SDLoc &dl) {
2488 EVT OpVT = N1.getValueType();
2489
2490 auto GetUndefBooleanConstant = [&]() {
2491 if (VT.getScalarType() == MVT::i1 ||
2492 TLI->getBooleanContents(OpVT) ==
2493 TargetLowering::UndefinedBooleanContent)
2494 return getUNDEF(VT);
2495 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2496 // so we cannot use getUNDEF(). Return zero instead.
2497 return getConstant(0, dl, VT);
2498 };
2499
2500 // These setcc operations always fold.
2501 switch (Cond) {
2502 default: break;
2503 case ISD::SETFALSE:
2504 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2505 case ISD::SETTRUE:
2506 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2507
2508 case ISD::SETOEQ:
2509 case ISD::SETOGT:
2510 case ISD::SETOGE:
2511 case ISD::SETOLT:
2512 case ISD::SETOLE:
2513 case ISD::SETONE:
2514 case ISD::SETO:
2515 case ISD::SETUO:
2516 case ISD::SETUEQ:
2517 case ISD::SETUNE:
2518 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2519 break;
2520 }
2521
2522 if (OpVT.isInteger()) {
2523 // For EQ and NE, we can always pick a value for the undef to make the
2524 // predicate pass or fail, so we can return undef.
2525 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2526 // icmp eq/ne X, undef -> undef.
2527 if ((N1.isUndef() || N2.isUndef()) &&
2528 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2529 return GetUndefBooleanConstant();
2530
2531 // If both operands are undef, we can return undef for int comparison.
2532 // icmp undef, undef -> undef.
2533 if (N1.isUndef() && N2.isUndef())
2534 return GetUndefBooleanConstant();
2535
2536 // icmp X, X -> true/false
2537 // icmp X, undef -> true/false because undef could be X.
2538 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2539 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2540 }
2541
2542 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2543 const APInt &C2 = N2C->getAPIntValue();
2544 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2545 const APInt &C1 = N1C->getAPIntValue();
2546
2547 return getBoolConstant(ICmpInst::compare(C1, C2, getICmpCondCode(Cond)),
2548 dl, VT, OpVT);
2549 }
2550 }
2551
2552 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2553 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2554
2555 if (N1CFP && N2CFP) {
2556 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2557 switch (Cond) {
2558 default: break;
2559 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2560 return GetUndefBooleanConstant();
2561 [[fallthrough]];
2562 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2563 OpVT);
2564 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2565 return GetUndefBooleanConstant();
2566 [[fallthrough]];
2567 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2568 R==APFloat::cmpLessThan, dl, VT,
2569 OpVT);
2570 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2571 return GetUndefBooleanConstant();
2572 [[fallthrough]];
2573 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2574 OpVT);
2575 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2576 return GetUndefBooleanConstant();
2577 [[fallthrough]];
2578 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2579 VT, OpVT);
2580 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2581 return GetUndefBooleanConstant();
2582 [[fallthrough]];
2583 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2584 R==APFloat::cmpEqual, dl, VT,
2585 OpVT);
2586 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2587 return GetUndefBooleanConstant();
2588 [[fallthrough]];
2589 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2590 R==APFloat::cmpEqual, dl, VT, OpVT);
2591 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2592 OpVT);
2593 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2594 OpVT);
2595 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2596 R==APFloat::cmpEqual, dl, VT,
2597 OpVT);
2598 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2599 OpVT);
2600 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2601 R==APFloat::cmpLessThan, dl, VT,
2602 OpVT);
2603 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2604 R==APFloat::cmpUnordered, dl, VT,
2605 OpVT);
2606 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2607 VT, OpVT);
2608 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2609 OpVT);
2610 }
2611 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2612 // Ensure that the constant occurs on the RHS.
2613 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2614 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2615 return SDValue();
2616 return getSetCC(dl, VT, N2, N1, SwappedCond);
2617 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2618 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2619 // If an operand is known to be a nan (or undef that could be a nan), we can
2620 // fold it.
2621 // Choosing NaN for the undef will always make unordered comparison succeed
2622 // and ordered comparison fails.
2623 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2624 switch (ISD::getUnorderedFlavor(Cond)) {
2625 default:
2626 llvm_unreachable("Unknown flavor!");
2627 case 0: // Known false.
2628 return getBoolConstant(false, dl, VT, OpVT);
2629 case 1: // Known true.
2630 return getBoolConstant(true, dl, VT, OpVT);
2631 case 2: // Undefined.
2632 return GetUndefBooleanConstant();
2633 }
2634 }
2635
2636 // Could not fold it.
2637 return SDValue();
2638 }
2639
2640 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2641 /// use this predicate to simplify operations downstream.
SignBitIsZero(SDValue Op,unsigned Depth) const2642 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2643 unsigned BitWidth = Op.getScalarValueSizeInBits();
2644 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2645 }
2646
2647 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2648 /// this predicate to simplify operations downstream. Mask is known to be zero
2649 /// for bits that V cannot have.
MaskedValueIsZero(SDValue V,const APInt & Mask,unsigned Depth) const2650 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2651 unsigned Depth) const {
2652 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2653 }
2654
2655 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2656 /// DemandedElts. We use this predicate to simplify operations downstream.
2657 /// Mask is known to be zero for bits that V cannot have.
MaskedValueIsZero(SDValue V,const APInt & Mask,const APInt & DemandedElts,unsigned Depth) const2658 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2659 const APInt &DemandedElts,
2660 unsigned Depth) const {
2661 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2662 }
2663
2664 /// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2665 /// DemandedElts. We use this predicate to simplify operations downstream.
MaskedVectorIsZero(SDValue V,const APInt & DemandedElts,unsigned Depth) const2666 bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2667 unsigned Depth /* = 0 */) const {
2668 return computeKnownBits(V, DemandedElts, Depth).isZero();
2669 }
2670
2671 /// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
MaskedValueIsAllOnes(SDValue V,const APInt & Mask,unsigned Depth) const2672 bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2673 unsigned Depth) const {
2674 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2675 }
2676
computeVectorKnownZeroElements(SDValue Op,const APInt & DemandedElts,unsigned Depth) const2677 APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2678 const APInt &DemandedElts,
2679 unsigned Depth) const {
2680 EVT VT = Op.getValueType();
2681 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2682
2683 unsigned NumElts = VT.getVectorNumElements();
2684 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2685
2686 APInt KnownZeroElements = APInt::getZero(NumElts);
2687 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2688 if (!DemandedElts[EltIdx])
2689 continue; // Don't query elements that are not demanded.
2690 APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
2691 if (MaskedVectorIsZero(Op, Mask, Depth))
2692 KnownZeroElements.setBit(EltIdx);
2693 }
2694 return KnownZeroElements;
2695 }
2696
2697 /// isSplatValue - Return true if the vector V has the same value
2698 /// across all DemandedElts. For scalable vectors, we don't know the
2699 /// number of lanes at compile time. Instead, we use a 1 bit APInt
2700 /// to represent a conservative value for all lanes; that is, that
2701 /// one bit value is implicitly splatted across all lanes.
isSplatValue(SDValue V,const APInt & DemandedElts,APInt & UndefElts,unsigned Depth) const2702 bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2703 APInt &UndefElts, unsigned Depth) const {
2704 unsigned Opcode = V.getOpcode();
2705 EVT VT = V.getValueType();
2706 assert(VT.isVector() && "Vector type expected");
2707 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2708 "scalable demanded bits are ignored");
2709
2710 if (!DemandedElts)
2711 return false; // No demanded elts, better to assume we don't know anything.
2712
2713 if (Depth >= MaxRecursionDepth)
2714 return false; // Limit search depth.
2715
2716 // Deal with some common cases here that work for both fixed and scalable
2717 // vector types.
2718 switch (Opcode) {
2719 case ISD::SPLAT_VECTOR:
2720 UndefElts = V.getOperand(0).isUndef()
2721 ? APInt::getAllOnes(DemandedElts.getBitWidth())
2722 : APInt(DemandedElts.getBitWidth(), 0);
2723 return true;
2724 case ISD::ADD:
2725 case ISD::SUB:
2726 case ISD::AND:
2727 case ISD::XOR:
2728 case ISD::OR: {
2729 APInt UndefLHS, UndefRHS;
2730 SDValue LHS = V.getOperand(0);
2731 SDValue RHS = V.getOperand(1);
2732 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
2733 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
2734 UndefElts = UndefLHS | UndefRHS;
2735 return true;
2736 }
2737 return false;
2738 }
2739 case ISD::ABS:
2740 case ISD::TRUNCATE:
2741 case ISD::SIGN_EXTEND:
2742 case ISD::ZERO_EXTEND:
2743 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
2744 default:
2745 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2746 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2747 return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this,
2748 Depth);
2749 break;
2750 }
2751
2752 // We don't support other cases than those above for scalable vectors at
2753 // the moment.
2754 if (VT.isScalableVector())
2755 return false;
2756
2757 unsigned NumElts = VT.getVectorNumElements();
2758 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2759 UndefElts = APInt::getZero(NumElts);
2760
2761 switch (Opcode) {
2762 case ISD::BUILD_VECTOR: {
2763 SDValue Scl;
2764 for (unsigned i = 0; i != NumElts; ++i) {
2765 SDValue Op = V.getOperand(i);
2766 if (Op.isUndef()) {
2767 UndefElts.setBit(i);
2768 continue;
2769 }
2770 if (!DemandedElts[i])
2771 continue;
2772 if (Scl && Scl != Op)
2773 return false;
2774 Scl = Op;
2775 }
2776 return true;
2777 }
2778 case ISD::VECTOR_SHUFFLE: {
2779 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
2780 APInt DemandedLHS = APInt::getZero(NumElts);
2781 APInt DemandedRHS = APInt::getZero(NumElts);
2782 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2783 for (int i = 0; i != (int)NumElts; ++i) {
2784 int M = Mask[i];
2785 if (M < 0) {
2786 UndefElts.setBit(i);
2787 continue;
2788 }
2789 if (!DemandedElts[i])
2790 continue;
2791 if (M < (int)NumElts)
2792 DemandedLHS.setBit(M);
2793 else
2794 DemandedRHS.setBit(M - NumElts);
2795 }
2796
2797 // If we aren't demanding either op, assume there's no splat.
2798 // If we are demanding both ops, assume there's no splat.
2799 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
2800 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
2801 return false;
2802
2803 // See if the demanded elts of the source op is a splat or we only demand
2804 // one element, which should always be a splat.
2805 // TODO: Handle source ops splats with undefs.
2806 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
2807 APInt SrcUndefs;
2808 return (SrcElts.popcount() == 1) ||
2809 (isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) &&
2810 (SrcElts & SrcUndefs).isZero());
2811 };
2812 if (!DemandedLHS.isZero())
2813 return CheckSplatSrc(V.getOperand(0), DemandedLHS);
2814 return CheckSplatSrc(V.getOperand(1), DemandedRHS);
2815 }
2816 case ISD::EXTRACT_SUBVECTOR: {
2817 // Offset the demanded elts by the subvector index.
2818 SDValue Src = V.getOperand(0);
2819 // We don't support scalable vectors at the moment.
2820 if (Src.getValueType().isScalableVector())
2821 return false;
2822 uint64_t Idx = V.getConstantOperandVal(1);
2823 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2824 APInt UndefSrcElts;
2825 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
2826 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2827 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2828 return true;
2829 }
2830 break;
2831 }
2832 case ISD::ANY_EXTEND_VECTOR_INREG:
2833 case ISD::SIGN_EXTEND_VECTOR_INREG:
2834 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2835 // Widen the demanded elts by the src element count.
2836 SDValue Src = V.getOperand(0);
2837 // We don't support scalable vectors at the moment.
2838 if (Src.getValueType().isScalableVector())
2839 return false;
2840 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2841 APInt UndefSrcElts;
2842 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
2843 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2844 UndefElts = UndefSrcElts.trunc(NumElts);
2845 return true;
2846 }
2847 break;
2848 }
2849 case ISD::BITCAST: {
2850 SDValue Src = V.getOperand(0);
2851 EVT SrcVT = Src.getValueType();
2852 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
2853 unsigned BitWidth = VT.getScalarSizeInBits();
2854
2855 // Ignore bitcasts from unsupported types.
2856 // TODO: Add fp support?
2857 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
2858 break;
2859
2860 // Bitcast 'small element' vector to 'large element' vector.
2861 if ((BitWidth % SrcBitWidth) == 0) {
2862 // See if each sub element is a splat.
2863 unsigned Scale = BitWidth / SrcBitWidth;
2864 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2865 APInt ScaledDemandedElts =
2866 APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2867 for (unsigned I = 0; I != Scale; ++I) {
2868 APInt SubUndefElts;
2869 APInt SubDemandedElt = APInt::getOneBitSet(Scale, I);
2870 APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt);
2871 SubDemandedElts &= ScaledDemandedElts;
2872 if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1))
2873 return false;
2874 // TODO: Add support for merging sub undef elements.
2875 if (!SubUndefElts.isZero())
2876 return false;
2877 }
2878 return true;
2879 }
2880 break;
2881 }
2882 }
2883
2884 return false;
2885 }
2886
2887 /// Helper wrapper to main isSplatValue function.
isSplatValue(SDValue V,bool AllowUndefs) const2888 bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
2889 EVT VT = V.getValueType();
2890 assert(VT.isVector() && "Vector type expected");
2891
2892 APInt UndefElts;
2893 // Since the number of lanes in a scalable vector is unknown at compile time,
2894 // we track one bit which is implicitly broadcast to all lanes. This means
2895 // that all lanes in a scalable vector are considered demanded.
2896 APInt DemandedElts
2897 = APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements());
2898 return isSplatValue(V, DemandedElts, UndefElts) &&
2899 (AllowUndefs || !UndefElts);
2900 }
2901
getSplatSourceVector(SDValue V,int & SplatIdx)2902 SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2903 V = peekThroughExtractSubvectors(V);
2904
2905 EVT VT = V.getValueType();
2906 unsigned Opcode = V.getOpcode();
2907 switch (Opcode) {
2908 default: {
2909 APInt UndefElts;
2910 // Since the number of lanes in a scalable vector is unknown at compile time,
2911 // we track one bit which is implicitly broadcast to all lanes. This means
2912 // that all lanes in a scalable vector are considered demanded.
2913 APInt DemandedElts
2914 = APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements());
2915
2916 if (isSplatValue(V, DemandedElts, UndefElts)) {
2917 if (VT.isScalableVector()) {
2918 // DemandedElts and UndefElts are ignored for scalable vectors, since
2919 // the only supported cases are SPLAT_VECTOR nodes.
2920 SplatIdx = 0;
2921 } else {
2922 // Handle case where all demanded elements are UNDEF.
2923 if (DemandedElts.isSubsetOf(UndefElts)) {
2924 SplatIdx = 0;
2925 return getUNDEF(VT);
2926 }
2927 SplatIdx = (UndefElts & DemandedElts).countr_one();
2928 }
2929 return V;
2930 }
2931 break;
2932 }
2933 case ISD::SPLAT_VECTOR:
2934 SplatIdx = 0;
2935 return V;
2936 case ISD::VECTOR_SHUFFLE: {
2937 assert(!VT.isScalableVector());
2938 // Check if this is a shuffle node doing a splat.
2939 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2940 // getTargetVShiftNode currently struggles without the splat source.
2941 auto *SVN = cast<ShuffleVectorSDNode>(V);
2942 if (!SVN->isSplat())
2943 break;
2944 int Idx = SVN->getSplatIndex();
2945 int NumElts = V.getValueType().getVectorNumElements();
2946 SplatIdx = Idx % NumElts;
2947 return V.getOperand(Idx / NumElts);
2948 }
2949 }
2950
2951 return SDValue();
2952 }
2953
getSplatValue(SDValue V,bool LegalTypes)2954 SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
2955 int SplatIdx;
2956 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2957 EVT SVT = SrcVector.getValueType().getScalarType();
2958 EVT LegalSVT = SVT;
2959 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
2960 if (!SVT.isInteger())
2961 return SDValue();
2962 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
2963 if (LegalSVT.bitsLT(SVT))
2964 return SDValue();
2965 }
2966 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
2967 getVectorIdxConstant(SplatIdx, SDLoc(V)));
2968 }
2969 return SDValue();
2970 }
2971
2972 const APInt *
getValidShiftAmountConstant(SDValue V,const APInt & DemandedElts) const2973 SelectionDAG::getValidShiftAmountConstant(SDValue V,
2974 const APInt &DemandedElts) const {
2975 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2976 V.getOpcode() == ISD::SRA) &&
2977 "Unknown shift node");
2978 unsigned BitWidth = V.getScalarValueSizeInBits();
2979 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) {
2980 // Shifting more than the bitwidth is not valid.
2981 const APInt &ShAmt = SA->getAPIntValue();
2982 if (ShAmt.ult(BitWidth))
2983 return &ShAmt;
2984 }
2985 return nullptr;
2986 }
2987
getValidMinimumShiftAmountConstant(SDValue V,const APInt & DemandedElts) const2988 const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(
2989 SDValue V, const APInt &DemandedElts) const {
2990 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2991 V.getOpcode() == ISD::SRA) &&
2992 "Unknown shift node");
2993 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2994 return ValidAmt;
2995 unsigned BitWidth = V.getScalarValueSizeInBits();
2996 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2997 if (!BV)
2998 return nullptr;
2999 const APInt *MinShAmt = nullptr;
3000 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3001 if (!DemandedElts[i])
3002 continue;
3003 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3004 if (!SA)
3005 return nullptr;
3006 // Shifting more than the bitwidth is not valid.
3007 const APInt &ShAmt = SA->getAPIntValue();
3008 if (ShAmt.uge(BitWidth))
3009 return nullptr;
3010 if (MinShAmt && MinShAmt->ule(ShAmt))
3011 continue;
3012 MinShAmt = &ShAmt;
3013 }
3014 return MinShAmt;
3015 }
3016
getValidMaximumShiftAmountConstant(SDValue V,const APInt & DemandedElts) const3017 const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(
3018 SDValue V, const APInt &DemandedElts) const {
3019 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3020 V.getOpcode() == ISD::SRA) &&
3021 "Unknown shift node");
3022 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3023 return ValidAmt;
3024 unsigned BitWidth = V.getScalarValueSizeInBits();
3025 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
3026 if (!BV)
3027 return nullptr;
3028 const APInt *MaxShAmt = nullptr;
3029 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3030 if (!DemandedElts[i])
3031 continue;
3032 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3033 if (!SA)
3034 return nullptr;
3035 // Shifting more than the bitwidth is not valid.
3036 const APInt &ShAmt = SA->getAPIntValue();
3037 if (ShAmt.uge(BitWidth))
3038 return nullptr;
3039 if (MaxShAmt && MaxShAmt->uge(ShAmt))
3040 continue;
3041 MaxShAmt = &ShAmt;
3042 }
3043 return MaxShAmt;
3044 }
3045
3046 /// Determine which bits of Op are known to be either zero or one and return
3047 /// them in Known. For vectors, the known bits are those that are shared by
3048 /// every vector element.
computeKnownBits(SDValue Op,unsigned Depth) const3049 KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3050 EVT VT = Op.getValueType();
3051
3052 // Since the number of lanes in a scalable vector is unknown at compile time,
3053 // we track one bit which is implicitly broadcast to all lanes. This means
3054 // that all lanes in a scalable vector are considered demanded.
3055 APInt DemandedElts = VT.isFixedLengthVector()
3056 ? APInt::getAllOnes(VT.getVectorNumElements())
3057 : APInt(1, 1);
3058 return computeKnownBits(Op, DemandedElts, Depth);
3059 }
3060
3061 /// Determine which bits of Op are known to be either zero or one and return
3062 /// them in Known. The DemandedElts argument allows us to only collect the known
3063 /// bits that are shared by the requested vector elements.
computeKnownBits(SDValue Op,const APInt & DemandedElts,unsigned Depth) const3064 KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3065 unsigned Depth) const {
3066 unsigned BitWidth = Op.getScalarValueSizeInBits();
3067
3068 KnownBits Known(BitWidth); // Don't know anything.
3069
3070 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3071 // We know all of the bits for a constant!
3072 return KnownBits::makeConstant(C->getAPIntValue());
3073 }
3074 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
3075 // We know all of the bits for a constant fp!
3076 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
3077 }
3078
3079 if (Depth >= MaxRecursionDepth)
3080 return Known; // Limit search depth.
3081
3082 KnownBits Known2;
3083 unsigned NumElts = DemandedElts.getBitWidth();
3084 assert((!Op.getValueType().isFixedLengthVector() ||
3085 NumElts == Op.getValueType().getVectorNumElements()) &&
3086 "Unexpected vector size");
3087
3088 if (!DemandedElts)
3089 return Known; // No demanded elts, better to assume we don't know anything.
3090
3091 unsigned Opcode = Op.getOpcode();
3092 switch (Opcode) {
3093 case ISD::MERGE_VALUES:
3094 return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts,
3095 Depth + 1);
3096 case ISD::SPLAT_VECTOR: {
3097 SDValue SrcOp = Op.getOperand(0);
3098 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3099 "Expected SPLAT_VECTOR implicit truncation");
3100 // Implicitly truncate the bits to match the official semantics of
3101 // SPLAT_VECTOR.
3102 Known = computeKnownBits(SrcOp, Depth + 1).trunc(BitWidth);
3103 break;
3104 }
3105 case ISD::SPLAT_VECTOR_PARTS: {
3106 unsigned ScalarSize = Op.getOperand(0).getScalarValueSizeInBits();
3107 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3108 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3109 for (auto [I, SrcOp] : enumerate(Op->ops())) {
3110 Known.insertBits(computeKnownBits(SrcOp, Depth + 1), ScalarSize * I);
3111 }
3112 break;
3113 }
3114 case ISD::BUILD_VECTOR:
3115 assert(!Op.getValueType().isScalableVector());
3116 // Collect the known bits that are shared by every demanded vector element.
3117 Known.Zero.setAllBits(); Known.One.setAllBits();
3118 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3119 if (!DemandedElts[i])
3120 continue;
3121
3122 SDValue SrcOp = Op.getOperand(i);
3123 Known2 = computeKnownBits(SrcOp, Depth + 1);
3124
3125 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3126 if (SrcOp.getValueSizeInBits() != BitWidth) {
3127 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3128 "Expected BUILD_VECTOR implicit truncation");
3129 Known2 = Known2.trunc(BitWidth);
3130 }
3131
3132 // Known bits are the values that are shared by every demanded element.
3133 Known = Known.intersectWith(Known2);
3134
3135 // If we don't know any bits, early out.
3136 if (Known.isUnknown())
3137 break;
3138 }
3139 break;
3140 case ISD::VECTOR_SHUFFLE: {
3141 assert(!Op.getValueType().isScalableVector());
3142 // Collect the known bits that are shared by every vector element referenced
3143 // by the shuffle.
3144 APInt DemandedLHS, DemandedRHS;
3145 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3146 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3147 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
3148 DemandedLHS, DemandedRHS))
3149 break;
3150
3151 // Known bits are the values that are shared by every demanded element.
3152 Known.Zero.setAllBits(); Known.One.setAllBits();
3153 if (!!DemandedLHS) {
3154 SDValue LHS = Op.getOperand(0);
3155 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
3156 Known = Known.intersectWith(Known2);
3157 }
3158 // If we don't know any bits, early out.
3159 if (Known.isUnknown())
3160 break;
3161 if (!!DemandedRHS) {
3162 SDValue RHS = Op.getOperand(1);
3163 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
3164 Known = Known.intersectWith(Known2);
3165 }
3166 break;
3167 }
3168 case ISD::VSCALE: {
3169 const Function &F = getMachineFunction().getFunction();
3170 const APInt &Multiplier = Op.getConstantOperandAPInt(0);
3171 Known = getVScaleRange(&F, BitWidth).multiply(Multiplier).toKnownBits();
3172 break;
3173 }
3174 case ISD::CONCAT_VECTORS: {
3175 if (Op.getValueType().isScalableVector())
3176 break;
3177 // Split DemandedElts and test each of the demanded subvectors.
3178 Known.Zero.setAllBits(); Known.One.setAllBits();
3179 EVT SubVectorVT = Op.getOperand(0).getValueType();
3180 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3181 unsigned NumSubVectors = Op.getNumOperands();
3182 for (unsigned i = 0; i != NumSubVectors; ++i) {
3183 APInt DemandedSub =
3184 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
3185 if (!!DemandedSub) {
3186 SDValue Sub = Op.getOperand(i);
3187 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
3188 Known = Known.intersectWith(Known2);
3189 }
3190 // If we don't know any bits, early out.
3191 if (Known.isUnknown())
3192 break;
3193 }
3194 break;
3195 }
3196 case ISD::INSERT_SUBVECTOR: {
3197 if (Op.getValueType().isScalableVector())
3198 break;
3199 // Demand any elements from the subvector and the remainder from the src its
3200 // inserted into.
3201 SDValue Src = Op.getOperand(0);
3202 SDValue Sub = Op.getOperand(1);
3203 uint64_t Idx = Op.getConstantOperandVal(2);
3204 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3205 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3206 APInt DemandedSrcElts = DemandedElts;
3207 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
3208
3209 Known.One.setAllBits();
3210 Known.Zero.setAllBits();
3211 if (!!DemandedSubElts) {
3212 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
3213 if (Known.isUnknown())
3214 break; // early-out.
3215 }
3216 if (!!DemandedSrcElts) {
3217 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3218 Known = Known.intersectWith(Known2);
3219 }
3220 break;
3221 }
3222 case ISD::EXTRACT_SUBVECTOR: {
3223 // Offset the demanded elts by the subvector index.
3224 SDValue Src = Op.getOperand(0);
3225 // Bail until we can represent demanded elements for scalable vectors.
3226 if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3227 break;
3228 uint64_t Idx = Op.getConstantOperandVal(1);
3229 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3230 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3231 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3232 break;
3233 }
3234 case ISD::SCALAR_TO_VECTOR: {
3235 if (Op.getValueType().isScalableVector())
3236 break;
3237 // We know about scalar_to_vector as much as we know about it source,
3238 // which becomes the first element of otherwise unknown vector.
3239 if (DemandedElts != 1)
3240 break;
3241
3242 SDValue N0 = Op.getOperand(0);
3243 Known = computeKnownBits(N0, Depth + 1);
3244 if (N0.getValueSizeInBits() != BitWidth)
3245 Known = Known.trunc(BitWidth);
3246
3247 break;
3248 }
3249 case ISD::BITCAST: {
3250 if (Op.getValueType().isScalableVector())
3251 break;
3252
3253 SDValue N0 = Op.getOperand(0);
3254 EVT SubVT = N0.getValueType();
3255 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3256
3257 // Ignore bitcasts from unsupported types.
3258 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3259 break;
3260
3261 // Fast handling of 'identity' bitcasts.
3262 if (BitWidth == SubBitWidth) {
3263 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
3264 break;
3265 }
3266
3267 bool IsLE = getDataLayout().isLittleEndian();
3268
3269 // Bitcast 'small element' vector to 'large element' scalar/vector.
3270 if ((BitWidth % SubBitWidth) == 0) {
3271 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3272
3273 // Collect known bits for the (larger) output by collecting the known
3274 // bits from each set of sub elements and shift these into place.
3275 // We need to separately call computeKnownBits for each set of
3276 // sub elements as the knownbits for each is likely to be different.
3277 unsigned SubScale = BitWidth / SubBitWidth;
3278 APInt SubDemandedElts(NumElts * SubScale, 0);
3279 for (unsigned i = 0; i != NumElts; ++i)
3280 if (DemandedElts[i])
3281 SubDemandedElts.setBit(i * SubScale);
3282
3283 for (unsigned i = 0; i != SubScale; ++i) {
3284 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
3285 Depth + 1);
3286 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3287 Known.insertBits(Known2, SubBitWidth * Shifts);
3288 }
3289 }
3290
3291 // Bitcast 'large element' scalar/vector to 'small element' vector.
3292 if ((SubBitWidth % BitWidth) == 0) {
3293 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3294
3295 // Collect known bits for the (smaller) output by collecting the known
3296 // bits from the overlapping larger input elements and extracting the
3297 // sub sections we actually care about.
3298 unsigned SubScale = SubBitWidth / BitWidth;
3299 APInt SubDemandedElts =
3300 APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);
3301 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
3302
3303 Known.Zero.setAllBits(); Known.One.setAllBits();
3304 for (unsigned i = 0; i != NumElts; ++i)
3305 if (DemandedElts[i]) {
3306 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3307 unsigned Offset = (Shifts % SubScale) * BitWidth;
3308 Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset));
3309 // If we don't know any bits, early out.
3310 if (Known.isUnknown())
3311 break;
3312 }
3313 }
3314 break;
3315 }
3316 case ISD::AND:
3317 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3318 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3319
3320 Known &= Known2;
3321 break;
3322 case ISD::OR:
3323 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3324 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3325
3326 Known |= Known2;
3327 break;
3328 case ISD::XOR:
3329 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3330 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3331
3332 Known ^= Known2;
3333 break;
3334 case ISD::MUL: {
3335 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3336 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3337 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3338 // TODO: SelfMultiply can be poison, but not undef.
3339 if (SelfMultiply)
3340 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3341 Op.getOperand(0), DemandedElts, false, Depth + 1);
3342 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3343
3344 // If the multiplication is known not to overflow, the product of a number
3345 // with itself is non-negative. Only do this if we didn't already computed
3346 // the opposite value for the sign bit.
3347 if (Op->getFlags().hasNoSignedWrap() &&
3348 Op.getOperand(0) == Op.getOperand(1) &&
3349 !Known.isNegative())
3350 Known.makeNonNegative();
3351 break;
3352 }
3353 case ISD::MULHU: {
3354 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3355 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3356 Known = KnownBits::mulhu(Known, Known2);
3357 break;
3358 }
3359 case ISD::MULHS: {
3360 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3361 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3362 Known = KnownBits::mulhs(Known, Known2);
3363 break;
3364 }
3365 case ISD::UMUL_LOHI: {
3366 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3367 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3368 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3369 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3370 if (Op.getResNo() == 0)
3371 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3372 else
3373 Known = KnownBits::mulhu(Known, Known2);
3374 break;
3375 }
3376 case ISD::SMUL_LOHI: {
3377 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3378 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3379 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3380 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3381 if (Op.getResNo() == 0)
3382 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3383 else
3384 Known = KnownBits::mulhs(Known, Known2);
3385 break;
3386 }
3387 case ISD::AVGCEILU: {
3388 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3389 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3390 Known = Known.zext(BitWidth + 1);
3391 Known2 = Known2.zext(BitWidth + 1);
3392 KnownBits One = KnownBits::makeConstant(APInt(1, 1));
3393 Known = KnownBits::computeForAddCarry(Known, Known2, One);
3394 Known = Known.extractBits(BitWidth, 1);
3395 break;
3396 }
3397 case ISD::SELECT:
3398 case ISD::VSELECT:
3399 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3400 // If we don't know any bits, early out.
3401 if (Known.isUnknown())
3402 break;
3403 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3404
3405 // Only known if known in both the LHS and RHS.
3406 Known = Known.intersectWith(Known2);
3407 break;
3408 case ISD::SELECT_CC:
3409 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3410 // If we don't know any bits, early out.
3411 if (Known.isUnknown())
3412 break;
3413 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3414
3415 // Only known if known in both the LHS and RHS.
3416 Known = Known.intersectWith(Known2);
3417 break;
3418 case ISD::SMULO:
3419 case ISD::UMULO:
3420 if (Op.getResNo() != 1)
3421 break;
3422 // The boolean result conforms to getBooleanContents.
3423 // If we know the result of a setcc has the top bits zero, use this info.
3424 // We know that we have an integer-based boolean since these operations
3425 // are only available for integer.
3426 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3427 TargetLowering::ZeroOrOneBooleanContent &&
3428 BitWidth > 1)
3429 Known.Zero.setBitsFrom(1);
3430 break;
3431 case ISD::SETCC:
3432 case ISD::SETCCCARRY:
3433 case ISD::STRICT_FSETCC:
3434 case ISD::STRICT_FSETCCS: {
3435 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3436 // If we know the result of a setcc has the top bits zero, use this info.
3437 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3438 TargetLowering::ZeroOrOneBooleanContent &&
3439 BitWidth > 1)
3440 Known.Zero.setBitsFrom(1);
3441 break;
3442 }
3443 case ISD::SHL:
3444 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3445 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3446 Known = KnownBits::shl(Known, Known2);
3447
3448 // Minimum shift low bits are known zero.
3449 if (const APInt *ShMinAmt =
3450 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3451 Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3452 break;
3453 case ISD::SRL:
3454 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3455 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3456 Known = KnownBits::lshr(Known, Known2);
3457
3458 // Minimum shift high bits are known zero.
3459 if (const APInt *ShMinAmt =
3460 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3461 Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3462 break;
3463 case ISD::SRA:
3464 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3465 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3466 Known = KnownBits::ashr(Known, Known2);
3467 break;
3468 case ISD::FSHL:
3469 case ISD::FSHR:
3470 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3471 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3472
3473 // For fshl, 0-shift returns the 1st arg.
3474 // For fshr, 0-shift returns the 2nd arg.
3475 if (Amt == 0) {
3476 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3477 DemandedElts, Depth + 1);
3478 break;
3479 }
3480
3481 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3482 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3483 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3484 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3485 if (Opcode == ISD::FSHL) {
3486 Known.One <<= Amt;
3487 Known.Zero <<= Amt;
3488 Known2.One.lshrInPlace(BitWidth - Amt);
3489 Known2.Zero.lshrInPlace(BitWidth - Amt);
3490 } else {
3491 Known.One <<= BitWidth - Amt;
3492 Known.Zero <<= BitWidth - Amt;
3493 Known2.One.lshrInPlace(Amt);
3494 Known2.Zero.lshrInPlace(Amt);
3495 }
3496 Known = Known.unionWith(Known2);
3497 }
3498 break;
3499 case ISD::SHL_PARTS:
3500 case ISD::SRA_PARTS:
3501 case ISD::SRL_PARTS: {
3502 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3503
3504 // Collect lo/hi source values and concatenate.
3505 unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits();
3506 unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits();
3507 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3508 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3509 Known = Known2.concat(Known);
3510
3511 // Collect shift amount.
3512 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3513
3514 if (Opcode == ISD::SHL_PARTS)
3515 Known = KnownBits::shl(Known, Known2);
3516 else if (Opcode == ISD::SRA_PARTS)
3517 Known = KnownBits::ashr(Known, Known2);
3518 else // if (Opcode == ISD::SRL_PARTS)
3519 Known = KnownBits::lshr(Known, Known2);
3520
3521 // TODO: Minimum shift low/high bits are known zero.
3522
3523 if (Op.getResNo() == 0)
3524 Known = Known.extractBits(LoBits, 0);
3525 else
3526 Known = Known.extractBits(HiBits, LoBits);
3527 break;
3528 }
3529 case ISD::SIGN_EXTEND_INREG: {
3530 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3531 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3532 Known = Known.sextInReg(EVT.getScalarSizeInBits());
3533 break;
3534 }
3535 case ISD::CTTZ:
3536 case ISD::CTTZ_ZERO_UNDEF: {
3537 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3538 // If we have a known 1, its position is our upper bound.
3539 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3540 unsigned LowBits = llvm::bit_width(PossibleTZ);
3541 Known.Zero.setBitsFrom(LowBits);
3542 break;
3543 }
3544 case ISD::CTLZ:
3545 case ISD::CTLZ_ZERO_UNDEF: {
3546 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3547 // If we have a known 1, its position is our upper bound.
3548 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3549 unsigned LowBits = llvm::bit_width(PossibleLZ);
3550 Known.Zero.setBitsFrom(LowBits);
3551 break;
3552 }
3553 case ISD::CTPOP: {
3554 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3555 // If we know some of the bits are zero, they can't be one.
3556 unsigned PossibleOnes = Known2.countMaxPopulation();
3557 Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes));
3558 break;
3559 }
3560 case ISD::PARITY: {
3561 // Parity returns 0 everywhere but the LSB.
3562 Known.Zero.setBitsFrom(1);
3563 break;
3564 }
3565 case ISD::LOAD: {
3566 LoadSDNode *LD = cast<LoadSDNode>(Op);
3567 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3568 if (ISD::isNON_EXTLoad(LD) && Cst) {
3569 // Determine any common known bits from the loaded constant pool value.
3570 Type *CstTy = Cst->getType();
3571 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3572 !Op.getValueType().isScalableVector()) {
3573 // If its a vector splat, then we can (quickly) reuse the scalar path.
3574 // NOTE: We assume all elements match and none are UNDEF.
3575 if (CstTy->isVectorTy()) {
3576 if (const Constant *Splat = Cst->getSplatValue()) {
3577 Cst = Splat;
3578 CstTy = Cst->getType();
3579 }
3580 }
3581 // TODO - do we need to handle different bitwidths?
3582 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3583 // Iterate across all vector elements finding common known bits.
3584 Known.One.setAllBits();
3585 Known.Zero.setAllBits();
3586 for (unsigned i = 0; i != NumElts; ++i) {
3587 if (!DemandedElts[i])
3588 continue;
3589 if (Constant *Elt = Cst->getAggregateElement(i)) {
3590 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3591 const APInt &Value = CInt->getValue();
3592 Known.One &= Value;
3593 Known.Zero &= ~Value;
3594 continue;
3595 }
3596 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3597 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3598 Known.One &= Value;
3599 Known.Zero &= ~Value;
3600 continue;
3601 }
3602 }
3603 Known.One.clearAllBits();
3604 Known.Zero.clearAllBits();
3605 break;
3606 }
3607 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3608 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3609 Known = KnownBits::makeConstant(CInt->getValue());
3610 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3611 Known =
3612 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
3613 }
3614 }
3615 }
3616 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
3617 // If this is a ZEXTLoad and we are looking at the loaded value.
3618 EVT VT = LD->getMemoryVT();
3619 unsigned MemBits = VT.getScalarSizeInBits();
3620 Known.Zero.setBitsFrom(MemBits);
3621 } else if (const MDNode *Ranges = LD->getRanges()) {
3622 EVT VT = LD->getValueType(0);
3623
3624 // TODO: Handle for extending loads
3625 if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
3626 if (VT.isVector()) {
3627 // Handle truncation to the first demanded element.
3628 // TODO: Figure out which demanded elements are covered
3629 if (DemandedElts != 1 || !getDataLayout().isLittleEndian())
3630 break;
3631
3632 // Handle the case where a load has a vector type, but scalar memory
3633 // with an attached range.
3634 EVT MemVT = LD->getMemoryVT();
3635 KnownBits KnownFull(MemVT.getSizeInBits());
3636
3637 computeKnownBitsFromRangeMetadata(*Ranges, KnownFull);
3638 Known = KnownFull.trunc(BitWidth);
3639 } else
3640 computeKnownBitsFromRangeMetadata(*Ranges, Known);
3641 }
3642 }
3643 break;
3644 }
3645 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3646 if (Op.getValueType().isScalableVector())
3647 break;
3648 EVT InVT = Op.getOperand(0).getValueType();
3649 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3650 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3651 Known = Known.zext(BitWidth);
3652 break;
3653 }
3654 case ISD::ZERO_EXTEND: {
3655 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3656 Known = Known.zext(BitWidth);
3657 break;
3658 }
3659 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3660 if (Op.getValueType().isScalableVector())
3661 break;
3662 EVT InVT = Op.getOperand(0).getValueType();
3663 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3664 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3665 // If the sign bit is known to be zero or one, then sext will extend
3666 // it to the top bits, else it will just zext.
3667 Known = Known.sext(BitWidth);
3668 break;
3669 }
3670 case ISD::SIGN_EXTEND: {
3671 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3672 // If the sign bit is known to be zero or one, then sext will extend
3673 // it to the top bits, else it will just zext.
3674 Known = Known.sext(BitWidth);
3675 break;
3676 }
3677 case ISD::ANY_EXTEND_VECTOR_INREG: {
3678 if (Op.getValueType().isScalableVector())
3679 break;
3680 EVT InVT = Op.getOperand(0).getValueType();
3681 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3682 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3683 Known = Known.anyext(BitWidth);
3684 break;
3685 }
3686 case ISD::ANY_EXTEND: {
3687 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3688 Known = Known.anyext(BitWidth);
3689 break;
3690 }
3691 case ISD::TRUNCATE: {
3692 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3693 Known = Known.trunc(BitWidth);
3694 break;
3695 }
3696 case ISD::AssertZext: {
3697 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3698 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3699 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3700 Known.Zero |= (~InMask);
3701 Known.One &= (~Known.Zero);
3702 break;
3703 }
3704 case ISD::AssertAlign: {
3705 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
3706 assert(LogOfAlign != 0);
3707
3708 // TODO: Should use maximum with source
3709 // If a node is guaranteed to be aligned, set low zero bits accordingly as
3710 // well as clearing one bits.
3711 Known.Zero.setLowBits(LogOfAlign);
3712 Known.One.clearLowBits(LogOfAlign);
3713 break;
3714 }
3715 case ISD::FGETSIGN:
3716 // All bits are zero except the low bit.
3717 Known.Zero.setBitsFrom(1);
3718 break;
3719 case ISD::ADD:
3720 case ISD::SUB: {
3721 SDNodeFlags Flags = Op.getNode()->getFlags();
3722 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3723 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3724 Known = KnownBits::computeForAddSub(Op.getOpcode() == ISD::ADD,
3725 Flags.hasNoSignedWrap(), Known, Known2);
3726 break;
3727 }
3728 case ISD::USUBO:
3729 case ISD::SSUBO:
3730 case ISD::USUBO_CARRY:
3731 case ISD::SSUBO_CARRY:
3732 if (Op.getResNo() == 1) {
3733 // If we know the result of a setcc has the top bits zero, use this info.
3734 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3735 TargetLowering::ZeroOrOneBooleanContent &&
3736 BitWidth > 1)
3737 Known.Zero.setBitsFrom(1);
3738 break;
3739 }
3740 [[fallthrough]];
3741 case ISD::SUBC: {
3742 assert(Op.getResNo() == 0 &&
3743 "We only compute knownbits for the difference here.");
3744
3745 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
3746 KnownBits Borrow(1);
3747 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
3748 Borrow = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3749 // Borrow has bit width 1
3750 Borrow = Borrow.trunc(1);
3751 } else {
3752 Borrow.setAllZero();
3753 }
3754
3755 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3756 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3757 Known = KnownBits::computeForSubBorrow(Known, Known2, Borrow);
3758 break;
3759 }
3760 case ISD::UADDO:
3761 case ISD::SADDO:
3762 case ISD::UADDO_CARRY:
3763 case ISD::SADDO_CARRY:
3764 if (Op.getResNo() == 1) {
3765 // If we know the result of a setcc has the top bits zero, use this info.
3766 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3767 TargetLowering::ZeroOrOneBooleanContent &&
3768 BitWidth > 1)
3769 Known.Zero.setBitsFrom(1);
3770 break;
3771 }
3772 [[fallthrough]];
3773 case ISD::ADDC:
3774 case ISD::ADDE: {
3775 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3776
3777 // With ADDE and UADDO_CARRY, a carry bit may be added in.
3778 KnownBits Carry(1);
3779 if (Opcode == ISD::ADDE)
3780 // Can't track carry from glue, set carry to unknown.
3781 Carry.resetAll();
3782 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
3783 Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3784 // Carry has bit width 1
3785 Carry = Carry.trunc(1);
3786 } else {
3787 Carry.setAllZero();
3788 }
3789
3790 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3791 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3792 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3793 break;
3794 }
3795 case ISD::UDIV: {
3796 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3797 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3798 Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact());
3799 break;
3800 }
3801 case ISD::SDIV: {
3802 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3803 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3804 Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact());
3805 break;
3806 }
3807 case ISD::SREM: {
3808 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3809 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3810 Known = KnownBits::srem(Known, Known2);
3811 break;
3812 }
3813 case ISD::UREM: {
3814 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3815 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3816 Known = KnownBits::urem(Known, Known2);
3817 break;
3818 }
3819 case ISD::EXTRACT_ELEMENT: {
3820 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3821 const unsigned Index = Op.getConstantOperandVal(1);
3822 const unsigned EltBitWidth = Op.getValueSizeInBits();
3823
3824 // Remove low part of known bits mask
3825 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3826 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3827
3828 // Remove high part of known bit mask
3829 Known = Known.trunc(EltBitWidth);
3830 break;
3831 }
3832 case ISD::EXTRACT_VECTOR_ELT: {
3833 SDValue InVec = Op.getOperand(0);
3834 SDValue EltNo = Op.getOperand(1);
3835 EVT VecVT = InVec.getValueType();
3836 // computeKnownBits not yet implemented for scalable vectors.
3837 if (VecVT.isScalableVector())
3838 break;
3839 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3840 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3841
3842 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3843 // anything about the extended bits.
3844 if (BitWidth > EltBitWidth)
3845 Known = Known.trunc(EltBitWidth);
3846
3847 // If we know the element index, just demand that vector element, else for
3848 // an unknown element index, ignore DemandedElts and demand them all.
3849 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
3850 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3851 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3852 DemandedSrcElts =
3853 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3854
3855 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
3856 if (BitWidth > EltBitWidth)
3857 Known = Known.anyext(BitWidth);
3858 break;
3859 }
3860 case ISD::INSERT_VECTOR_ELT: {
3861 if (Op.getValueType().isScalableVector())
3862 break;
3863
3864 // If we know the element index, split the demand between the
3865 // source vector and the inserted element, otherwise assume we need
3866 // the original demanded vector elements and the value.
3867 SDValue InVec = Op.getOperand(0);
3868 SDValue InVal = Op.getOperand(1);
3869 SDValue EltNo = Op.getOperand(2);
3870 bool DemandedVal = true;
3871 APInt DemandedVecElts = DemandedElts;
3872 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3873 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3874 unsigned EltIdx = CEltNo->getZExtValue();
3875 DemandedVal = !!DemandedElts[EltIdx];
3876 DemandedVecElts.clearBit(EltIdx);
3877 }
3878 Known.One.setAllBits();
3879 Known.Zero.setAllBits();
3880 if (DemandedVal) {
3881 Known2 = computeKnownBits(InVal, Depth + 1);
3882 Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth));
3883 }
3884 if (!!DemandedVecElts) {
3885 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
3886 Known = Known.intersectWith(Known2);
3887 }
3888 break;
3889 }
3890 case ISD::BITREVERSE: {
3891 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3892 Known = Known2.reverseBits();
3893 break;
3894 }
3895 case ISD::BSWAP: {
3896 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3897 Known = Known2.byteSwap();
3898 break;
3899 }
3900 case ISD::ABS: {
3901 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3902 Known = Known2.abs();
3903 break;
3904 }
3905 case ISD::USUBSAT: {
3906 // The result of usubsat will never be larger than the LHS.
3907 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3908 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
3909 break;
3910 }
3911 case ISD::UMIN: {
3912 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3913 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3914 Known = KnownBits::umin(Known, Known2);
3915 break;
3916 }
3917 case ISD::UMAX: {
3918 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3919 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3920 Known = KnownBits::umax(Known, Known2);
3921 break;
3922 }
3923 case ISD::SMIN:
3924 case ISD::SMAX: {
3925 // If we have a clamp pattern, we know that the number of sign bits will be
3926 // the minimum of the clamp min/max range.
3927 bool IsMax = (Opcode == ISD::SMAX);
3928 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3929 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3930 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3931 CstHigh =
3932 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3933 if (CstLow && CstHigh) {
3934 if (!IsMax)
3935 std::swap(CstLow, CstHigh);
3936
3937 const APInt &ValueLow = CstLow->getAPIntValue();
3938 const APInt &ValueHigh = CstHigh->getAPIntValue();
3939 if (ValueLow.sle(ValueHigh)) {
3940 unsigned LowSignBits = ValueLow.getNumSignBits();
3941 unsigned HighSignBits = ValueHigh.getNumSignBits();
3942 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3943 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3944 Known.One.setHighBits(MinSignBits);
3945 break;
3946 }
3947 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3948 Known.Zero.setHighBits(MinSignBits);
3949 break;
3950 }
3951 }
3952 }
3953
3954 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3955 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3956 if (IsMax)
3957 Known = KnownBits::smax(Known, Known2);
3958 else
3959 Known = KnownBits::smin(Known, Known2);
3960
3961 // For SMAX, if CstLow is non-negative we know the result will be
3962 // non-negative and thus all sign bits are 0.
3963 // TODO: There's an equivalent of this for smin with negative constant for
3964 // known ones.
3965 if (IsMax && CstLow) {
3966 const APInt &ValueLow = CstLow->getAPIntValue();
3967 if (ValueLow.isNonNegative()) {
3968 unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3969 Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits()));
3970 }
3971 }
3972
3973 break;
3974 }
3975 case ISD::FP_TO_UINT_SAT: {
3976 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
3977 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3978 Known.Zero |= APInt::getBitsSetFrom(BitWidth, VT.getScalarSizeInBits());
3979 break;
3980 }
3981 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
3982 if (Op.getResNo() == 1) {
3983 // The boolean result conforms to getBooleanContents.
3984 // If we know the result of a setcc has the top bits zero, use this info.
3985 // We know that we have an integer-based boolean since these operations
3986 // are only available for integer.
3987 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3988 TargetLowering::ZeroOrOneBooleanContent &&
3989 BitWidth > 1)
3990 Known.Zero.setBitsFrom(1);
3991 break;
3992 }
3993 [[fallthrough]];
3994 case ISD::ATOMIC_CMP_SWAP:
3995 case ISD::ATOMIC_SWAP:
3996 case ISD::ATOMIC_LOAD_ADD:
3997 case ISD::ATOMIC_LOAD_SUB:
3998 case ISD::ATOMIC_LOAD_AND:
3999 case ISD::ATOMIC_LOAD_CLR:
4000 case ISD::ATOMIC_LOAD_OR:
4001 case ISD::ATOMIC_LOAD_XOR:
4002 case ISD::ATOMIC_LOAD_NAND:
4003 case ISD::ATOMIC_LOAD_MIN:
4004 case ISD::ATOMIC_LOAD_MAX:
4005 case ISD::ATOMIC_LOAD_UMIN:
4006 case ISD::ATOMIC_LOAD_UMAX:
4007 case ISD::ATOMIC_LOAD: {
4008 unsigned MemBits =
4009 cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4010 // If we are looking at the loaded value.
4011 if (Op.getResNo() == 0) {
4012 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4013 Known.Zero.setBitsFrom(MemBits);
4014 }
4015 break;
4016 }
4017 case ISD::FrameIndex:
4018 case ISD::TargetFrameIndex:
4019 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
4020 Known, getMachineFunction());
4021 break;
4022
4023 default:
4024 if (Opcode < ISD::BUILTIN_OP_END)
4025 break;
4026 [[fallthrough]];
4027 case ISD::INTRINSIC_WO_CHAIN:
4028 case ISD::INTRINSIC_W_CHAIN:
4029 case ISD::INTRINSIC_VOID:
4030 // TODO: Probably okay to remove after audit; here to reduce change size
4031 // in initial enablement patch for scalable vectors
4032 if (Op.getValueType().isScalableVector())
4033 break;
4034
4035 // Allow the target to implement this method for its nodes.
4036 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
4037 break;
4038 }
4039
4040 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
4041 return Known;
4042 }
4043
4044 /// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
mapOverflowResult(ConstantRange::OverflowResult OR)4045 static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4046 switch (OR) {
4047 case ConstantRange::OverflowResult::MayOverflow:
4048 return SelectionDAG::OFK_Sometime;
4049 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4050 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4051 return SelectionDAG::OFK_Always;
4052 case ConstantRange::OverflowResult::NeverOverflows:
4053 return SelectionDAG::OFK_Never;
4054 }
4055 llvm_unreachable("Unknown OverflowResult");
4056 }
4057
4058 SelectionDAG::OverflowKind
computeOverflowForSignedAdd(SDValue N0,SDValue N1) const4059 SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4060 // X + 0 never overflow
4061 if (isNullConstant(N1))
4062 return OFK_Never;
4063
4064 // If both operands each have at least two sign bits, the addition
4065 // cannot overflow.
4066 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4067 return OFK_Never;
4068
4069 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4070 return OFK_Sometime;
4071 }
4072
4073 SelectionDAG::OverflowKind
computeOverflowForUnsignedAdd(SDValue N0,SDValue N1) const4074 SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4075 // X + 0 never overflow
4076 if (isNullConstant(N1))
4077 return OFK_Never;
4078
4079 // mulhi + 1 never overflow
4080 KnownBits N1Known = computeKnownBits(N1);
4081 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4082 N1Known.getMaxValue().ult(2))
4083 return OFK_Never;
4084
4085 KnownBits N0Known = computeKnownBits(N0);
4086 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4087 N0Known.getMaxValue().ult(2))
4088 return OFK_Never;
4089
4090 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4091 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4092 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4093 return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range));
4094 }
4095
4096 SelectionDAG::OverflowKind
computeOverflowForSignedSub(SDValue N0,SDValue N1) const4097 SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4098 // X - 0 never overflow
4099 if (isNullConstant(N1))
4100 return OFK_Never;
4101
4102 // If both operands each have at least two sign bits, the subtraction
4103 // cannot overflow.
4104 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4105 return OFK_Never;
4106
4107 KnownBits N0Known = computeKnownBits(N0);
4108 KnownBits N1Known = computeKnownBits(N1);
4109 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, true);
4110 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, true);
4111 return mapOverflowResult(N0Range.signedSubMayOverflow(N1Range));
4112 }
4113
4114 SelectionDAG::OverflowKind
computeOverflowForUnsignedSub(SDValue N0,SDValue N1) const4115 SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4116 // X - 0 never overflow
4117 if (isNullConstant(N1))
4118 return OFK_Never;
4119
4120 KnownBits N0Known = computeKnownBits(N0);
4121 KnownBits N1Known = computeKnownBits(N1);
4122 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4123 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4124 return mapOverflowResult(N0Range.unsignedSubMayOverflow(N1Range));
4125 }
4126
4127 SelectionDAG::OverflowKind
computeOverflowForUnsignedMul(SDValue N0,SDValue N1) const4128 SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4129 // X * 0 and X * 1 never overflow.
4130 if (isNullConstant(N1) || isOneConstant(N1))
4131 return OFK_Never;
4132
4133 KnownBits N0Known = computeKnownBits(N0);
4134 KnownBits N1Known = computeKnownBits(N1);
4135 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4136 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4137 return mapOverflowResult(N0Range.unsignedMulMayOverflow(N1Range));
4138 }
4139
4140 SelectionDAG::OverflowKind
computeOverflowForSignedMul(SDValue N0,SDValue N1) const4141 SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4142 // X * 0 and X * 1 never overflow.
4143 if (isNullConstant(N1) || isOneConstant(N1))
4144 return OFK_Never;
4145
4146 // Get the size of the result.
4147 unsigned BitWidth = N0.getScalarValueSizeInBits();
4148
4149 // Sum of the sign bits.
4150 unsigned SignBits = ComputeNumSignBits(N0) + ComputeNumSignBits(N1);
4151
4152 // If we have enough sign bits, then there's no overflow.
4153 if (SignBits > BitWidth + 1)
4154 return OFK_Never;
4155
4156 if (SignBits == BitWidth + 1) {
4157 // The overflow occurs when the true multiplication of the
4158 // the operands is the minimum negative number.
4159 KnownBits N0Known = computeKnownBits(N0);
4160 KnownBits N1Known = computeKnownBits(N1);
4161 // If one of the operands is non-negative, then there's no
4162 // overflow.
4163 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4164 return OFK_Never;
4165 }
4166
4167 return OFK_Sometime;
4168 }
4169
isKnownToBeAPowerOfTwo(SDValue Val,unsigned Depth) const4170 bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {
4171 if (Depth >= MaxRecursionDepth)
4172 return false; // Limit search depth.
4173
4174 EVT OpVT = Val.getValueType();
4175 unsigned BitWidth = OpVT.getScalarSizeInBits();
4176
4177 // Is the constant a known power of 2?
4178 if (ISD::matchUnaryPredicate(Val, [BitWidth](ConstantSDNode *C) {
4179 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4180 }))
4181 return true;
4182
4183 // A left-shift of a constant one will have exactly one bit set because
4184 // shifting the bit off the end is undefined.
4185 if (Val.getOpcode() == ISD::SHL) {
4186 auto *C = isConstOrConstSplat(Val.getOperand(0));
4187 if (C && C->getAPIntValue() == 1)
4188 return true;
4189 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4190 isKnownNeverZero(Val, Depth);
4191 }
4192
4193 // Similarly, a logical right-shift of a constant sign-bit will have exactly
4194 // one bit set.
4195 if (Val.getOpcode() == ISD::SRL) {
4196 auto *C = isConstOrConstSplat(Val.getOperand(0));
4197 if (C && C->getAPIntValue().isSignMask())
4198 return true;
4199 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4200 isKnownNeverZero(Val, Depth);
4201 }
4202
4203 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4204 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4205
4206 // Are all operands of a build vector constant powers of two?
4207 if (Val.getOpcode() == ISD::BUILD_VECTOR)
4208 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
4209 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
4210 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4211 return false;
4212 }))
4213 return true;
4214
4215 // Is the operand of a splat vector a constant power of two?
4216 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4217 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val->getOperand(0)))
4218 if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2())
4219 return true;
4220
4221 // vscale(power-of-two) is a power-of-two for some targets
4222 if (Val.getOpcode() == ISD::VSCALE &&
4223 getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4224 isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1))
4225 return true;
4226
4227 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4228 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4229 return isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1) &&
4230 isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4231
4232 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4233 return isKnownToBeAPowerOfTwo(Val.getOperand(2), Depth + 1) &&
4234 isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1);
4235
4236 if (Val.getOpcode() == ISD::AND) {
4237 // Looking for `x & -x` pattern:
4238 // If x == 0:
4239 // x & -x -> 0
4240 // If x != 0:
4241 // x & -x -> non-zero pow2
4242 // so if we find the pattern return whether we know `x` is non-zero.
4243 for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx) {
4244 SDValue NegOp = Val.getOperand(OpIdx);
4245 if (NegOp.getOpcode() == ISD::SUB &&
4246 NegOp.getOperand(1) == Val.getOperand(1 - OpIdx) &&
4247 isNullOrNullSplat(NegOp.getOperand(0)))
4248 return isKnownNeverZero(Val.getOperand(1 - OpIdx), Depth);
4249 }
4250 }
4251
4252 if (Val.getOpcode() == ISD::ZERO_EXTEND)
4253 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4254
4255 // More could be done here, though the above checks are enough
4256 // to handle some common cases.
4257 return false;
4258 }
4259
ComputeNumSignBits(SDValue Op,unsigned Depth) const4260 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4261 EVT VT = Op.getValueType();
4262
4263 // Since the number of lanes in a scalable vector is unknown at compile time,
4264 // we track one bit which is implicitly broadcast to all lanes. This means
4265 // that all lanes in a scalable vector are considered demanded.
4266 APInt DemandedElts = VT.isFixedLengthVector()
4267 ? APInt::getAllOnes(VT.getVectorNumElements())
4268 : APInt(1, 1);
4269 return ComputeNumSignBits(Op, DemandedElts, Depth);
4270 }
4271
ComputeNumSignBits(SDValue Op,const APInt & DemandedElts,unsigned Depth) const4272 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4273 unsigned Depth) const {
4274 EVT VT = Op.getValueType();
4275 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4276 unsigned VTBits = VT.getScalarSizeInBits();
4277 unsigned NumElts = DemandedElts.getBitWidth();
4278 unsigned Tmp, Tmp2;
4279 unsigned FirstAnswer = 1;
4280
4281 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4282 const APInt &Val = C->getAPIntValue();
4283 return Val.getNumSignBits();
4284 }
4285
4286 if (Depth >= MaxRecursionDepth)
4287 return 1; // Limit search depth.
4288
4289 if (!DemandedElts)
4290 return 1; // No demanded elts, better to assume we don't know anything.
4291
4292 unsigned Opcode = Op.getOpcode();
4293 switch (Opcode) {
4294 default: break;
4295 case ISD::AssertSext:
4296 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4297 return VTBits-Tmp+1;
4298 case ISD::AssertZext:
4299 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4300 return VTBits-Tmp;
4301 case ISD::MERGE_VALUES:
4302 return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts,
4303 Depth + 1);
4304 case ISD::SPLAT_VECTOR: {
4305 // Check if the sign bits of source go down as far as the truncated value.
4306 unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits();
4307 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4308 if (NumSrcSignBits > (NumSrcBits - VTBits))
4309 return NumSrcSignBits - (NumSrcBits - VTBits);
4310 break;
4311 }
4312 case ISD::BUILD_VECTOR:
4313 assert(!VT.isScalableVector());
4314 Tmp = VTBits;
4315 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4316 if (!DemandedElts[i])
4317 continue;
4318
4319 SDValue SrcOp = Op.getOperand(i);
4320 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4321 // for constant nodes to ensure we only look at the sign bits.
4322 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SrcOp)) {
4323 APInt T = C->getAPIntValue().trunc(VTBits);
4324 Tmp2 = T.getNumSignBits();
4325 } else {
4326 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
4327
4328 if (SrcOp.getValueSizeInBits() != VTBits) {
4329 assert(SrcOp.getValueSizeInBits() > VTBits &&
4330 "Expected BUILD_VECTOR implicit truncation");
4331 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4332 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4333 }
4334 }
4335 Tmp = std::min(Tmp, Tmp2);
4336 }
4337 return Tmp;
4338
4339 case ISD::VECTOR_SHUFFLE: {
4340 // Collect the minimum number of sign bits that are shared by every vector
4341 // element referenced by the shuffle.
4342 APInt DemandedLHS, DemandedRHS;
4343 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
4344 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4345 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
4346 DemandedLHS, DemandedRHS))
4347 return 1;
4348
4349 Tmp = std::numeric_limits<unsigned>::max();
4350 if (!!DemandedLHS)
4351 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
4352 if (!!DemandedRHS) {
4353 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
4354 Tmp = std::min(Tmp, Tmp2);
4355 }
4356 // If we don't know anything, early out and try computeKnownBits fall-back.
4357 if (Tmp == 1)
4358 break;
4359 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4360 return Tmp;
4361 }
4362
4363 case ISD::BITCAST: {
4364 if (VT.isScalableVector())
4365 break;
4366 SDValue N0 = Op.getOperand(0);
4367 EVT SrcVT = N0.getValueType();
4368 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4369
4370 // Ignore bitcasts from unsupported types..
4371 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4372 break;
4373
4374 // Fast handling of 'identity' bitcasts.
4375 if (VTBits == SrcBits)
4376 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
4377
4378 bool IsLE = getDataLayout().isLittleEndian();
4379
4380 // Bitcast 'large element' scalar/vector to 'small element' vector.
4381 if ((SrcBits % VTBits) == 0) {
4382 assert(VT.isVector() && "Expected bitcast to vector");
4383
4384 unsigned Scale = SrcBits / VTBits;
4385 APInt SrcDemandedElts =
4386 APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale);
4387
4388 // Fast case - sign splat can be simply split across the small elements.
4389 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
4390 if (Tmp == SrcBits)
4391 return VTBits;
4392
4393 // Slow case - determine how far the sign extends into each sub-element.
4394 Tmp2 = VTBits;
4395 for (unsigned i = 0; i != NumElts; ++i)
4396 if (DemandedElts[i]) {
4397 unsigned SubOffset = i % Scale;
4398 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4399 SubOffset = SubOffset * VTBits;
4400 if (Tmp <= SubOffset)
4401 return 1;
4402 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
4403 }
4404 return Tmp2;
4405 }
4406 break;
4407 }
4408
4409 case ISD::FP_TO_SINT_SAT:
4410 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4411 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4412 return VTBits - Tmp + 1;
4413 case ISD::SIGN_EXTEND:
4414 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
4415 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
4416 case ISD::SIGN_EXTEND_INREG:
4417 // Max of the input and what this extends.
4418 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4419 Tmp = VTBits-Tmp+1;
4420 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4421 return std::max(Tmp, Tmp2);
4422 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4423 if (VT.isScalableVector())
4424 break;
4425 SDValue Src = Op.getOperand(0);
4426 EVT SrcVT = Src.getValueType();
4427 APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements());
4428 Tmp = VTBits - SrcVT.getScalarSizeInBits();
4429 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
4430 }
4431 case ISD::SRA:
4432 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4433 // SRA X, C -> adds C sign bits.
4434 if (const APInt *ShAmt =
4435 getValidMinimumShiftAmountConstant(Op, DemandedElts))
4436 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
4437 return Tmp;
4438 case ISD::SHL:
4439 if (const APInt *ShAmt =
4440 getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
4441 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
4442 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4443 if (ShAmt->ult(Tmp))
4444 return Tmp - ShAmt->getZExtValue();
4445 }
4446 break;
4447 case ISD::AND:
4448 case ISD::OR:
4449 case ISD::XOR: // NOT is handled here.
4450 // Logical binary ops preserve the number of sign bits at the worst.
4451 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4452 if (Tmp != 1) {
4453 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4454 FirstAnswer = std::min(Tmp, Tmp2);
4455 // We computed what we know about the sign bits as our first
4456 // answer. Now proceed to the generic code that uses
4457 // computeKnownBits, and pick whichever answer is better.
4458 }
4459 break;
4460
4461 case ISD::SELECT:
4462 case ISD::VSELECT:
4463 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4464 if (Tmp == 1) return 1; // Early out.
4465 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4466 return std::min(Tmp, Tmp2);
4467 case ISD::SELECT_CC:
4468 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4469 if (Tmp == 1) return 1; // Early out.
4470 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
4471 return std::min(Tmp, Tmp2);
4472
4473 case ISD::SMIN:
4474 case ISD::SMAX: {
4475 // If we have a clamp pattern, we know that the number of sign bits will be
4476 // the minimum of the clamp min/max range.
4477 bool IsMax = (Opcode == ISD::SMAX);
4478 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4479 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
4480 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4481 CstHigh =
4482 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
4483 if (CstLow && CstHigh) {
4484 if (!IsMax)
4485 std::swap(CstLow, CstHigh);
4486 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
4487 Tmp = CstLow->getAPIntValue().getNumSignBits();
4488 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4489 return std::min(Tmp, Tmp2);
4490 }
4491 }
4492
4493 // Fallback - just get the minimum number of sign bits of the operands.
4494 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4495 if (Tmp == 1)
4496 return 1; // Early out.
4497 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4498 return std::min(Tmp, Tmp2);
4499 }
4500 case ISD::UMIN:
4501 case ISD::UMAX:
4502 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4503 if (Tmp == 1)
4504 return 1; // Early out.
4505 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4506 return std::min(Tmp, Tmp2);
4507 case ISD::SADDO:
4508 case ISD::UADDO:
4509 case ISD::SADDO_CARRY:
4510 case ISD::UADDO_CARRY:
4511 case ISD::SSUBO:
4512 case ISD::USUBO:
4513 case ISD::SSUBO_CARRY:
4514 case ISD::USUBO_CARRY:
4515 case ISD::SMULO:
4516 case ISD::UMULO:
4517 if (Op.getResNo() != 1)
4518 break;
4519 // The boolean result conforms to getBooleanContents. Fall through.
4520 // If setcc returns 0/-1, all bits are sign bits.
4521 // We know that we have an integer-based boolean since these operations
4522 // are only available for integer.
4523 if (TLI->getBooleanContents(VT.isVector(), false) ==
4524 TargetLowering::ZeroOrNegativeOneBooleanContent)
4525 return VTBits;
4526 break;
4527 case ISD::SETCC:
4528 case ISD::SETCCCARRY:
4529 case ISD::STRICT_FSETCC:
4530 case ISD::STRICT_FSETCCS: {
4531 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
4532 // If setcc returns 0/-1, all bits are sign bits.
4533 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
4534 TargetLowering::ZeroOrNegativeOneBooleanContent)
4535 return VTBits;
4536 break;
4537 }
4538 case ISD::ROTL:
4539 case ISD::ROTR:
4540 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4541
4542 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
4543 if (Tmp == VTBits)
4544 return VTBits;
4545
4546 if (ConstantSDNode *C =
4547 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
4548 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
4549
4550 // Handle rotate right by N like a rotate left by 32-N.
4551 if (Opcode == ISD::ROTR)
4552 RotAmt = (VTBits - RotAmt) % VTBits;
4553
4554 // If we aren't rotating out all of the known-in sign bits, return the
4555 // number that are left. This handles rotl(sext(x), 1) for example.
4556 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
4557 }
4558 break;
4559 case ISD::ADD:
4560 case ISD::ADDC:
4561 // Add can have at most one carry bit. Thus we know that the output
4562 // is, at worst, one more bit than the inputs.
4563 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4564 if (Tmp == 1) return 1; // Early out.
4565
4566 // Special case decrementing a value (ADD X, -1):
4567 if (ConstantSDNode *CRHS =
4568 isConstOrConstSplat(Op.getOperand(1), DemandedElts))
4569 if (CRHS->isAllOnes()) {
4570 KnownBits Known =
4571 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4572
4573 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4574 // sign bits set.
4575 if ((Known.Zero | 1).isAllOnes())
4576 return VTBits;
4577
4578 // If we are subtracting one from a positive number, there is no carry
4579 // out of the result.
4580 if (Known.isNonNegative())
4581 return Tmp;
4582 }
4583
4584 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4585 if (Tmp2 == 1) return 1; // Early out.
4586 return std::min(Tmp, Tmp2) - 1;
4587 case ISD::SUB:
4588 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4589 if (Tmp2 == 1) return 1; // Early out.
4590
4591 // Handle NEG.
4592 if (ConstantSDNode *CLHS =
4593 isConstOrConstSplat(Op.getOperand(0), DemandedElts))
4594 if (CLHS->isZero()) {
4595 KnownBits Known =
4596 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4597 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4598 // sign bits set.
4599 if ((Known.Zero | 1).isAllOnes())
4600 return VTBits;
4601
4602 // If the input is known to be positive (the sign bit is known clear),
4603 // the output of the NEG has the same number of sign bits as the input.
4604 if (Known.isNonNegative())
4605 return Tmp2;
4606
4607 // Otherwise, we treat this like a SUB.
4608 }
4609
4610 // Sub can have at most one carry bit. Thus we know that the output
4611 // is, at worst, one more bit than the inputs.
4612 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4613 if (Tmp == 1) return 1; // Early out.
4614 return std::min(Tmp, Tmp2) - 1;
4615 case ISD::MUL: {
4616 // The output of the Mul can be at most twice the valid bits in the inputs.
4617 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4618 if (SignBitsOp0 == 1)
4619 break;
4620 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
4621 if (SignBitsOp1 == 1)
4622 break;
4623 unsigned OutValidBits =
4624 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4625 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4626 }
4627 case ISD::SREM:
4628 // The sign bit is the LHS's sign bit, except when the result of the
4629 // remainder is zero. The magnitude of the result should be less than or
4630 // equal to the magnitude of the LHS. Therefore, the result should have
4631 // at least as many sign bits as the left hand side.
4632 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4633 case ISD::TRUNCATE: {
4634 // Check if the sign bits of source go down as far as the truncated value.
4635 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
4636 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4637 if (NumSrcSignBits > (NumSrcBits - VTBits))
4638 return NumSrcSignBits - (NumSrcBits - VTBits);
4639 break;
4640 }
4641 case ISD::EXTRACT_ELEMENT: {
4642 if (VT.isScalableVector())
4643 break;
4644 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
4645 const int BitWidth = Op.getValueSizeInBits();
4646 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
4647
4648 // Get reverse index (starting from 1), Op1 value indexes elements from
4649 // little end. Sign starts at big end.
4650 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
4651
4652 // If the sign portion ends in our element the subtraction gives correct
4653 // result. Otherwise it gives either negative or > bitwidth result
4654 return std::clamp(KnownSign - rIndex * BitWidth, 0, BitWidth);
4655 }
4656 case ISD::INSERT_VECTOR_ELT: {
4657 if (VT.isScalableVector())
4658 break;
4659 // If we know the element index, split the demand between the
4660 // source vector and the inserted element, otherwise assume we need
4661 // the original demanded vector elements and the value.
4662 SDValue InVec = Op.getOperand(0);
4663 SDValue InVal = Op.getOperand(1);
4664 SDValue EltNo = Op.getOperand(2);
4665 bool DemandedVal = true;
4666 APInt DemandedVecElts = DemandedElts;
4667 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4668 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4669 unsigned EltIdx = CEltNo->getZExtValue();
4670 DemandedVal = !!DemandedElts[EltIdx];
4671 DemandedVecElts.clearBit(EltIdx);
4672 }
4673 Tmp = std::numeric_limits<unsigned>::max();
4674 if (DemandedVal) {
4675 // TODO - handle implicit truncation of inserted elements.
4676 if (InVal.getScalarValueSizeInBits() != VTBits)
4677 break;
4678 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
4679 Tmp = std::min(Tmp, Tmp2);
4680 }
4681 if (!!DemandedVecElts) {
4682 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
4683 Tmp = std::min(Tmp, Tmp2);
4684 }
4685 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4686 return Tmp;
4687 }
4688 case ISD::EXTRACT_VECTOR_ELT: {
4689 assert(!VT.isScalableVector());
4690 SDValue InVec = Op.getOperand(0);
4691 SDValue EltNo = Op.getOperand(1);
4692 EVT VecVT = InVec.getValueType();
4693 // ComputeNumSignBits not yet implemented for scalable vectors.
4694 if (VecVT.isScalableVector())
4695 break;
4696 const unsigned BitWidth = Op.getValueSizeInBits();
4697 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
4698 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4699
4700 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4701 // anything about sign bits. But if the sizes match we can derive knowledge
4702 // about sign bits from the vector operand.
4703 if (BitWidth != EltBitWidth)
4704 break;
4705
4706 // If we know the element index, just demand that vector element, else for
4707 // an unknown element index, ignore DemandedElts and demand them all.
4708 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
4709 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4710 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4711 DemandedSrcElts =
4712 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4713
4714 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
4715 }
4716 case ISD::EXTRACT_SUBVECTOR: {
4717 // Offset the demanded elts by the subvector index.
4718 SDValue Src = Op.getOperand(0);
4719 // Bail until we can represent demanded elements for scalable vectors.
4720 if (Src.getValueType().isScalableVector())
4721 break;
4722 uint64_t Idx = Op.getConstantOperandVal(1);
4723 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4724 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
4725 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4726 }
4727 case ISD::CONCAT_VECTORS: {
4728 if (VT.isScalableVector())
4729 break;
4730 // Determine the minimum number of sign bits across all demanded
4731 // elts of the input vectors. Early out if the result is already 1.
4732 Tmp = std::numeric_limits<unsigned>::max();
4733 EVT SubVectorVT = Op.getOperand(0).getValueType();
4734 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4735 unsigned NumSubVectors = Op.getNumOperands();
4736 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4737 APInt DemandedSub =
4738 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
4739 if (!DemandedSub)
4740 continue;
4741 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
4742 Tmp = std::min(Tmp, Tmp2);
4743 }
4744 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4745 return Tmp;
4746 }
4747 case ISD::INSERT_SUBVECTOR: {
4748 if (VT.isScalableVector())
4749 break;
4750 // Demand any elements from the subvector and the remainder from the src its
4751 // inserted into.
4752 SDValue Src = Op.getOperand(0);
4753 SDValue Sub = Op.getOperand(1);
4754 uint64_t Idx = Op.getConstantOperandVal(2);
4755 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4756 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
4757 APInt DemandedSrcElts = DemandedElts;
4758 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
4759
4760 Tmp = std::numeric_limits<unsigned>::max();
4761 if (!!DemandedSubElts) {
4762 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
4763 if (Tmp == 1)
4764 return 1; // early-out
4765 }
4766 if (!!DemandedSrcElts) {
4767 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4768 Tmp = std::min(Tmp, Tmp2);
4769 }
4770 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4771 return Tmp;
4772 }
4773 case ISD::LOAD: {
4774 LoadSDNode *LD = cast<LoadSDNode>(Op);
4775 if (const MDNode *Ranges = LD->getRanges()) {
4776 if (DemandedElts != 1)
4777 break;
4778
4779 ConstantRange CR = getConstantRangeFromMetadata(*Ranges);
4780 if (VTBits > CR.getBitWidth()) {
4781 switch (LD->getExtensionType()) {
4782 case ISD::SEXTLOAD:
4783 CR = CR.signExtend(VTBits);
4784 break;
4785 case ISD::ZEXTLOAD:
4786 CR = CR.zeroExtend(VTBits);
4787 break;
4788 default:
4789 break;
4790 }
4791 }
4792
4793 if (VTBits != CR.getBitWidth())
4794 break;
4795 return std::min(CR.getSignedMin().getNumSignBits(),
4796 CR.getSignedMax().getNumSignBits());
4797 }
4798
4799 break;
4800 }
4801 case ISD::ATOMIC_CMP_SWAP:
4802 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4803 case ISD::ATOMIC_SWAP:
4804 case ISD::ATOMIC_LOAD_ADD:
4805 case ISD::ATOMIC_LOAD_SUB:
4806 case ISD::ATOMIC_LOAD_AND:
4807 case ISD::ATOMIC_LOAD_CLR:
4808 case ISD::ATOMIC_LOAD_OR:
4809 case ISD::ATOMIC_LOAD_XOR:
4810 case ISD::ATOMIC_LOAD_NAND:
4811 case ISD::ATOMIC_LOAD_MIN:
4812 case ISD::ATOMIC_LOAD_MAX:
4813 case ISD::ATOMIC_LOAD_UMIN:
4814 case ISD::ATOMIC_LOAD_UMAX:
4815 case ISD::ATOMIC_LOAD: {
4816 Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4817 // If we are looking at the loaded value.
4818 if (Op.getResNo() == 0) {
4819 if (Tmp == VTBits)
4820 return 1; // early-out
4821 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
4822 return VTBits - Tmp + 1;
4823 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4824 return VTBits - Tmp;
4825 }
4826 break;
4827 }
4828 }
4829
4830 // If we are looking at the loaded value of the SDNode.
4831 if (Op.getResNo() == 0) {
4832 // Handle LOADX separately here. EXTLOAD case will fallthrough.
4833 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
4834 unsigned ExtType = LD->getExtensionType();
4835 switch (ExtType) {
4836 default: break;
4837 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4838 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4839 return VTBits - Tmp + 1;
4840 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4841 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4842 return VTBits - Tmp;
4843 case ISD::NON_EXTLOAD:
4844 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4845 // We only need to handle vectors - computeKnownBits should handle
4846 // scalar cases.
4847 Type *CstTy = Cst->getType();
4848 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
4849 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
4850 VTBits == CstTy->getScalarSizeInBits()) {
4851 Tmp = VTBits;
4852 for (unsigned i = 0; i != NumElts; ++i) {
4853 if (!DemandedElts[i])
4854 continue;
4855 if (Constant *Elt = Cst->getAggregateElement(i)) {
4856 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4857 const APInt &Value = CInt->getValue();
4858 Tmp = std::min(Tmp, Value.getNumSignBits());
4859 continue;
4860 }
4861 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4862 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4863 Tmp = std::min(Tmp, Value.getNumSignBits());
4864 continue;
4865 }
4866 }
4867 // Unknown type. Conservatively assume no bits match sign bit.
4868 return 1;
4869 }
4870 return Tmp;
4871 }
4872 }
4873 break;
4874 }
4875 }
4876 }
4877
4878 // Allow the target to implement this method for its nodes.
4879 if (Opcode >= ISD::BUILTIN_OP_END ||
4880 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4881 Opcode == ISD::INTRINSIC_W_CHAIN ||
4882 Opcode == ISD::INTRINSIC_VOID) {
4883 // TODO: This can probably be removed once target code is audited. This
4884 // is here purely to reduce patch size and review complexity.
4885 if (!VT.isScalableVector()) {
4886 unsigned NumBits =
4887 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
4888 if (NumBits > 1)
4889 FirstAnswer = std::max(FirstAnswer, NumBits);
4890 }
4891 }
4892
4893 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4894 // use this information.
4895 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4896 return std::max(FirstAnswer, Known.countMinSignBits());
4897 }
4898
ComputeMaxSignificantBits(SDValue Op,unsigned Depth) const4899 unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
4900 unsigned Depth) const {
4901 unsigned SignBits = ComputeNumSignBits(Op, Depth);
4902 return Op.getScalarValueSizeInBits() - SignBits + 1;
4903 }
4904
ComputeMaxSignificantBits(SDValue Op,const APInt & DemandedElts,unsigned Depth) const4905 unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
4906 const APInt &DemandedElts,
4907 unsigned Depth) const {
4908 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
4909 return Op.getScalarValueSizeInBits() - SignBits + 1;
4910 }
4911
isGuaranteedNotToBeUndefOrPoison(SDValue Op,bool PoisonOnly,unsigned Depth) const4912 bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
4913 unsigned Depth) const {
4914 // Early out for FREEZE.
4915 if (Op.getOpcode() == ISD::FREEZE)
4916 return true;
4917
4918 // TODO: Assume we don't know anything for now.
4919 EVT VT = Op.getValueType();
4920 if (VT.isScalableVector())
4921 return false;
4922
4923 APInt DemandedElts = VT.isVector()
4924 ? APInt::getAllOnes(VT.getVectorNumElements())
4925 : APInt(1, 1);
4926 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
4927 }
4928
isGuaranteedNotToBeUndefOrPoison(SDValue Op,const APInt & DemandedElts,bool PoisonOnly,unsigned Depth) const4929 bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
4930 const APInt &DemandedElts,
4931 bool PoisonOnly,
4932 unsigned Depth) const {
4933 unsigned Opcode = Op.getOpcode();
4934
4935 // Early out for FREEZE.
4936 if (Opcode == ISD::FREEZE)
4937 return true;
4938
4939 if (Depth >= MaxRecursionDepth)
4940 return false; // Limit search depth.
4941
4942 if (isIntOrFPConstant(Op))
4943 return true;
4944
4945 switch (Opcode) {
4946 case ISD::VALUETYPE:
4947 case ISD::FrameIndex:
4948 case ISD::TargetFrameIndex:
4949 return true;
4950
4951 case ISD::UNDEF:
4952 return PoisonOnly;
4953
4954 case ISD::BUILD_VECTOR:
4955 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
4956 // this shouldn't affect the result.
4957 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
4958 if (!DemandedElts[i])
4959 continue;
4960 if (!isGuaranteedNotToBeUndefOrPoison(Op.getOperand(i), PoisonOnly,
4961 Depth + 1))
4962 return false;
4963 }
4964 return true;
4965
4966 // TODO: Search for noundef attributes from library functions.
4967
4968 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
4969
4970 default:
4971 // Allow the target to implement this method for its nodes.
4972 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
4973 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
4974 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4975 Op, DemandedElts, *this, PoisonOnly, Depth);
4976 break;
4977 }
4978
4979 // If Op can't create undef/poison and none of its operands are undef/poison
4980 // then Op is never undef/poison.
4981 // NOTE: TargetNodes should handle this in themselves in
4982 // isGuaranteedNotToBeUndefOrPoisonForTargetNode.
4983 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
4984 Depth) &&
4985 all_of(Op->ops(), [&](SDValue V) {
4986 return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1);
4987 });
4988 }
4989
canCreateUndefOrPoison(SDValue Op,bool PoisonOnly,bool ConsiderFlags,unsigned Depth) const4990 bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
4991 bool ConsiderFlags,
4992 unsigned Depth) const {
4993 // TODO: Assume we don't know anything for now.
4994 EVT VT = Op.getValueType();
4995 if (VT.isScalableVector())
4996 return true;
4997
4998 APInt DemandedElts = VT.isVector()
4999 ? APInt::getAllOnes(VT.getVectorNumElements())
5000 : APInt(1, 1);
5001 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5002 Depth);
5003 }
5004
canCreateUndefOrPoison(SDValue Op,const APInt & DemandedElts,bool PoisonOnly,bool ConsiderFlags,unsigned Depth) const5005 bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5006 bool PoisonOnly, bool ConsiderFlags,
5007 unsigned Depth) const {
5008 // TODO: Assume we don't know anything for now.
5009 EVT VT = Op.getValueType();
5010 if (VT.isScalableVector())
5011 return true;
5012
5013 unsigned Opcode = Op.getOpcode();
5014 switch (Opcode) {
5015 case ISD::FREEZE:
5016 case ISD::CONCAT_VECTORS:
5017 case ISD::INSERT_SUBVECTOR:
5018 case ISD::AND:
5019 case ISD::XOR:
5020 case ISD::ROTL:
5021 case ISD::ROTR:
5022 case ISD::FSHL:
5023 case ISD::FSHR:
5024 case ISD::BSWAP:
5025 case ISD::CTPOP:
5026 case ISD::BITREVERSE:
5027 case ISD::PARITY:
5028 case ISD::SIGN_EXTEND:
5029 case ISD::TRUNCATE:
5030 case ISD::SIGN_EXTEND_INREG:
5031 case ISD::SIGN_EXTEND_VECTOR_INREG:
5032 case ISD::ZERO_EXTEND_VECTOR_INREG:
5033 case ISD::BITCAST:
5034 case ISD::BUILD_VECTOR:
5035 case ISD::BUILD_PAIR:
5036 return false;
5037
5038 // Matches hasPoisonGeneratingFlags().
5039 case ISD::ZERO_EXTEND:
5040 return ConsiderFlags && Op->getFlags().hasNonNeg();
5041
5042 case ISD::ADD:
5043 case ISD::SUB:
5044 case ISD::MUL:
5045 // Matches hasPoisonGeneratingFlags().
5046 return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5047 Op->getFlags().hasNoUnsignedWrap());
5048
5049 case ISD::SHL:
5050 // If the max shift amount isn't in range, then the shift can create poison.
5051 if (!getValidMaximumShiftAmountConstant(Op, DemandedElts))
5052 return true;
5053
5054 // Matches hasPoisonGeneratingFlags().
5055 return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5056 Op->getFlags().hasNoUnsignedWrap());
5057
5058 // Matches hasPoisonGeneratingFlags().
5059 case ISD::OR:
5060 return ConsiderFlags && Op->getFlags().hasDisjoint();
5061
5062 case ISD::INSERT_VECTOR_ELT:{
5063 // Ensure that the element index is in bounds.
5064 EVT VecVT = Op.getOperand(0).getValueType();
5065 KnownBits KnownIdx = computeKnownBits(Op.getOperand(2), Depth + 1);
5066 return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
5067 }
5068
5069 default:
5070 // Allow the target to implement this method for its nodes.
5071 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5072 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5073 return TLI->canCreateUndefOrPoisonForTargetNode(
5074 Op, DemandedElts, *this, PoisonOnly, ConsiderFlags, Depth);
5075 break;
5076 }
5077
5078 // Be conservative and return true.
5079 return true;
5080 }
5081
isADDLike(SDValue Op) const5082 bool SelectionDAG::isADDLike(SDValue Op) const {
5083 unsigned Opcode = Op.getOpcode();
5084 if (Opcode == ISD::OR)
5085 return Op->getFlags().hasDisjoint() ||
5086 haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1));
5087 if (Opcode == ISD::XOR)
5088 return isMinSignedConstant(Op.getOperand(1));
5089 return false;
5090 }
5091
isBaseWithConstantOffset(SDValue Op) const5092 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
5093 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
5094 !isa<ConstantSDNode>(Op.getOperand(1)))
5095 return false;
5096
5097 if (Op.getOpcode() == ISD::OR &&
5098 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
5099 return false;
5100
5101 return true;
5102 }
5103
isKnownNeverNaN(SDValue Op,bool SNaN,unsigned Depth) const5104 bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
5105 // If we're told that NaNs won't happen, assume they won't.
5106 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5107 return true;
5108
5109 if (Depth >= MaxRecursionDepth)
5110 return false; // Limit search depth.
5111
5112 // If the value is a constant, we can obviously see if it is a NaN or not.
5113 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
5114 return !C->getValueAPF().isNaN() ||
5115 (SNaN && !C->getValueAPF().isSignaling());
5116 }
5117
5118 unsigned Opcode = Op.getOpcode();
5119 switch (Opcode) {
5120 case ISD::FADD:
5121 case ISD::FSUB:
5122 case ISD::FMUL:
5123 case ISD::FDIV:
5124 case ISD::FREM:
5125 case ISD::FSIN:
5126 case ISD::FCOS:
5127 case ISD::FMA:
5128 case ISD::FMAD: {
5129 if (SNaN)
5130 return true;
5131 // TODO: Need isKnownNeverInfinity
5132 return false;
5133 }
5134 case ISD::FCANONICALIZE:
5135 case ISD::FEXP:
5136 case ISD::FEXP2:
5137 case ISD::FEXP10:
5138 case ISD::FTRUNC:
5139 case ISD::FFLOOR:
5140 case ISD::FCEIL:
5141 case ISD::FROUND:
5142 case ISD::FROUNDEVEN:
5143 case ISD::FRINT:
5144 case ISD::LRINT:
5145 case ISD::LLRINT:
5146 case ISD::FNEARBYINT:
5147 case ISD::FLDEXP: {
5148 if (SNaN)
5149 return true;
5150 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5151 }
5152 case ISD::FABS:
5153 case ISD::FNEG:
5154 case ISD::FCOPYSIGN: {
5155 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5156 }
5157 case ISD::SELECT:
5158 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
5159 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
5160 case ISD::FP_EXTEND:
5161 case ISD::FP_ROUND: {
5162 if (SNaN)
5163 return true;
5164 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5165 }
5166 case ISD::SINT_TO_FP:
5167 case ISD::UINT_TO_FP:
5168 return true;
5169 case ISD::FSQRT: // Need is known positive
5170 case ISD::FLOG:
5171 case ISD::FLOG2:
5172 case ISD::FLOG10:
5173 case ISD::FPOWI:
5174 case ISD::FPOW: {
5175 if (SNaN)
5176 return true;
5177 // TODO: Refine on operand
5178 return false;
5179 }
5180 case ISD::FMINNUM:
5181 case ISD::FMAXNUM: {
5182 // Only one needs to be known not-nan, since it will be returned if the
5183 // other ends up being one.
5184 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
5185 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5186 }
5187 case ISD::FMINNUM_IEEE:
5188 case ISD::FMAXNUM_IEEE: {
5189 if (SNaN)
5190 return true;
5191 // This can return a NaN if either operand is an sNaN, or if both operands
5192 // are NaN.
5193 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
5194 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
5195 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
5196 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
5197 }
5198 case ISD::FMINIMUM:
5199 case ISD::FMAXIMUM: {
5200 // TODO: Does this quiet or return the origina NaN as-is?
5201 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
5202 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5203 }
5204 case ISD::EXTRACT_VECTOR_ELT: {
5205 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5206 }
5207 case ISD::BUILD_VECTOR: {
5208 for (const SDValue &Opnd : Op->ops())
5209 if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1))
5210 return false;
5211 return true;
5212 }
5213 default:
5214 if (Opcode >= ISD::BUILTIN_OP_END ||
5215 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5216 Opcode == ISD::INTRINSIC_W_CHAIN ||
5217 Opcode == ISD::INTRINSIC_VOID) {
5218 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
5219 }
5220
5221 return false;
5222 }
5223 }
5224
isKnownNeverZeroFloat(SDValue Op) const5225 bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
5226 assert(Op.getValueType().isFloatingPoint() &&
5227 "Floating point type expected");
5228
5229 // If the value is a constant, we can obviously see if it is a zero or not.
5230 return ISD::matchUnaryFpPredicate(
5231 Op, [](ConstantFPSDNode *C) { return !C->isZero(); });
5232 }
5233
isKnownNeverZero(SDValue Op,unsigned Depth) const5234 bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
5235 if (Depth >= MaxRecursionDepth)
5236 return false; // Limit search depth.
5237
5238 assert(!Op.getValueType().isFloatingPoint() &&
5239 "Floating point types unsupported - use isKnownNeverZeroFloat");
5240
5241 // If the value is a constant, we can obviously see if it is a zero or not.
5242 if (ISD::matchUnaryPredicate(Op,
5243 [](ConstantSDNode *C) { return !C->isZero(); }))
5244 return true;
5245
5246 // TODO: Recognize more cases here. Most of the cases are also incomplete to
5247 // some degree.
5248 switch (Op.getOpcode()) {
5249 default:
5250 break;
5251
5252 case ISD::OR:
5253 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5254 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5255
5256 case ISD::VSELECT:
5257 case ISD::SELECT:
5258 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5259 isKnownNeverZero(Op.getOperand(2), Depth + 1);
5260
5261 case ISD::SHL: {
5262 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5263 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5264 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5265 // 1 << X is never zero.
5266 if (ValKnown.One[0])
5267 return true;
5268 // If max shift cnt of known ones is non-zero, result is non-zero.
5269 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5270 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5271 !ValKnown.One.shl(MaxCnt).isZero())
5272 return true;
5273 break;
5274 }
5275 case ISD::UADDSAT:
5276 case ISD::UMAX:
5277 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5278 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5279
5280 // TODO for smin/smax: If either operand is known negative/positive
5281 // respectively we don't need the other to be known at all.
5282 case ISD::SMAX:
5283 case ISD::SMIN:
5284 case ISD::UMIN:
5285 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5286 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5287
5288 case ISD::ROTL:
5289 case ISD::ROTR:
5290 case ISD::BITREVERSE:
5291 case ISD::BSWAP:
5292 case ISD::CTPOP:
5293 case ISD::ABS:
5294 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5295
5296 case ISD::SRA:
5297 case ISD::SRL: {
5298 if (Op->getFlags().hasExact())
5299 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5300 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5301 if (ValKnown.isNegative())
5302 return true;
5303 // If max shift cnt of known ones is non-zero, result is non-zero.
5304 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5305 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5306 !ValKnown.One.lshr(MaxCnt).isZero())
5307 return true;
5308 break;
5309 }
5310 case ISD::UDIV:
5311 case ISD::SDIV:
5312 // div exact can only produce a zero if the dividend is zero.
5313 // TODO: For udiv this is also true if Op1 u<= Op0
5314 if (Op->getFlags().hasExact())
5315 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5316 break;
5317
5318 case ISD::ADD:
5319 if (Op->getFlags().hasNoUnsignedWrap())
5320 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5321 isKnownNeverZero(Op.getOperand(0), Depth + 1))
5322 return true;
5323 // TODO: There are a lot more cases we can prove for add.
5324 break;
5325
5326 case ISD::SUB: {
5327 if (isNullConstant(Op.getOperand(0)))
5328 return isKnownNeverZero(Op.getOperand(1), Depth + 1);
5329
5330 std::optional<bool> ne =
5331 KnownBits::ne(computeKnownBits(Op.getOperand(0), Depth + 1),
5332 computeKnownBits(Op.getOperand(1), Depth + 1));
5333 return ne && *ne;
5334 }
5335
5336 case ISD::MUL:
5337 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5338 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5339 isKnownNeverZero(Op.getOperand(0), Depth + 1))
5340 return true;
5341 break;
5342
5343 case ISD::ZERO_EXTEND:
5344 case ISD::SIGN_EXTEND:
5345 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5346 }
5347
5348 return computeKnownBits(Op, Depth).isNonZero();
5349 }
5350
isEqualTo(SDValue A,SDValue B) const5351 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
5352 // Check the obvious case.
5353 if (A == B) return true;
5354
5355 // For negative and positive zero.
5356 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
5357 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
5358 if (CA->isZero() && CB->isZero()) return true;
5359
5360 // Otherwise they may not be equal.
5361 return false;
5362 }
5363
5364 // Only bits set in Mask must be negated, other bits may be arbitrary.
getBitwiseNotOperand(SDValue V,SDValue Mask,bool AllowUndefs)5365 SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
5366 if (isBitwiseNot(V, AllowUndefs))
5367 return V.getOperand(0);
5368
5369 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
5370 // bits in the non-extended part.
5371 ConstantSDNode *MaskC = isConstOrConstSplat(Mask);
5372 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
5373 return SDValue();
5374 SDValue ExtArg = V.getOperand(0);
5375 if (ExtArg.getScalarValueSizeInBits() >=
5376 MaskC->getAPIntValue().getActiveBits() &&
5377 isBitwiseNot(ExtArg, AllowUndefs) &&
5378 ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE &&
5379 ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType())
5380 return ExtArg.getOperand(0).getOperand(0);
5381 return SDValue();
5382 }
5383
haveNoCommonBitsSetCommutative(SDValue A,SDValue B)5384 static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
5385 // Match masked merge pattern (X & ~M) op (Y & M)
5386 // Including degenerate case (X & ~M) op M
5387 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
5388 SDValue Other) {
5389 if (SDValue NotOperand =
5390 getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) {
5391 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
5392 NotOperand->getOpcode() == ISD::TRUNCATE)
5393 NotOperand = NotOperand->getOperand(0);
5394
5395 if (Other == NotOperand)
5396 return true;
5397 if (Other->getOpcode() == ISD::AND)
5398 return NotOperand == Other->getOperand(0) ||
5399 NotOperand == Other->getOperand(1);
5400 }
5401 return false;
5402 };
5403
5404 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
5405 A = A->getOperand(0);
5406
5407 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
5408 B = B->getOperand(0);
5409
5410 if (A->getOpcode() == ISD::AND)
5411 return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) ||
5412 MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B);
5413 return false;
5414 }
5415
5416 // FIXME: unify with llvm::haveNoCommonBitsSet.
haveNoCommonBitsSet(SDValue A,SDValue B) const5417 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
5418 assert(A.getValueType() == B.getValueType() &&
5419 "Values must have the same type");
5420 if (haveNoCommonBitsSetCommutative(A, B) ||
5421 haveNoCommonBitsSetCommutative(B, A))
5422 return true;
5423 return KnownBits::haveNoCommonBitsSet(computeKnownBits(A),
5424 computeKnownBits(B));
5425 }
5426
FoldSTEP_VECTOR(const SDLoc & DL,EVT VT,SDValue Step,SelectionDAG & DAG)5427 static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
5428 SelectionDAG &DAG) {
5429 if (cast<ConstantSDNode>(Step)->isZero())
5430 return DAG.getConstant(0, DL, VT);
5431
5432 return SDValue();
5433 }
5434
FoldBUILD_VECTOR(const SDLoc & DL,EVT VT,ArrayRef<SDValue> Ops,SelectionDAG & DAG)5435 static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
5436 ArrayRef<SDValue> Ops,
5437 SelectionDAG &DAG) {
5438 int NumOps = Ops.size();
5439 assert(NumOps != 0 && "Can't build an empty vector!");
5440 assert(!VT.isScalableVector() &&
5441 "BUILD_VECTOR cannot be used with scalable types");
5442 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
5443 "Incorrect element count in BUILD_VECTOR!");
5444
5445 // BUILD_VECTOR of UNDEFs is UNDEF.
5446 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
5447 return DAG.getUNDEF(VT);
5448
5449 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
5450 SDValue IdentitySrc;
5451 bool IsIdentity = true;
5452 for (int i = 0; i != NumOps; ++i) {
5453 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5454 Ops[i].getOperand(0).getValueType() != VT ||
5455 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
5456 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
5457 Ops[i].getConstantOperandAPInt(1) != i) {
5458 IsIdentity = false;
5459 break;
5460 }
5461 IdentitySrc = Ops[i].getOperand(0);
5462 }
5463 if (IsIdentity)
5464 return IdentitySrc;
5465
5466 return SDValue();
5467 }
5468
5469 /// Try to simplify vector concatenation to an input value, undef, or build
5470 /// vector.
foldCONCAT_VECTORS(const SDLoc & DL,EVT VT,ArrayRef<SDValue> Ops,SelectionDAG & DAG)5471 static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
5472 ArrayRef<SDValue> Ops,
5473 SelectionDAG &DAG) {
5474 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
5475 assert(llvm::all_of(Ops,
5476 [Ops](SDValue Op) {
5477 return Ops[0].getValueType() == Op.getValueType();
5478 }) &&
5479 "Concatenation of vectors with inconsistent value types!");
5480 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
5481 VT.getVectorElementCount() &&
5482 "Incorrect element count in vector concatenation!");
5483
5484 if (Ops.size() == 1)
5485 return Ops[0];
5486
5487 // Concat of UNDEFs is UNDEF.
5488 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
5489 return DAG.getUNDEF(VT);
5490
5491 // Scan the operands and look for extract operations from a single source
5492 // that correspond to insertion at the same location via this concatenation:
5493 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
5494 SDValue IdentitySrc;
5495 bool IsIdentity = true;
5496 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
5497 SDValue Op = Ops[i];
5498 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
5499 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5500 Op.getOperand(0).getValueType() != VT ||
5501 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
5502 Op.getConstantOperandVal(1) != IdentityIndex) {
5503 IsIdentity = false;
5504 break;
5505 }
5506 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
5507 "Unexpected identity source vector for concat of extracts");
5508 IdentitySrc = Op.getOperand(0);
5509 }
5510 if (IsIdentity) {
5511 assert(IdentitySrc && "Failed to set source vector of extracts");
5512 return IdentitySrc;
5513 }
5514
5515 // The code below this point is only designed to work for fixed width
5516 // vectors, so we bail out for now.
5517 if (VT.isScalableVector())
5518 return SDValue();
5519
5520 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
5521 // simplified to one big BUILD_VECTOR.
5522 // FIXME: Add support for SCALAR_TO_VECTOR as well.
5523 EVT SVT = VT.getScalarType();
5524 SmallVector<SDValue, 16> Elts;
5525 for (SDValue Op : Ops) {
5526 EVT OpVT = Op.getValueType();
5527 if (Op.isUndef())
5528 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
5529 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
5530 Elts.append(Op->op_begin(), Op->op_end());
5531 else
5532 return SDValue();
5533 }
5534
5535 // BUILD_VECTOR requires all inputs to be of the same type, find the
5536 // maximum type and extend them all.
5537 for (SDValue Op : Elts)
5538 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
5539
5540 if (SVT.bitsGT(VT.getScalarType())) {
5541 for (SDValue &Op : Elts) {
5542 if (Op.isUndef())
5543 Op = DAG.getUNDEF(SVT);
5544 else
5545 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
5546 ? DAG.getZExtOrTrunc(Op, DL, SVT)
5547 : DAG.getSExtOrTrunc(Op, DL, SVT);
5548 }
5549 }
5550
5551 SDValue V = DAG.getBuildVector(VT, DL, Elts);
5552 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
5553 return V;
5554 }
5555
5556 /// Gets or creates the specified node.
getNode(unsigned Opcode,const SDLoc & DL,EVT VT)5557 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
5558 FoldingSetNodeID ID;
5559 AddNodeIDNode(ID, Opcode, getVTList(VT), std::nullopt);
5560 void *IP = nullptr;
5561 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
5562 return SDValue(E, 0);
5563
5564 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5565 getVTList(VT));
5566 CSEMap.InsertNode(N, IP);
5567
5568 InsertNode(N);
5569 SDValue V = SDValue(N, 0);
5570 NewSDValueDbgMsg(V, "Creating new node: ", this);
5571 return V;
5572 }
5573
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,SDValue N1)5574 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5575 SDValue N1) {
5576 SDNodeFlags Flags;
5577 if (Inserter)
5578 Flags = Inserter->getFlags();
5579 return getNode(Opcode, DL, VT, N1, Flags);
5580 }
5581
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,SDValue N1,const SDNodeFlags Flags)5582 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5583 SDValue N1, const SDNodeFlags Flags) {
5584 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
5585
5586 // Constant fold unary operations with a vector integer or float operand.
5587 switch (Opcode) {
5588 default:
5589 // FIXME: Entirely reasonable to perform folding of other unary
5590 // operations here as the need arises.
5591 break;
5592 case ISD::FNEG:
5593 case ISD::FABS:
5594 case ISD::FCEIL:
5595 case ISD::FTRUNC:
5596 case ISD::FFLOOR:
5597 case ISD::FP_EXTEND:
5598 case ISD::FP_TO_SINT:
5599 case ISD::FP_TO_UINT:
5600 case ISD::FP_TO_FP16:
5601 case ISD::FP_TO_BF16:
5602 case ISD::TRUNCATE:
5603 case ISD::ANY_EXTEND:
5604 case ISD::ZERO_EXTEND:
5605 case ISD::SIGN_EXTEND:
5606 case ISD::UINT_TO_FP:
5607 case ISD::SINT_TO_FP:
5608 case ISD::FP16_TO_FP:
5609 case ISD::BF16_TO_FP:
5610 case ISD::BITCAST:
5611 case ISD::ABS:
5612 case ISD::BITREVERSE:
5613 case ISD::BSWAP:
5614 case ISD::CTLZ:
5615 case ISD::CTLZ_ZERO_UNDEF:
5616 case ISD::CTTZ:
5617 case ISD::CTTZ_ZERO_UNDEF:
5618 case ISD::CTPOP:
5619 case ISD::STEP_VECTOR: {
5620 SDValue Ops = {N1};
5621 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
5622 return Fold;
5623 }
5624 }
5625
5626 unsigned OpOpcode = N1.getNode()->getOpcode();
5627 switch (Opcode) {
5628 case ISD::STEP_VECTOR:
5629 assert(VT.isScalableVector() &&
5630 "STEP_VECTOR can only be used with scalable types");
5631 assert(OpOpcode == ISD::TargetConstant &&
5632 VT.getVectorElementType() == N1.getValueType() &&
5633 "Unexpected step operand");
5634 break;
5635 case ISD::FREEZE:
5636 assert(VT == N1.getValueType() && "Unexpected VT!");
5637 if (isGuaranteedNotToBeUndefOrPoison(N1, /*PoisonOnly*/ false,
5638 /*Depth*/ 1))
5639 return N1;
5640 break;
5641 case ISD::TokenFactor:
5642 case ISD::MERGE_VALUES:
5643 case ISD::CONCAT_VECTORS:
5644 return N1; // Factor, merge or concat of one node? No need.
5645 case ISD::BUILD_VECTOR: {
5646 // Attempt to simplify BUILD_VECTOR.
5647 SDValue Ops[] = {N1};
5648 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5649 return V;
5650 break;
5651 }
5652 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
5653 case ISD::FP_EXTEND:
5654 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
5655 "Invalid FP cast!");
5656 if (N1.getValueType() == VT) return N1; // noop conversion.
5657 assert((!VT.isVector() || VT.getVectorElementCount() ==
5658 N1.getValueType().getVectorElementCount()) &&
5659 "Vector element count mismatch!");
5660 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
5661 if (N1.isUndef())
5662 return getUNDEF(VT);
5663 break;
5664 case ISD::FP_TO_SINT:
5665 case ISD::FP_TO_UINT:
5666 if (N1.isUndef())
5667 return getUNDEF(VT);
5668 break;
5669 case ISD::SINT_TO_FP:
5670 case ISD::UINT_TO_FP:
5671 // [us]itofp(undef) = 0, because the result value is bounded.
5672 if (N1.isUndef())
5673 return getConstantFP(0.0, DL, VT);
5674 break;
5675 case ISD::SIGN_EXTEND:
5676 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5677 "Invalid SIGN_EXTEND!");
5678 assert(VT.isVector() == N1.getValueType().isVector() &&
5679 "SIGN_EXTEND result type type should be vector iff the operand "
5680 "type is vector!");
5681 if (N1.getValueType() == VT) return N1; // noop extension
5682 assert((!VT.isVector() || VT.getVectorElementCount() ==
5683 N1.getValueType().getVectorElementCount()) &&
5684 "Vector element count mismatch!");
5685 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
5686 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
5687 return getNode(OpOpcode, DL, VT, N1.getOperand(0));
5688 if (OpOpcode == ISD::UNDEF)
5689 // sext(undef) = 0, because the top bits will all be the same.
5690 return getConstant(0, DL, VT);
5691 break;
5692 case ISD::ZERO_EXTEND:
5693 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5694 "Invalid ZERO_EXTEND!");
5695 assert(VT.isVector() == N1.getValueType().isVector() &&
5696 "ZERO_EXTEND result type type should be vector iff the operand "
5697 "type is vector!");
5698 if (N1.getValueType() == VT) return N1; // noop extension
5699 assert((!VT.isVector() || VT.getVectorElementCount() ==
5700 N1.getValueType().getVectorElementCount()) &&
5701 "Vector element count mismatch!");
5702 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
5703 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
5704 return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0));
5705 if (OpOpcode == ISD::UNDEF)
5706 // zext(undef) = 0, because the top bits will be zero.
5707 return getConstant(0, DL, VT);
5708
5709 // Skip unnecessary zext_inreg pattern:
5710 // (zext (trunc x)) -> x iff the upper bits are known zero.
5711 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
5712 // use to recognise zext_inreg patterns.
5713 if (OpOpcode == ISD::TRUNCATE) {
5714 SDValue OpOp = N1.getOperand(0);
5715 if (OpOp.getValueType() == VT) {
5716 if (OpOp.getOpcode() != ISD::AND) {
5717 APInt HiBits = APInt::getBitsSetFrom(VT.getScalarSizeInBits(),
5718 N1.getScalarValueSizeInBits());
5719 if (MaskedValueIsZero(OpOp, HiBits)) {
5720 transferDbgValues(N1, OpOp);
5721 return OpOp;
5722 }
5723 }
5724 }
5725 }
5726 break;
5727 case ISD::ANY_EXTEND:
5728 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5729 "Invalid ANY_EXTEND!");
5730 assert(VT.isVector() == N1.getValueType().isVector() &&
5731 "ANY_EXTEND result type type should be vector iff the operand "
5732 "type is vector!");
5733 if (N1.getValueType() == VT) return N1; // noop extension
5734 assert((!VT.isVector() || VT.getVectorElementCount() ==
5735 N1.getValueType().getVectorElementCount()) &&
5736 "Vector element count mismatch!");
5737 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
5738
5739 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5740 OpOpcode == ISD::ANY_EXTEND)
5741 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
5742 return getNode(OpOpcode, DL, VT, N1.getOperand(0));
5743 if (OpOpcode == ISD::UNDEF)
5744 return getUNDEF(VT);
5745
5746 // (ext (trunc x)) -> x
5747 if (OpOpcode == ISD::TRUNCATE) {
5748 SDValue OpOp = N1.getOperand(0);
5749 if (OpOp.getValueType() == VT) {
5750 transferDbgValues(N1, OpOp);
5751 return OpOp;
5752 }
5753 }
5754 break;
5755 case ISD::TRUNCATE:
5756 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5757 "Invalid TRUNCATE!");
5758 assert(VT.isVector() == N1.getValueType().isVector() &&
5759 "TRUNCATE result type type should be vector iff the operand "
5760 "type is vector!");
5761 if (N1.getValueType() == VT) return N1; // noop truncate
5762 assert((!VT.isVector() || VT.getVectorElementCount() ==
5763 N1.getValueType().getVectorElementCount()) &&
5764 "Vector element count mismatch!");
5765 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
5766 if (OpOpcode == ISD::TRUNCATE)
5767 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
5768 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5769 OpOpcode == ISD::ANY_EXTEND) {
5770 // If the source is smaller than the dest, we still need an extend.
5771 if (N1.getOperand(0).getValueType().getScalarType().bitsLT(
5772 VT.getScalarType()))
5773 return getNode(OpOpcode, DL, VT, N1.getOperand(0));
5774 if (N1.getOperand(0).getValueType().bitsGT(VT))
5775 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
5776 return N1.getOperand(0);
5777 }
5778 if (OpOpcode == ISD::UNDEF)
5779 return getUNDEF(VT);
5780 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
5781 return getVScale(DL, VT,
5782 N1.getConstantOperandAPInt(0).trunc(VT.getSizeInBits()));
5783 break;
5784 case ISD::ANY_EXTEND_VECTOR_INREG:
5785 case ISD::ZERO_EXTEND_VECTOR_INREG:
5786 case ISD::SIGN_EXTEND_VECTOR_INREG:
5787 assert(VT.isVector() && "This DAG node is restricted to vector types.");
5788 assert(N1.getValueType().bitsLE(VT) &&
5789 "The input must be the same size or smaller than the result.");
5790 assert(VT.getVectorMinNumElements() <
5791 N1.getValueType().getVectorMinNumElements() &&
5792 "The destination vector type must have fewer lanes than the input.");
5793 break;
5794 case ISD::ABS:
5795 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
5796 if (OpOpcode == ISD::UNDEF)
5797 return getConstant(0, DL, VT);
5798 break;
5799 case ISD::BSWAP:
5800 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
5801 assert((VT.getScalarSizeInBits() % 16 == 0) &&
5802 "BSWAP types must be a multiple of 16 bits!");
5803 if (OpOpcode == ISD::UNDEF)
5804 return getUNDEF(VT);
5805 // bswap(bswap(X)) -> X.
5806 if (OpOpcode == ISD::BSWAP)
5807 return N1.getOperand(0);
5808 break;
5809 case ISD::BITREVERSE:
5810 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
5811 if (OpOpcode == ISD::UNDEF)
5812 return getUNDEF(VT);
5813 break;
5814 case ISD::BITCAST:
5815 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
5816 "Cannot BITCAST between types of different sizes!");
5817 if (VT == N1.getValueType()) return N1; // noop conversion.
5818 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5819 return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0));
5820 if (OpOpcode == ISD::UNDEF)
5821 return getUNDEF(VT);
5822 break;
5823 case ISD::SCALAR_TO_VECTOR:
5824 assert(VT.isVector() && !N1.getValueType().isVector() &&
5825 (VT.getVectorElementType() == N1.getValueType() ||
5826 (VT.getVectorElementType().isInteger() &&
5827 N1.getValueType().isInteger() &&
5828 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
5829 "Illegal SCALAR_TO_VECTOR node!");
5830 if (OpOpcode == ISD::UNDEF)
5831 return getUNDEF(VT);
5832 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
5833 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
5834 isa<ConstantSDNode>(N1.getOperand(1)) &&
5835 N1.getConstantOperandVal(1) == 0 &&
5836 N1.getOperand(0).getValueType() == VT)
5837 return N1.getOperand(0);
5838 break;
5839 case ISD::FNEG:
5840 // Negation of an unknown bag of bits is still completely undefined.
5841 if (OpOpcode == ISD::UNDEF)
5842 return getUNDEF(VT);
5843
5844 if (OpOpcode == ISD::FNEG) // --X -> X
5845 return N1.getOperand(0);
5846 break;
5847 case ISD::FABS:
5848 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
5849 return getNode(ISD::FABS, DL, VT, N1.getOperand(0));
5850 break;
5851 case ISD::VSCALE:
5852 assert(VT == N1.getValueType() && "Unexpected VT!");
5853 break;
5854 case ISD::CTPOP:
5855 if (N1.getValueType().getScalarType() == MVT::i1)
5856 return N1;
5857 break;
5858 case ISD::CTLZ:
5859 case ISD::CTTZ:
5860 if (N1.getValueType().getScalarType() == MVT::i1)
5861 return getNOT(DL, N1, N1.getValueType());
5862 break;
5863 case ISD::VECREDUCE_ADD:
5864 if (N1.getValueType().getScalarType() == MVT::i1)
5865 return getNode(ISD::VECREDUCE_XOR, DL, VT, N1);
5866 break;
5867 case ISD::VECREDUCE_SMIN:
5868 case ISD::VECREDUCE_UMAX:
5869 if (N1.getValueType().getScalarType() == MVT::i1)
5870 return getNode(ISD::VECREDUCE_OR, DL, VT, N1);
5871 break;
5872 case ISD::VECREDUCE_SMAX:
5873 case ISD::VECREDUCE_UMIN:
5874 if (N1.getValueType().getScalarType() == MVT::i1)
5875 return getNode(ISD::VECREDUCE_AND, DL, VT, N1);
5876 break;
5877 }
5878
5879 SDNode *N;
5880 SDVTList VTs = getVTList(VT);
5881 SDValue Ops[] = {N1};
5882 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
5883 FoldingSetNodeID ID;
5884 AddNodeIDNode(ID, Opcode, VTs, Ops);
5885 void *IP = nullptr;
5886 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5887 E->intersectFlagsWith(Flags);
5888 return SDValue(E, 0);
5889 }
5890
5891 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5892 N->setFlags(Flags);
5893 createOperands(N, Ops);
5894 CSEMap.InsertNode(N, IP);
5895 } else {
5896 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5897 createOperands(N, Ops);
5898 }
5899
5900 InsertNode(N);
5901 SDValue V = SDValue(N, 0);
5902 NewSDValueDbgMsg(V, "Creating new node: ", this);
5903 return V;
5904 }
5905
FoldValue(unsigned Opcode,const APInt & C1,const APInt & C2)5906 static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
5907 const APInt &C2) {
5908 switch (Opcode) {
5909 case ISD::ADD: return C1 + C2;
5910 case ISD::SUB: return C1 - C2;
5911 case ISD::MUL: return C1 * C2;
5912 case ISD::AND: return C1 & C2;
5913 case ISD::OR: return C1 | C2;
5914 case ISD::XOR: return C1 ^ C2;
5915 case ISD::SHL: return C1 << C2;
5916 case ISD::SRL: return C1.lshr(C2);
5917 case ISD::SRA: return C1.ashr(C2);
5918 case ISD::ROTL: return C1.rotl(C2);
5919 case ISD::ROTR: return C1.rotr(C2);
5920 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
5921 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
5922 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
5923 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
5924 case ISD::SADDSAT: return C1.sadd_sat(C2);
5925 case ISD::UADDSAT: return C1.uadd_sat(C2);
5926 case ISD::SSUBSAT: return C1.ssub_sat(C2);
5927 case ISD::USUBSAT: return C1.usub_sat(C2);
5928 case ISD::SSHLSAT: return C1.sshl_sat(C2);
5929 case ISD::USHLSAT: return C1.ushl_sat(C2);
5930 case ISD::UDIV:
5931 if (!C2.getBoolValue())
5932 break;
5933 return C1.udiv(C2);
5934 case ISD::UREM:
5935 if (!C2.getBoolValue())
5936 break;
5937 return C1.urem(C2);
5938 case ISD::SDIV:
5939 if (!C2.getBoolValue())
5940 break;
5941 return C1.sdiv(C2);
5942 case ISD::SREM:
5943 if (!C2.getBoolValue())
5944 break;
5945 return C1.srem(C2);
5946 case ISD::MULHS: {
5947 unsigned FullWidth = C1.getBitWidth() * 2;
5948 APInt C1Ext = C1.sext(FullWidth);
5949 APInt C2Ext = C2.sext(FullWidth);
5950 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5951 }
5952 case ISD::MULHU: {
5953 unsigned FullWidth = C1.getBitWidth() * 2;
5954 APInt C1Ext = C1.zext(FullWidth);
5955 APInt C2Ext = C2.zext(FullWidth);
5956 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5957 }
5958 case ISD::AVGFLOORS: {
5959 unsigned FullWidth = C1.getBitWidth() + 1;
5960 APInt C1Ext = C1.sext(FullWidth);
5961 APInt C2Ext = C2.sext(FullWidth);
5962 return (C1Ext + C2Ext).extractBits(C1.getBitWidth(), 1);
5963 }
5964 case ISD::AVGFLOORU: {
5965 unsigned FullWidth = C1.getBitWidth() + 1;
5966 APInt C1Ext = C1.zext(FullWidth);
5967 APInt C2Ext = C2.zext(FullWidth);
5968 return (C1Ext + C2Ext).extractBits(C1.getBitWidth(), 1);
5969 }
5970 case ISD::AVGCEILS: {
5971 unsigned FullWidth = C1.getBitWidth() + 1;
5972 APInt C1Ext = C1.sext(FullWidth);
5973 APInt C2Ext = C2.sext(FullWidth);
5974 return (C1Ext + C2Ext + 1).extractBits(C1.getBitWidth(), 1);
5975 }
5976 case ISD::AVGCEILU: {
5977 unsigned FullWidth = C1.getBitWidth() + 1;
5978 APInt C1Ext = C1.zext(FullWidth);
5979 APInt C2Ext = C2.zext(FullWidth);
5980 return (C1Ext + C2Ext + 1).extractBits(C1.getBitWidth(), 1);
5981 }
5982 case ISD::ABDS:
5983 return APIntOps::smax(C1, C2) - APIntOps::smin(C1, C2);
5984 case ISD::ABDU:
5985 return APIntOps::umax(C1, C2) - APIntOps::umin(C1, C2);
5986 }
5987 return std::nullopt;
5988 }
5989
5990 // Handle constant folding with UNDEF.
5991 // TODO: Handle more cases.
FoldValueWithUndef(unsigned Opcode,const APInt & C1,bool IsUndef1,const APInt & C2,bool IsUndef2)5992 static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
5993 bool IsUndef1, const APInt &C2,
5994 bool IsUndef2) {
5995 if (!(IsUndef1 || IsUndef2))
5996 return FoldValue(Opcode, C1, C2);
5997
5998 // Fold and(x, undef) -> 0
5999 // Fold mul(x, undef) -> 0
6000 if (Opcode == ISD::AND || Opcode == ISD::MUL)
6001 return APInt::getZero(C1.getBitWidth());
6002
6003 return std::nullopt;
6004 }
6005
FoldSymbolOffset(unsigned Opcode,EVT VT,const GlobalAddressSDNode * GA,const SDNode * N2)6006 SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
6007 const GlobalAddressSDNode *GA,
6008 const SDNode *N2) {
6009 if (GA->getOpcode() != ISD::GlobalAddress)
6010 return SDValue();
6011 if (!TLI->isOffsetFoldingLegal(GA))
6012 return SDValue();
6013 auto *C2 = dyn_cast<ConstantSDNode>(N2);
6014 if (!C2)
6015 return SDValue();
6016 int64_t Offset = C2->getSExtValue();
6017 switch (Opcode) {
6018 case ISD::ADD: break;
6019 case ISD::SUB: Offset = -uint64_t(Offset); break;
6020 default: return SDValue();
6021 }
6022 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
6023 GA->getOffset() + uint64_t(Offset));
6024 }
6025
isUndef(unsigned Opcode,ArrayRef<SDValue> Ops)6026 bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
6027 switch (Opcode) {
6028 case ISD::SDIV:
6029 case ISD::UDIV:
6030 case ISD::SREM:
6031 case ISD::UREM: {
6032 // If a divisor is zero/undef or any element of a divisor vector is
6033 // zero/undef, the whole op is undef.
6034 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
6035 SDValue Divisor = Ops[1];
6036 if (Divisor.isUndef() || isNullConstant(Divisor))
6037 return true;
6038
6039 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
6040 llvm::any_of(Divisor->op_values(),
6041 [](SDValue V) { return V.isUndef() ||
6042 isNullConstant(V); });
6043 // TODO: Handle signed overflow.
6044 }
6045 // TODO: Handle oversized shifts.
6046 default:
6047 return false;
6048 }
6049 }
6050
FoldConstantArithmetic(unsigned Opcode,const SDLoc & DL,EVT VT,ArrayRef<SDValue> Ops)6051 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
6052 EVT VT, ArrayRef<SDValue> Ops) {
6053 // If the opcode is a target-specific ISD node, there's nothing we can
6054 // do here and the operand rules may not line up with the below, so
6055 // bail early.
6056 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
6057 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
6058 // foldCONCAT_VECTORS in getNode before this is called.
6059 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
6060 return SDValue();
6061
6062 unsigned NumOps = Ops.size();
6063 if (NumOps == 0)
6064 return SDValue();
6065
6066 if (isUndef(Opcode, Ops))
6067 return getUNDEF(VT);
6068
6069 // Handle unary special cases.
6070 if (NumOps == 1) {
6071 SDValue N1 = Ops[0];
6072
6073 // Constant fold unary operations with an integer constant operand. Even
6074 // opaque constant will be folded, because the folding of unary operations
6075 // doesn't create new constants with different values. Nevertheless, the
6076 // opaque flag is preserved during folding to prevent future folding with
6077 // other constants.
6078 if (auto *C = dyn_cast<ConstantSDNode>(N1)) {
6079 const APInt &Val = C->getAPIntValue();
6080 switch (Opcode) {
6081 case ISD::SIGN_EXTEND:
6082 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6083 C->isTargetOpcode(), C->isOpaque());
6084 case ISD::TRUNCATE:
6085 if (C->isOpaque())
6086 break;
6087 [[fallthrough]];
6088 case ISD::ZERO_EXTEND:
6089 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6090 C->isTargetOpcode(), C->isOpaque());
6091 case ISD::ANY_EXTEND:
6092 // Some targets like RISCV prefer to sign extend some types.
6093 if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT))
6094 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6095 C->isTargetOpcode(), C->isOpaque());
6096 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6097 C->isTargetOpcode(), C->isOpaque());
6098 case ISD::ABS:
6099 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
6100 C->isOpaque());
6101 case ISD::BITREVERSE:
6102 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
6103 C->isOpaque());
6104 case ISD::BSWAP:
6105 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
6106 C->isOpaque());
6107 case ISD::CTPOP:
6108 return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(),
6109 C->isOpaque());
6110 case ISD::CTLZ:
6111 case ISD::CTLZ_ZERO_UNDEF:
6112 return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(),
6113 C->isOpaque());
6114 case ISD::CTTZ:
6115 case ISD::CTTZ_ZERO_UNDEF:
6116 return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),
6117 C->isOpaque());
6118 case ISD::UINT_TO_FP:
6119 case ISD::SINT_TO_FP: {
6120 APFloat apf(EVTToAPFloatSemantics(VT),
6121 APInt::getZero(VT.getSizeInBits()));
6122 (void)apf.convertFromAPInt(Val, Opcode == ISD::SINT_TO_FP,
6123 APFloat::rmNearestTiesToEven);
6124 return getConstantFP(apf, DL, VT);
6125 }
6126 case ISD::FP16_TO_FP:
6127 case ISD::BF16_TO_FP: {
6128 bool Ignored;
6129 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
6130 : APFloat::BFloat(),
6131 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
6132
6133 // This can return overflow, underflow, or inexact; we don't care.
6134 // FIXME need to be more flexible about rounding mode.
6135 (void)FPV.convert(EVTToAPFloatSemantics(VT),
6136 APFloat::rmNearestTiesToEven, &Ignored);
6137 return getConstantFP(FPV, DL, VT);
6138 }
6139 case ISD::STEP_VECTOR:
6140 if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this))
6141 return V;
6142 break;
6143 case ISD::BITCAST:
6144 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
6145 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
6146 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
6147 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
6148 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
6149 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
6150 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
6151 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
6152 break;
6153 }
6154 }
6155
6156 // Constant fold unary operations with a floating point constant operand.
6157 if (auto *C = dyn_cast<ConstantFPSDNode>(N1)) {
6158 APFloat V = C->getValueAPF(); // make copy
6159 switch (Opcode) {
6160 case ISD::FNEG:
6161 V.changeSign();
6162 return getConstantFP(V, DL, VT);
6163 case ISD::FABS:
6164 V.clearSign();
6165 return getConstantFP(V, DL, VT);
6166 case ISD::FCEIL: {
6167 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
6168 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6169 return getConstantFP(V, DL, VT);
6170 return SDValue();
6171 }
6172 case ISD::FTRUNC: {
6173 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
6174 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6175 return getConstantFP(V, DL, VT);
6176 return SDValue();
6177 }
6178 case ISD::FFLOOR: {
6179 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
6180 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6181 return getConstantFP(V, DL, VT);
6182 return SDValue();
6183 }
6184 case ISD::FP_EXTEND: {
6185 bool ignored;
6186 // This can return overflow, underflow, or inexact; we don't care.
6187 // FIXME need to be more flexible about rounding mode.
6188 (void)V.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
6189 &ignored);
6190 return getConstantFP(V, DL, VT);
6191 }
6192 case ISD::FP_TO_SINT:
6193 case ISD::FP_TO_UINT: {
6194 bool ignored;
6195 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
6196 // FIXME need to be more flexible about rounding mode.
6197 APFloat::opStatus s =
6198 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
6199 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
6200 break;
6201 return getConstant(IntVal, DL, VT);
6202 }
6203 case ISD::FP_TO_FP16:
6204 case ISD::FP_TO_BF16: {
6205 bool Ignored;
6206 // This can return overflow, underflow, or inexact; we don't care.
6207 // FIXME need to be more flexible about rounding mode.
6208 (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
6209 : APFloat::BFloat(),
6210 APFloat::rmNearestTiesToEven, &Ignored);
6211 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6212 }
6213 case ISD::BITCAST:
6214 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
6215 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6216 VT);
6217 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
6218 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6219 VT);
6220 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
6221 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
6222 VT);
6223 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
6224 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6225 break;
6226 }
6227 }
6228
6229 // Early-out if we failed to constant fold a bitcast.
6230 if (Opcode == ISD::BITCAST)
6231 return SDValue();
6232 }
6233
6234 // Handle binops special cases.
6235 if (NumOps == 2) {
6236 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
6237 return CFP;
6238
6239 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6240 if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
6241 if (C1->isOpaque() || C2->isOpaque())
6242 return SDValue();
6243
6244 std::optional<APInt> FoldAttempt =
6245 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
6246 if (!FoldAttempt)
6247 return SDValue();
6248
6249 SDValue Folded = getConstant(*FoldAttempt, DL, VT);
6250 assert((!Folded || !VT.isVector()) &&
6251 "Can't fold vectors ops with scalar operands");
6252 return Folded;
6253 }
6254 }
6255
6256 // fold (add Sym, c) -> Sym+c
6257 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[0]))
6258 return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode());
6259 if (TLI->isCommutativeBinOp(Opcode))
6260 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[1]))
6261 return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode());
6262 }
6263
6264 // This is for vector folding only from here on.
6265 if (!VT.isVector())
6266 return SDValue();
6267
6268 ElementCount NumElts = VT.getVectorElementCount();
6269
6270 // See if we can fold through bitcasted integer ops.
6271 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
6272 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6273 Ops[0].getOpcode() == ISD::BITCAST &&
6274 Ops[1].getOpcode() == ISD::BITCAST) {
6275 SDValue N1 = peekThroughBitcasts(Ops[0]);
6276 SDValue N2 = peekThroughBitcasts(Ops[1]);
6277 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
6278 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
6279 EVT BVVT = N1.getValueType();
6280 if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) {
6281 bool IsLE = getDataLayout().isLittleEndian();
6282 unsigned EltBits = VT.getScalarSizeInBits();
6283 SmallVector<APInt> RawBits1, RawBits2;
6284 BitVector UndefElts1, UndefElts2;
6285 if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
6286 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
6287 SmallVector<APInt> RawBits;
6288 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
6289 std::optional<APInt> Fold = FoldValueWithUndef(
6290 Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]);
6291 if (!Fold)
6292 break;
6293 RawBits.push_back(*Fold);
6294 }
6295 if (RawBits.size() == NumElts.getFixedValue()) {
6296 // We have constant folded, but we need to cast this again back to
6297 // the original (possibly legalized) type.
6298 SmallVector<APInt> DstBits;
6299 BitVector DstUndefs;
6300 BuildVectorSDNode::recastRawBits(IsLE, BVVT.getScalarSizeInBits(),
6301 DstBits, RawBits, DstUndefs,
6302 BitVector(RawBits.size(), false));
6303 EVT BVEltVT = BV1->getOperand(0).getValueType();
6304 unsigned BVEltBits = BVEltVT.getSizeInBits();
6305 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(BVEltVT));
6306 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
6307 if (DstUndefs[I])
6308 continue;
6309 Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT);
6310 }
6311 return getBitcast(VT, getBuildVector(BVVT, DL, Ops));
6312 }
6313 }
6314 }
6315 }
6316
6317 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
6318 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
6319 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
6320 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
6321 APInt RHSVal;
6322 if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) {
6323 APInt NewStep = Opcode == ISD::MUL
6324 ? Ops[0].getConstantOperandAPInt(0) * RHSVal
6325 : Ops[0].getConstantOperandAPInt(0) << RHSVal;
6326 return getStepVector(DL, VT, NewStep);
6327 }
6328 }
6329
6330 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
6331 return !Op.getValueType().isVector() ||
6332 Op.getValueType().getVectorElementCount() == NumElts;
6333 };
6334
6335 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
6336 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
6337 Op.getOpcode() == ISD::BUILD_VECTOR ||
6338 Op.getOpcode() == ISD::SPLAT_VECTOR;
6339 };
6340
6341 // All operands must be vector types with the same number of elements as
6342 // the result type and must be either UNDEF or a build/splat vector
6343 // or UNDEF scalars.
6344 if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||
6345 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
6346 return SDValue();
6347
6348 // If we are comparing vectors, then the result needs to be a i1 boolean that
6349 // is then extended back to the legal result type depending on how booleans
6350 // are represented.
6351 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
6352 ISD::NodeType ExtendCode =
6353 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
6354 ? TargetLowering::getExtendForContent(TLI->getBooleanContents(VT))
6355 : ISD::SIGN_EXTEND;
6356
6357 // Find legal integer scalar type for constant promotion and
6358 // ensure that its scalar size is at least as large as source.
6359 EVT LegalSVT = VT.getScalarType();
6360 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
6361 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
6362 if (LegalSVT.bitsLT(VT.getScalarType()))
6363 return SDValue();
6364 }
6365
6366 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
6367 // only have one operand to check. For fixed-length vector types we may have
6368 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
6369 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
6370
6371 // Constant fold each scalar lane separately.
6372 SmallVector<SDValue, 4> ScalarResults;
6373 for (unsigned I = 0; I != NumVectorElts; I++) {
6374 SmallVector<SDValue, 4> ScalarOps;
6375 for (SDValue Op : Ops) {
6376 EVT InSVT = Op.getValueType().getScalarType();
6377 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
6378 Op.getOpcode() != ISD::SPLAT_VECTOR) {
6379 if (Op.isUndef())
6380 ScalarOps.push_back(getUNDEF(InSVT));
6381 else
6382 ScalarOps.push_back(Op);
6383 continue;
6384 }
6385
6386 SDValue ScalarOp =
6387 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
6388 EVT ScalarVT = ScalarOp.getValueType();
6389
6390 // Build vector (integer) scalar operands may need implicit
6391 // truncation - do this before constant folding.
6392 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) {
6393 // Don't create illegally-typed nodes unless they're constants or undef
6394 // - if we fail to constant fold we can't guarantee the (dead) nodes
6395 // we're creating will be cleaned up before being visited for
6396 // legalization.
6397 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
6398 !isa<ConstantSDNode>(ScalarOp) &&
6399 TLI->getTypeAction(*getContext(), InSVT) !=
6400 TargetLowering::TypeLegal)
6401 return SDValue();
6402 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
6403 }
6404
6405 ScalarOps.push_back(ScalarOp);
6406 }
6407
6408 // Constant fold the scalar operands.
6409 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps);
6410
6411 // Legalize the (integer) scalar constant if necessary.
6412 if (LegalSVT != SVT)
6413 ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult);
6414
6415 // Scalar folding only succeeded if the result is a constant or UNDEF.
6416 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
6417 ScalarResult.getOpcode() != ISD::ConstantFP)
6418 return SDValue();
6419 ScalarResults.push_back(ScalarResult);
6420 }
6421
6422 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
6423 : getBuildVector(VT, DL, ScalarResults);
6424 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
6425 return V;
6426 }
6427
foldConstantFPMath(unsigned Opcode,const SDLoc & DL,EVT VT,ArrayRef<SDValue> Ops)6428 SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
6429 EVT VT, ArrayRef<SDValue> Ops) {
6430 // TODO: Add support for unary/ternary fp opcodes.
6431 if (Ops.size() != 2)
6432 return SDValue();
6433
6434 // TODO: We don't do any constant folding for strict FP opcodes here, but we
6435 // should. That will require dealing with a potentially non-default
6436 // rounding mode, checking the "opStatus" return value from the APFloat
6437 // math calculations, and possibly other variations.
6438 SDValue N1 = Ops[0];
6439 SDValue N2 = Ops[1];
6440 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
6441 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
6442 if (N1CFP && N2CFP) {
6443 APFloat C1 = N1CFP->getValueAPF(); // make copy
6444 const APFloat &C2 = N2CFP->getValueAPF();
6445 switch (Opcode) {
6446 case ISD::FADD:
6447 C1.add(C2, APFloat::rmNearestTiesToEven);
6448 return getConstantFP(C1, DL, VT);
6449 case ISD::FSUB:
6450 C1.subtract(C2, APFloat::rmNearestTiesToEven);
6451 return getConstantFP(C1, DL, VT);
6452 case ISD::FMUL:
6453 C1.multiply(C2, APFloat::rmNearestTiesToEven);
6454 return getConstantFP(C1, DL, VT);
6455 case ISD::FDIV:
6456 C1.divide(C2, APFloat::rmNearestTiesToEven);
6457 return getConstantFP(C1, DL, VT);
6458 case ISD::FREM:
6459 C1.mod(C2);
6460 return getConstantFP(C1, DL, VT);
6461 case ISD::FCOPYSIGN:
6462 C1.copySign(C2);
6463 return getConstantFP(C1, DL, VT);
6464 case ISD::FMINNUM:
6465 return getConstantFP(minnum(C1, C2), DL, VT);
6466 case ISD::FMAXNUM:
6467 return getConstantFP(maxnum(C1, C2), DL, VT);
6468 case ISD::FMINIMUM:
6469 return getConstantFP(minimum(C1, C2), DL, VT);
6470 case ISD::FMAXIMUM:
6471 return getConstantFP(maximum(C1, C2), DL, VT);
6472 default: break;
6473 }
6474 }
6475 if (N1CFP && Opcode == ISD::FP_ROUND) {
6476 APFloat C1 = N1CFP->getValueAPF(); // make copy
6477 bool Unused;
6478 // This can return overflow, underflow, or inexact; we don't care.
6479 // FIXME need to be more flexible about rounding mode.
6480 (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
6481 &Unused);
6482 return getConstantFP(C1, DL, VT);
6483 }
6484
6485 switch (Opcode) {
6486 case ISD::FSUB:
6487 // -0.0 - undef --> undef (consistent with "fneg undef")
6488 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))
6489 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
6490 return getUNDEF(VT);
6491 [[fallthrough]];
6492
6493 case ISD::FADD:
6494 case ISD::FMUL:
6495 case ISD::FDIV:
6496 case ISD::FREM:
6497 // If both operands are undef, the result is undef. If 1 operand is undef,
6498 // the result is NaN. This should match the behavior of the IR optimizer.
6499 if (N1.isUndef() && N2.isUndef())
6500 return getUNDEF(VT);
6501 if (N1.isUndef() || N2.isUndef())
6502 return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
6503 }
6504 return SDValue();
6505 }
6506
getAssertAlign(const SDLoc & DL,SDValue Val,Align A)6507 SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
6508 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
6509
6510 // There's no need to assert on a byte-aligned pointer. All pointers are at
6511 // least byte aligned.
6512 if (A == Align(1))
6513 return Val;
6514
6515 FoldingSetNodeID ID;
6516 AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val});
6517 ID.AddInteger(A.value());
6518
6519 void *IP = nullptr;
6520 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6521 return SDValue(E, 0);
6522
6523 auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(),
6524 Val.getValueType(), A);
6525 createOperands(N, {Val});
6526
6527 CSEMap.InsertNode(N, IP);
6528 InsertNode(N);
6529
6530 SDValue V(N, 0);
6531 NewSDValueDbgMsg(V, "Creating new node: ", this);
6532 return V;
6533 }
6534
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,SDValue N1,SDValue N2)6535 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6536 SDValue N1, SDValue N2) {
6537 SDNodeFlags Flags;
6538 if (Inserter)
6539 Flags = Inserter->getFlags();
6540 return getNode(Opcode, DL, VT, N1, N2, Flags);
6541 }
6542
canonicalizeCommutativeBinop(unsigned Opcode,SDValue & N1,SDValue & N2) const6543 void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
6544 SDValue &N2) const {
6545 if (!TLI->isCommutativeBinOp(Opcode))
6546 return;
6547
6548 // Canonicalize:
6549 // binop(const, nonconst) -> binop(nonconst, const)
6550 SDNode *N1C = isConstantIntBuildVectorOrConstantInt(N1);
6551 SDNode *N2C = isConstantIntBuildVectorOrConstantInt(N2);
6552 SDNode *N1CFP = isConstantFPBuildVectorOrConstantFP(N1);
6553 SDNode *N2CFP = isConstantFPBuildVectorOrConstantFP(N2);
6554 if ((N1C && !N2C) || (N1CFP && !N2CFP))
6555 std::swap(N1, N2);
6556
6557 // Canonicalize:
6558 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
6559 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6560 N2.getOpcode() == ISD::STEP_VECTOR)
6561 std::swap(N1, N2);
6562 }
6563
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,SDValue N1,SDValue N2,const SDNodeFlags Flags)6564 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6565 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
6566 assert(N1.getOpcode() != ISD::DELETED_NODE &&
6567 N2.getOpcode() != ISD::DELETED_NODE &&
6568 "Operand is DELETED_NODE!");
6569
6570 canonicalizeCommutativeBinop(Opcode, N1, N2);
6571
6572 auto *N1C = dyn_cast<ConstantSDNode>(N1);
6573 auto *N2C = dyn_cast<ConstantSDNode>(N2);
6574
6575 // Don't allow undefs in vector splats - we might be returning N2 when folding
6576 // to zero etc.
6577 ConstantSDNode *N2CV =
6578 isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
6579
6580 switch (Opcode) {
6581 default: break;
6582 case ISD::TokenFactor:
6583 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
6584 N2.getValueType() == MVT::Other && "Invalid token factor!");
6585 // Fold trivial token factors.
6586 if (N1.getOpcode() == ISD::EntryToken) return N2;
6587 if (N2.getOpcode() == ISD::EntryToken) return N1;
6588 if (N1 == N2) return N1;
6589 break;
6590 case ISD::BUILD_VECTOR: {
6591 // Attempt to simplify BUILD_VECTOR.
6592 SDValue Ops[] = {N1, N2};
6593 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
6594 return V;
6595 break;
6596 }
6597 case ISD::CONCAT_VECTORS: {
6598 SDValue Ops[] = {N1, N2};
6599 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
6600 return V;
6601 break;
6602 }
6603 case ISD::AND:
6604 assert(VT.isInteger() && "This operator does not apply to FP types!");
6605 assert(N1.getValueType() == N2.getValueType() &&
6606 N1.getValueType() == VT && "Binary operator types must match!");
6607 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
6608 // worth handling here.
6609 if (N2CV && N2CV->isZero())
6610 return N2;
6611 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
6612 return N1;
6613 break;
6614 case ISD::OR:
6615 case ISD::XOR:
6616 case ISD::ADD:
6617 case ISD::SUB:
6618 assert(VT.isInteger() && "This operator does not apply to FP types!");
6619 assert(N1.getValueType() == N2.getValueType() &&
6620 N1.getValueType() == VT && "Binary operator types must match!");
6621 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
6622 // it's worth handling here.
6623 if (N2CV && N2CV->isZero())
6624 return N1;
6625 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
6626 VT.getVectorElementType() == MVT::i1)
6627 return getNode(ISD::XOR, DL, VT, N1, N2);
6628 break;
6629 case ISD::MUL:
6630 assert(VT.isInteger() && "This operator does not apply to FP types!");
6631 assert(N1.getValueType() == N2.getValueType() &&
6632 N1.getValueType() == VT && "Binary operator types must match!");
6633 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6634 return getNode(ISD::AND, DL, VT, N1, N2);
6635 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6636 const APInt &MulImm = N1->getConstantOperandAPInt(0);
6637 const APInt &N2CImm = N2C->getAPIntValue();
6638 return getVScale(DL, VT, MulImm * N2CImm);
6639 }
6640 break;
6641 case ISD::UDIV:
6642 case ISD::UREM:
6643 case ISD::MULHU:
6644 case ISD::MULHS:
6645 case ISD::SDIV:
6646 case ISD::SREM:
6647 case ISD::SADDSAT:
6648 case ISD::SSUBSAT:
6649 case ISD::UADDSAT:
6650 case ISD::USUBSAT:
6651 assert(VT.isInteger() && "This operator does not apply to FP types!");
6652 assert(N1.getValueType() == N2.getValueType() &&
6653 N1.getValueType() == VT && "Binary operator types must match!");
6654 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
6655 // fold (add_sat x, y) -> (or x, y) for bool types.
6656 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
6657 return getNode(ISD::OR, DL, VT, N1, N2);
6658 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
6659 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
6660 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
6661 }
6662 break;
6663 case ISD::ABDS:
6664 case ISD::ABDU:
6665 assert(VT.isInteger() && "This operator does not apply to FP types!");
6666 assert(N1.getValueType() == N2.getValueType() &&
6667 N1.getValueType() == VT && "Binary operator types must match!");
6668 break;
6669 case ISD::SMIN:
6670 case ISD::UMAX:
6671 assert(VT.isInteger() && "This operator does not apply to FP types!");
6672 assert(N1.getValueType() == N2.getValueType() &&
6673 N1.getValueType() == VT && "Binary operator types must match!");
6674 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6675 return getNode(ISD::OR, DL, VT, N1, N2);
6676 break;
6677 case ISD::SMAX:
6678 case ISD::UMIN:
6679 assert(VT.isInteger() && "This operator does not apply to FP types!");
6680 assert(N1.getValueType() == N2.getValueType() &&
6681 N1.getValueType() == VT && "Binary operator types must match!");
6682 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6683 return getNode(ISD::AND, DL, VT, N1, N2);
6684 break;
6685 case ISD::FADD:
6686 case ISD::FSUB:
6687 case ISD::FMUL:
6688 case ISD::FDIV:
6689 case ISD::FREM:
6690 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
6691 assert(N1.getValueType() == N2.getValueType() &&
6692 N1.getValueType() == VT && "Binary operator types must match!");
6693 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
6694 return V;
6695 break;
6696 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
6697 assert(N1.getValueType() == VT &&
6698 N1.getValueType().isFloatingPoint() &&
6699 N2.getValueType().isFloatingPoint() &&
6700 "Invalid FCOPYSIGN!");
6701 break;
6702 case ISD::SHL:
6703 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6704 const APInt &MulImm = N1->getConstantOperandAPInt(0);
6705 const APInt &ShiftImm = N2C->getAPIntValue();
6706 return getVScale(DL, VT, MulImm << ShiftImm);
6707 }
6708 [[fallthrough]];
6709 case ISD::SRA:
6710 case ISD::SRL:
6711 if (SDValue V = simplifyShift(N1, N2))
6712 return V;
6713 [[fallthrough]];
6714 case ISD::ROTL:
6715 case ISD::ROTR:
6716 assert(VT == N1.getValueType() &&
6717 "Shift operators return type must be the same as their first arg");
6718 assert(VT.isInteger() && N2.getValueType().isInteger() &&
6719 "Shifts only work on integers");
6720 assert((!VT.isVector() || VT == N2.getValueType()) &&
6721 "Vector shift amounts must be in the same as their first arg");
6722 // Verify that the shift amount VT is big enough to hold valid shift
6723 // amounts. This catches things like trying to shift an i1024 value by an
6724 // i8, which is easy to fall into in generic code that uses
6725 // TLI.getShiftAmount().
6726 assert(N2.getValueType().getScalarSizeInBits() >=
6727 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
6728 "Invalid use of small shift amount with oversized value!");
6729
6730 // Always fold shifts of i1 values so the code generator doesn't need to
6731 // handle them. Since we know the size of the shift has to be less than the
6732 // size of the value, the shift/rotate count is guaranteed to be zero.
6733 if (VT == MVT::i1)
6734 return N1;
6735 if (N2CV && N2CV->isZero())
6736 return N1;
6737 break;
6738 case ISD::FP_ROUND:
6739 assert(VT.isFloatingPoint() &&
6740 N1.getValueType().isFloatingPoint() &&
6741 VT.bitsLE(N1.getValueType()) &&
6742 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
6743 "Invalid FP_ROUND!");
6744 if (N1.getValueType() == VT) return N1; // noop conversion.
6745 break;
6746 case ISD::AssertSext:
6747 case ISD::AssertZext: {
6748 EVT EVT = cast<VTSDNode>(N2)->getVT();
6749 assert(VT == N1.getValueType() && "Not an inreg extend!");
6750 assert(VT.isInteger() && EVT.isInteger() &&
6751 "Cannot *_EXTEND_INREG FP types");
6752 assert(!EVT.isVector() &&
6753 "AssertSExt/AssertZExt type should be the vector element type "
6754 "rather than the vector type!");
6755 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
6756 if (VT.getScalarType() == EVT) return N1; // noop assertion.
6757 break;
6758 }
6759 case ISD::SIGN_EXTEND_INREG: {
6760 EVT EVT = cast<VTSDNode>(N2)->getVT();
6761 assert(VT == N1.getValueType() && "Not an inreg extend!");
6762 assert(VT.isInteger() && EVT.isInteger() &&
6763 "Cannot *_EXTEND_INREG FP types");
6764 assert(EVT.isVector() == VT.isVector() &&
6765 "SIGN_EXTEND_INREG type should be vector iff the operand "
6766 "type is vector!");
6767 assert((!EVT.isVector() ||
6768 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
6769 "Vector element counts must match in SIGN_EXTEND_INREG");
6770 assert(EVT.bitsLE(VT) && "Not extending!");
6771 if (EVT == VT) return N1; // Not actually extending
6772
6773 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
6774 unsigned FromBits = EVT.getScalarSizeInBits();
6775 Val <<= Val.getBitWidth() - FromBits;
6776 Val.ashrInPlace(Val.getBitWidth() - FromBits);
6777 return getConstant(Val, DL, ConstantVT);
6778 };
6779
6780 if (N1C) {
6781 const APInt &Val = N1C->getAPIntValue();
6782 return SignExtendInReg(Val, VT);
6783 }
6784
6785 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
6786 SmallVector<SDValue, 8> Ops;
6787 llvm::EVT OpVT = N1.getOperand(0).getValueType();
6788 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
6789 SDValue Op = N1.getOperand(i);
6790 if (Op.isUndef()) {
6791 Ops.push_back(getUNDEF(OpVT));
6792 continue;
6793 }
6794 ConstantSDNode *C = cast<ConstantSDNode>(Op);
6795 APInt Val = C->getAPIntValue();
6796 Ops.push_back(SignExtendInReg(Val, OpVT));
6797 }
6798 return getBuildVector(VT, DL, Ops);
6799 }
6800
6801 if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6802 isa<ConstantSDNode>(N1.getOperand(0)))
6803 return getNode(
6804 ISD::SPLAT_VECTOR, DL, VT,
6805 SignExtendInReg(N1.getConstantOperandAPInt(0),
6806 N1.getOperand(0).getValueType()));
6807 break;
6808 }
6809 case ISD::FP_TO_SINT_SAT:
6810 case ISD::FP_TO_UINT_SAT: {
6811 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
6812 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
6813 assert(N1.getValueType().isVector() == VT.isVector() &&
6814 "FP_TO_*INT_SAT type should be vector iff the operand type is "
6815 "vector!");
6816 assert((!VT.isVector() || VT.getVectorElementCount() ==
6817 N1.getValueType().getVectorElementCount()) &&
6818 "Vector element counts must match in FP_TO_*INT_SAT");
6819 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
6820 "Type to saturate to must be a scalar.");
6821 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
6822 "Not extending!");
6823 break;
6824 }
6825 case ISD::EXTRACT_VECTOR_ELT:
6826 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
6827 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
6828 element type of the vector.");
6829
6830 // Extract from an undefined value or using an undefined index is undefined.
6831 if (N1.isUndef() || N2.isUndef())
6832 return getUNDEF(VT);
6833
6834 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
6835 // vectors. For scalable vectors we will provide appropriate support for
6836 // dealing with arbitrary indices.
6837 if (N2C && N1.getValueType().isFixedLengthVector() &&
6838 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
6839 return getUNDEF(VT);
6840
6841 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
6842 // expanding copies of large vectors from registers. This only works for
6843 // fixed length vectors, since we need to know the exact number of
6844 // elements.
6845 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
6846 N1.getOperand(0).getValueType().isFixedLengthVector()) {
6847 unsigned Factor =
6848 N1.getOperand(0).getValueType().getVectorNumElements();
6849 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
6850 N1.getOperand(N2C->getZExtValue() / Factor),
6851 getVectorIdxConstant(N2C->getZExtValue() % Factor, DL));
6852 }
6853
6854 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
6855 // lowering is expanding large vector constants.
6856 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
6857 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
6858 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
6859 N1.getValueType().isFixedLengthVector()) &&
6860 "BUILD_VECTOR used for scalable vectors");
6861 unsigned Index =
6862 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
6863 SDValue Elt = N1.getOperand(Index);
6864
6865 if (VT != Elt.getValueType())
6866 // If the vector element type is not legal, the BUILD_VECTOR operands
6867 // are promoted and implicitly truncated, and the result implicitly
6868 // extended. Make that explicit here.
6869 Elt = getAnyExtOrTrunc(Elt, DL, VT);
6870
6871 return Elt;
6872 }
6873
6874 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
6875 // operations are lowered to scalars.
6876 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
6877 // If the indices are the same, return the inserted element else
6878 // if the indices are known different, extract the element from
6879 // the original vector.
6880 SDValue N1Op2 = N1.getOperand(2);
6881 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
6882
6883 if (N1Op2C && N2C) {
6884 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
6885 if (VT == N1.getOperand(1).getValueType())
6886 return N1.getOperand(1);
6887 if (VT.isFloatingPoint()) {
6888 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
6889 return getFPExtendOrRound(N1.getOperand(1), DL, VT);
6890 }
6891 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
6892 }
6893 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
6894 }
6895 }
6896
6897 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
6898 // when vector types are scalarized and v1iX is legal.
6899 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
6900 // Here we are completely ignoring the extract element index (N2),
6901 // which is fine for fixed width vectors, since any index other than 0
6902 // is undefined anyway. However, this cannot be ignored for scalable
6903 // vectors - in theory we could support this, but we don't want to do this
6904 // without a profitability check.
6905 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6906 N1.getValueType().isFixedLengthVector() &&
6907 N1.getValueType().getVectorNumElements() == 1) {
6908 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
6909 N1.getOperand(1));
6910 }
6911 break;
6912 case ISD::EXTRACT_ELEMENT:
6913 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
6914 assert(!N1.getValueType().isVector() && !VT.isVector() &&
6915 (N1.getValueType().isInteger() == VT.isInteger()) &&
6916 N1.getValueType() != VT &&
6917 "Wrong types for EXTRACT_ELEMENT!");
6918
6919 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
6920 // 64-bit integers into 32-bit parts. Instead of building the extract of
6921 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
6922 if (N1.getOpcode() == ISD::BUILD_PAIR)
6923 return N1.getOperand(N2C->getZExtValue());
6924
6925 // EXTRACT_ELEMENT of a constant int is also very common.
6926 if (N1C) {
6927 unsigned ElementSize = VT.getSizeInBits();
6928 unsigned Shift = ElementSize * N2C->getZExtValue();
6929 const APInt &Val = N1C->getAPIntValue();
6930 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
6931 }
6932 break;
6933 case ISD::EXTRACT_SUBVECTOR: {
6934 EVT N1VT = N1.getValueType();
6935 assert(VT.isVector() && N1VT.isVector() &&
6936 "Extract subvector VTs must be vectors!");
6937 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
6938 "Extract subvector VTs must have the same element type!");
6939 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
6940 "Cannot extract a scalable vector from a fixed length vector!");
6941 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
6942 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
6943 "Extract subvector must be from larger vector to smaller vector!");
6944 assert(N2C && "Extract subvector index must be a constant");
6945 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
6946 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
6947 N1VT.getVectorMinNumElements()) &&
6948 "Extract subvector overflow!");
6949 assert(N2C->getAPIntValue().getBitWidth() ==
6950 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
6951 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
6952
6953 // Trivial extraction.
6954 if (VT == N1VT)
6955 return N1;
6956
6957 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
6958 if (N1.isUndef())
6959 return getUNDEF(VT);
6960
6961 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
6962 // the concat have the same type as the extract.
6963 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
6964 VT == N1.getOperand(0).getValueType()) {
6965 unsigned Factor = VT.getVectorMinNumElements();
6966 return N1.getOperand(N2C->getZExtValue() / Factor);
6967 }
6968
6969 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
6970 // during shuffle legalization.
6971 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
6972 VT == N1.getOperand(1).getValueType())
6973 return N1.getOperand(1);
6974 break;
6975 }
6976 }
6977
6978 // Perform trivial constant folding.
6979 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
6980 return SV;
6981
6982 // Canonicalize an UNDEF to the RHS, even over a constant.
6983 if (N1.isUndef()) {
6984 if (TLI->isCommutativeBinOp(Opcode)) {
6985 std::swap(N1, N2);
6986 } else {
6987 switch (Opcode) {
6988 case ISD::SUB:
6989 return getUNDEF(VT); // fold op(undef, arg2) -> undef
6990 case ISD::SIGN_EXTEND_INREG:
6991 case ISD::UDIV:
6992 case ISD::SDIV:
6993 case ISD::UREM:
6994 case ISD::SREM:
6995 case ISD::SSUBSAT:
6996 case ISD::USUBSAT:
6997 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
6998 }
6999 }
7000 }
7001
7002 // Fold a bunch of operators when the RHS is undef.
7003 if (N2.isUndef()) {
7004 switch (Opcode) {
7005 case ISD::XOR:
7006 if (N1.isUndef())
7007 // Handle undef ^ undef -> 0 special case. This is a common
7008 // idiom (misuse).
7009 return getConstant(0, DL, VT);
7010 [[fallthrough]];
7011 case ISD::ADD:
7012 case ISD::SUB:
7013 case ISD::UDIV:
7014 case ISD::SDIV:
7015 case ISD::UREM:
7016 case ISD::SREM:
7017 return getUNDEF(VT); // fold op(arg1, undef) -> undef
7018 case ISD::MUL:
7019 case ISD::AND:
7020 case ISD::SSUBSAT:
7021 case ISD::USUBSAT:
7022 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
7023 case ISD::OR:
7024 case ISD::SADDSAT:
7025 case ISD::UADDSAT:
7026 return getAllOnesConstant(DL, VT);
7027 }
7028 }
7029
7030 // Memoize this node if possible.
7031 SDNode *N;
7032 SDVTList VTs = getVTList(VT);
7033 SDValue Ops[] = {N1, N2};
7034 if (VT != MVT::Glue) {
7035 FoldingSetNodeID ID;
7036 AddNodeIDNode(ID, Opcode, VTs, Ops);
7037 void *IP = nullptr;
7038 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7039 E->intersectFlagsWith(Flags);
7040 return SDValue(E, 0);
7041 }
7042
7043 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7044 N->setFlags(Flags);
7045 createOperands(N, Ops);
7046 CSEMap.InsertNode(N, IP);
7047 } else {
7048 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7049 createOperands(N, Ops);
7050 }
7051
7052 InsertNode(N);
7053 SDValue V = SDValue(N, 0);
7054 NewSDValueDbgMsg(V, "Creating new node: ", this);
7055 return V;
7056 }
7057
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,SDValue N1,SDValue N2,SDValue N3)7058 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7059 SDValue N1, SDValue N2, SDValue N3) {
7060 SDNodeFlags Flags;
7061 if (Inserter)
7062 Flags = Inserter->getFlags();
7063 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
7064 }
7065
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,SDValue N1,SDValue N2,SDValue N3,const SDNodeFlags Flags)7066 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7067 SDValue N1, SDValue N2, SDValue N3,
7068 const SDNodeFlags Flags) {
7069 assert(N1.getOpcode() != ISD::DELETED_NODE &&
7070 N2.getOpcode() != ISD::DELETED_NODE &&
7071 N3.getOpcode() != ISD::DELETED_NODE &&
7072 "Operand is DELETED_NODE!");
7073 // Perform various simplifications.
7074 switch (Opcode) {
7075 case ISD::FMA:
7076 case ISD::FMAD: {
7077 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7078 assert(N1.getValueType() == VT && N2.getValueType() == VT &&
7079 N3.getValueType() == VT && "FMA types must match!");
7080 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
7081 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
7082 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
7083 if (N1CFP && N2CFP && N3CFP) {
7084 APFloat V1 = N1CFP->getValueAPF();
7085 const APFloat &V2 = N2CFP->getValueAPF();
7086 const APFloat &V3 = N3CFP->getValueAPF();
7087 if (Opcode == ISD::FMAD) {
7088 V1.multiply(V2, APFloat::rmNearestTiesToEven);
7089 V1.add(V3, APFloat::rmNearestTiesToEven);
7090 } else
7091 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
7092 return getConstantFP(V1, DL, VT);
7093 }
7094 break;
7095 }
7096 case ISD::BUILD_VECTOR: {
7097 // Attempt to simplify BUILD_VECTOR.
7098 SDValue Ops[] = {N1, N2, N3};
7099 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7100 return V;
7101 break;
7102 }
7103 case ISD::CONCAT_VECTORS: {
7104 SDValue Ops[] = {N1, N2, N3};
7105 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7106 return V;
7107 break;
7108 }
7109 case ISD::SETCC: {
7110 assert(VT.isInteger() && "SETCC result type must be an integer!");
7111 assert(N1.getValueType() == N2.getValueType() &&
7112 "SETCC operands must have the same type!");
7113 assert(VT.isVector() == N1.getValueType().isVector() &&
7114 "SETCC type should be vector iff the operand type is vector!");
7115 assert((!VT.isVector() || VT.getVectorElementCount() ==
7116 N1.getValueType().getVectorElementCount()) &&
7117 "SETCC vector element counts must match!");
7118 // Use FoldSetCC to simplify SETCC's.
7119 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
7120 return V;
7121 // Vector constant folding.
7122 SDValue Ops[] = {N1, N2, N3};
7123 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
7124 NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
7125 return V;
7126 }
7127 break;
7128 }
7129 case ISD::SELECT:
7130 case ISD::VSELECT:
7131 if (SDValue V = simplifySelect(N1, N2, N3))
7132 return V;
7133 break;
7134 case ISD::VECTOR_SHUFFLE:
7135 llvm_unreachable("should use getVectorShuffle constructor!");
7136 case ISD::VECTOR_SPLICE: {
7137 if (cast<ConstantSDNode>(N3)->isZero())
7138 return N1;
7139 break;
7140 }
7141 case ISD::INSERT_VECTOR_ELT: {
7142 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
7143 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
7144 // for scalable vectors where we will generate appropriate code to
7145 // deal with out-of-bounds cases correctly.
7146 if (N3C && N1.getValueType().isFixedLengthVector() &&
7147 N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
7148 return getUNDEF(VT);
7149
7150 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
7151 if (N3.isUndef())
7152 return getUNDEF(VT);
7153
7154 // If the inserted element is an UNDEF, just use the input vector.
7155 if (N2.isUndef())
7156 return N1;
7157
7158 break;
7159 }
7160 case ISD::INSERT_SUBVECTOR: {
7161 // Inserting undef into undef is still undef.
7162 if (N1.isUndef() && N2.isUndef())
7163 return getUNDEF(VT);
7164
7165 EVT N2VT = N2.getValueType();
7166 assert(VT == N1.getValueType() &&
7167 "Dest and insert subvector source types must match!");
7168 assert(VT.isVector() && N2VT.isVector() &&
7169 "Insert subvector VTs must be vectors!");
7170 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
7171 "Insert subvector VTs must have the same element type!");
7172 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
7173 "Cannot insert a scalable vector into a fixed length vector!");
7174 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7175 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
7176 "Insert subvector must be from smaller vector to larger vector!");
7177 assert(isa<ConstantSDNode>(N3) &&
7178 "Insert subvector index must be constant");
7179 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7180 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
7181 VT.getVectorMinNumElements()) &&
7182 "Insert subvector overflow!");
7183 assert(N3->getAsAPIntVal().getBitWidth() ==
7184 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7185 "Constant index for INSERT_SUBVECTOR has an invalid size");
7186
7187 // Trivial insertion.
7188 if (VT == N2VT)
7189 return N2;
7190
7191 // If this is an insert of an extracted vector into an undef vector, we
7192 // can just use the input to the extract.
7193 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7194 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT)
7195 return N2.getOperand(0);
7196 break;
7197 }
7198 case ISD::BITCAST:
7199 // Fold bit_convert nodes from a type to themselves.
7200 if (N1.getValueType() == VT)
7201 return N1;
7202 break;
7203 case ISD::VP_TRUNCATE:
7204 case ISD::VP_SIGN_EXTEND:
7205 case ISD::VP_ZERO_EXTEND:
7206 // Don't create noop casts.
7207 if (N1.getValueType() == VT)
7208 return N1;
7209 break;
7210 }
7211
7212 // Memoize node if it doesn't produce a glue result.
7213 SDNode *N;
7214 SDVTList VTs = getVTList(VT);
7215 SDValue Ops[] = {N1, N2, N3};
7216 if (VT != MVT::Glue) {
7217 FoldingSetNodeID ID;
7218 AddNodeIDNode(ID, Opcode, VTs, Ops);
7219 void *IP = nullptr;
7220 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7221 E->intersectFlagsWith(Flags);
7222 return SDValue(E, 0);
7223 }
7224
7225 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7226 N->setFlags(Flags);
7227 createOperands(N, Ops);
7228 CSEMap.InsertNode(N, IP);
7229 } else {
7230 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7231 createOperands(N, Ops);
7232 }
7233
7234 InsertNode(N);
7235 SDValue V = SDValue(N, 0);
7236 NewSDValueDbgMsg(V, "Creating new node: ", this);
7237 return V;
7238 }
7239
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,SDValue N1,SDValue N2,SDValue N3,SDValue N4)7240 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7241 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7242 SDValue Ops[] = { N1, N2, N3, N4 };
7243 return getNode(Opcode, DL, VT, Ops);
7244 }
7245
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,SDValue N1,SDValue N2,SDValue N3,SDValue N4,SDValue N5)7246 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7247 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7248 SDValue N5) {
7249 SDValue Ops[] = { N1, N2, N3, N4, N5 };
7250 return getNode(Opcode, DL, VT, Ops);
7251 }
7252
7253 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
7254 /// the incoming stack arguments to be loaded from the stack.
getStackArgumentTokenFactor(SDValue Chain)7255 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
7256 SmallVector<SDValue, 8> ArgChains;
7257
7258 // Include the original chain at the beginning of the list. When this is
7259 // used by target LowerCall hooks, this helps legalize find the
7260 // CALLSEQ_BEGIN node.
7261 ArgChains.push_back(Chain);
7262
7263 // Add a chain value for each stack argument.
7264 for (SDNode *U : getEntryNode().getNode()->uses())
7265 if (LoadSDNode *L = dyn_cast<LoadSDNode>(U))
7266 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
7267 if (FI->getIndex() < 0)
7268 ArgChains.push_back(SDValue(L, 1));
7269
7270 // Build a tokenfactor for all the chains.
7271 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
7272 }
7273
7274 /// getMemsetValue - Vectorized representation of the memset value
7275 /// operand.
getMemsetValue(SDValue Value,EVT VT,SelectionDAG & DAG,const SDLoc & dl)7276 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
7277 const SDLoc &dl) {
7278 assert(!Value.isUndef());
7279
7280 unsigned NumBits = VT.getScalarSizeInBits();
7281 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
7282 assert(C->getAPIntValue().getBitWidth() == 8);
7283 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
7284 if (VT.isInteger()) {
7285 bool IsOpaque = VT.getSizeInBits() > 64 ||
7286 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
7287 return DAG.getConstant(Val, dl, VT, false, IsOpaque);
7288 }
7289 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
7290 VT);
7291 }
7292
7293 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
7294 EVT IntVT = VT.getScalarType();
7295 if (!IntVT.isInteger())
7296 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
7297
7298 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
7299 if (NumBits > 8) {
7300 // Use a multiplication with 0x010101... to extend the input to the
7301 // required length.
7302 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
7303 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
7304 DAG.getConstant(Magic, dl, IntVT));
7305 }
7306
7307 if (VT != Value.getValueType() && !VT.isInteger())
7308 Value = DAG.getBitcast(VT.getScalarType(), Value);
7309 if (VT != Value.getValueType())
7310 Value = DAG.getSplatBuildVector(VT, dl, Value);
7311
7312 return Value;
7313 }
7314
7315 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
7316 /// used when a memcpy is turned into a memset when the source is a constant
7317 /// string ptr.
getMemsetStringVal(EVT VT,const SDLoc & dl,SelectionDAG & DAG,const TargetLowering & TLI,const ConstantDataArraySlice & Slice)7318 static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
7319 const TargetLowering &TLI,
7320 const ConstantDataArraySlice &Slice) {
7321 // Handle vector with all elements zero.
7322 if (Slice.Array == nullptr) {
7323 if (VT.isInteger())
7324 return DAG.getConstant(0, dl, VT);
7325 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
7326 return DAG.getConstantFP(0.0, dl, VT);
7327 if (VT.isVector()) {
7328 unsigned NumElts = VT.getVectorNumElements();
7329 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
7330 return DAG.getNode(ISD::BITCAST, dl, VT,
7331 DAG.getConstant(0, dl,
7332 EVT::getVectorVT(*DAG.getContext(),
7333 EltVT, NumElts)));
7334 }
7335 llvm_unreachable("Expected type!");
7336 }
7337
7338 assert(!VT.isVector() && "Can't handle vector type here!");
7339 unsigned NumVTBits = VT.getSizeInBits();
7340 unsigned NumVTBytes = NumVTBits / 8;
7341 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
7342
7343 APInt Val(NumVTBits, 0);
7344 if (DAG.getDataLayout().isLittleEndian()) {
7345 for (unsigned i = 0; i != NumBytes; ++i)
7346 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
7347 } else {
7348 for (unsigned i = 0; i != NumBytes; ++i)
7349 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
7350 }
7351
7352 // If the "cost" of materializing the integer immediate is less than the cost
7353 // of a load, then it is cost effective to turn the load into the immediate.
7354 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
7355 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
7356 return DAG.getConstant(Val, dl, VT);
7357 return SDValue();
7358 }
7359
getMemBasePlusOffset(SDValue Base,TypeSize Offset,const SDLoc & DL,const SDNodeFlags Flags)7360 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
7361 const SDLoc &DL,
7362 const SDNodeFlags Flags) {
7363 EVT VT = Base.getValueType();
7364 SDValue Index;
7365
7366 if (Offset.isScalable())
7367 Index = getVScale(DL, Base.getValueType(),
7368 APInt(Base.getValueSizeInBits().getFixedValue(),
7369 Offset.getKnownMinValue()));
7370 else
7371 Index = getConstant(Offset.getFixedValue(), DL, VT);
7372
7373 return getMemBasePlusOffset(Base, Index, DL, Flags);
7374 }
7375
getMemBasePlusOffset(SDValue Ptr,SDValue Offset,const SDLoc & DL,const SDNodeFlags Flags)7376 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
7377 const SDLoc &DL,
7378 const SDNodeFlags Flags) {
7379 assert(Offset.getValueType().isInteger());
7380 EVT BasePtrVT = Ptr.getValueType();
7381 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags);
7382 }
7383
7384 /// Returns true if memcpy source is constant data.
isMemSrcFromConstant(SDValue Src,ConstantDataArraySlice & Slice)7385 static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
7386 uint64_t SrcDelta = 0;
7387 GlobalAddressSDNode *G = nullptr;
7388 if (Src.getOpcode() == ISD::GlobalAddress)
7389 G = cast<GlobalAddressSDNode>(Src);
7390 else if (Src.getOpcode() == ISD::ADD &&
7391 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
7392 Src.getOperand(1).getOpcode() == ISD::Constant) {
7393 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
7394 SrcDelta = Src.getConstantOperandVal(1);
7395 }
7396 if (!G)
7397 return false;
7398
7399 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
7400 SrcDelta + G->getOffset());
7401 }
7402
shouldLowerMemFuncForSize(const MachineFunction & MF,SelectionDAG & DAG)7403 static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
7404 SelectionDAG &DAG) {
7405 // On Darwin, -Os means optimize for size without hurting performance, so
7406 // only really optimize for size when -Oz (MinSize) is used.
7407 if (MF.getTarget().getTargetTriple().isOSDarwin())
7408 return MF.getFunction().hasMinSize();
7409 return DAG.shouldOptForSize();
7410 }
7411
chainLoadsAndStoresForMemcpy(SelectionDAG & DAG,const SDLoc & dl,SmallVector<SDValue,32> & OutChains,unsigned From,unsigned To,SmallVector<SDValue,16> & OutLoadChains,SmallVector<SDValue,16> & OutStoreChains)7412 static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
7413 SmallVector<SDValue, 32> &OutChains, unsigned From,
7414 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
7415 SmallVector<SDValue, 16> &OutStoreChains) {
7416 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
7417 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
7418 SmallVector<SDValue, 16> GluedLoadChains;
7419 for (unsigned i = From; i < To; ++i) {
7420 OutChains.push_back(OutLoadChains[i]);
7421 GluedLoadChains.push_back(OutLoadChains[i]);
7422 }
7423
7424 // Chain for all loads.
7425 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
7426 GluedLoadChains);
7427
7428 for (unsigned i = From; i < To; ++i) {
7429 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
7430 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
7431 ST->getBasePtr(), ST->getMemoryVT(),
7432 ST->getMemOperand());
7433 OutChains.push_back(NewStore);
7434 }
7435 }
7436
getMemcpyLoadsAndStores(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Dst,SDValue Src,uint64_t Size,Align Alignment,bool isVol,bool AlwaysInline,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo,const AAMDNodes & AAInfo,AAResults * AA)7437 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
7438 SDValue Chain, SDValue Dst, SDValue Src,
7439 uint64_t Size, Align Alignment,
7440 bool isVol, bool AlwaysInline,
7441 MachinePointerInfo DstPtrInfo,
7442 MachinePointerInfo SrcPtrInfo,
7443 const AAMDNodes &AAInfo, AAResults *AA) {
7444 // Turn a memcpy of undef to nop.
7445 // FIXME: We need to honor volatile even is Src is undef.
7446 if (Src.isUndef())
7447 return Chain;
7448
7449 // Expand memcpy to a series of load and store ops if the size operand falls
7450 // below a certain threshold.
7451 // TODO: In the AlwaysInline case, if the size is big then generate a loop
7452 // rather than maybe a humongous number of loads and stores.
7453 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7454 const DataLayout &DL = DAG.getDataLayout();
7455 LLVMContext &C = *DAG.getContext();
7456 std::vector<EVT> MemOps;
7457 bool DstAlignCanChange = false;
7458 MachineFunction &MF = DAG.getMachineFunction();
7459 MachineFrameInfo &MFI = MF.getFrameInfo();
7460 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7461 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7462 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7463 DstAlignCanChange = true;
7464 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
7465 if (!SrcAlign || Alignment > *SrcAlign)
7466 SrcAlign = Alignment;
7467 assert(SrcAlign && "SrcAlign must be set");
7468 ConstantDataArraySlice Slice;
7469 // If marked as volatile, perform a copy even when marked as constant.
7470 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
7471 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
7472 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
7473 const MemOp Op = isZeroConstant
7474 ? MemOp::Set(Size, DstAlignCanChange, Alignment,
7475 /*IsZeroMemset*/ true, isVol)
7476 : MemOp::Copy(Size, DstAlignCanChange, Alignment,
7477 *SrcAlign, isVol, CopyFromConstant);
7478 if (!TLI.findOptimalMemOpLowering(
7479 MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
7480 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
7481 return SDValue();
7482
7483 if (DstAlignCanChange) {
7484 Type *Ty = MemOps[0].getTypeForEVT(C);
7485 Align NewAlign = DL.getABITypeAlign(Ty);
7486
7487 // Don't promote to an alignment that would require dynamic stack
7488 // realignment which may conflict with optimizations such as tail call
7489 // optimization.
7490 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7491 if (!TRI->hasStackRealignment(MF))
7492 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7493 NewAlign = NewAlign.previous();
7494
7495 if (NewAlign > Alignment) {
7496 // Give the stack frame object a larger alignment if needed.
7497 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7498 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7499 Alignment = NewAlign;
7500 }
7501 }
7502
7503 // Prepare AAInfo for loads/stores after lowering this memcpy.
7504 AAMDNodes NewAAInfo = AAInfo;
7505 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7506
7507 const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);
7508 bool isConstant =
7509 AA && SrcVal &&
7510 AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));
7511
7512 MachineMemOperand::Flags MMOFlags =
7513 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
7514 SmallVector<SDValue, 16> OutLoadChains;
7515 SmallVector<SDValue, 16> OutStoreChains;
7516 SmallVector<SDValue, 32> OutChains;
7517 unsigned NumMemOps = MemOps.size();
7518 uint64_t SrcOff = 0, DstOff = 0;
7519 for (unsigned i = 0; i != NumMemOps; ++i) {
7520 EVT VT = MemOps[i];
7521 unsigned VTSize = VT.getSizeInBits() / 8;
7522 SDValue Value, Store;
7523
7524 if (VTSize > Size) {
7525 // Issuing an unaligned load / store pair that overlaps with the previous
7526 // pair. Adjust the offset accordingly.
7527 assert(i == NumMemOps-1 && i != 0);
7528 SrcOff -= VTSize - Size;
7529 DstOff -= VTSize - Size;
7530 }
7531
7532 if (CopyFromConstant &&
7533 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
7534 // It's unlikely a store of a vector immediate can be done in a single
7535 // instruction. It would require a load from a constantpool first.
7536 // We only handle zero vectors here.
7537 // FIXME: Handle other cases where store of vector immediate is done in
7538 // a single instruction.
7539 ConstantDataArraySlice SubSlice;
7540 if (SrcOff < Slice.Length) {
7541 SubSlice = Slice;
7542 SubSlice.move(SrcOff);
7543 } else {
7544 // This is an out-of-bounds access and hence UB. Pretend we read zero.
7545 SubSlice.Array = nullptr;
7546 SubSlice.Offset = 0;
7547 SubSlice.Length = VTSize;
7548 }
7549 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
7550 if (Value.getNode()) {
7551 Store = DAG.getStore(
7552 Chain, dl, Value,
7553 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7554 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7555 OutChains.push_back(Store);
7556 }
7557 }
7558
7559 if (!Store.getNode()) {
7560 // The type might not be legal for the target. This should only happen
7561 // if the type is smaller than a legal type, as on PPC, so the right
7562 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
7563 // to Load/Store if NVT==VT.
7564 // FIXME does the case above also need this?
7565 EVT NVT = TLI.getTypeToTransformTo(C, VT);
7566 assert(NVT.bitsGE(VT));
7567
7568 bool isDereferenceable =
7569 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
7570 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7571 if (isDereferenceable)
7572 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
7573 if (isConstant)
7574 SrcMMOFlags |= MachineMemOperand::MOInvariant;
7575
7576 Value = DAG.getExtLoad(
7577 ISD::EXTLOAD, dl, NVT, Chain,
7578 DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
7579 SrcPtrInfo.getWithOffset(SrcOff), VT,
7580 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
7581 OutLoadChains.push_back(Value.getValue(1));
7582
7583 Store = DAG.getTruncStore(
7584 Chain, dl, Value,
7585 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7586 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
7587 OutStoreChains.push_back(Store);
7588 }
7589 SrcOff += VTSize;
7590 DstOff += VTSize;
7591 Size -= VTSize;
7592 }
7593
7594 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
7595 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
7596 unsigned NumLdStInMemcpy = OutStoreChains.size();
7597
7598 if (NumLdStInMemcpy) {
7599 // It may be that memcpy might be converted to memset if it's memcpy
7600 // of constants. In such a case, we won't have loads and stores, but
7601 // just stores. In the absence of loads, there is nothing to gang up.
7602 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
7603 // If target does not care, just leave as it.
7604 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
7605 OutChains.push_back(OutLoadChains[i]);
7606 OutChains.push_back(OutStoreChains[i]);
7607 }
7608 } else {
7609 // Ld/St less than/equal limit set by target.
7610 if (NumLdStInMemcpy <= GluedLdStLimit) {
7611 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
7612 NumLdStInMemcpy, OutLoadChains,
7613 OutStoreChains);
7614 } else {
7615 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
7616 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
7617 unsigned GlueIter = 0;
7618
7619 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
7620 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
7621 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
7622
7623 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
7624 OutLoadChains, OutStoreChains);
7625 GlueIter += GluedLdStLimit;
7626 }
7627
7628 // Residual ld/st.
7629 if (RemainingLdStInMemcpy) {
7630 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
7631 RemainingLdStInMemcpy, OutLoadChains,
7632 OutStoreChains);
7633 }
7634 }
7635 }
7636 }
7637 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7638 }
7639
getMemmoveLoadsAndStores(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Dst,SDValue Src,uint64_t Size,Align Alignment,bool isVol,bool AlwaysInline,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo,const AAMDNodes & AAInfo)7640 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
7641 SDValue Chain, SDValue Dst, SDValue Src,
7642 uint64_t Size, Align Alignment,
7643 bool isVol, bool AlwaysInline,
7644 MachinePointerInfo DstPtrInfo,
7645 MachinePointerInfo SrcPtrInfo,
7646 const AAMDNodes &AAInfo) {
7647 // Turn a memmove of undef to nop.
7648 // FIXME: We need to honor volatile even is Src is undef.
7649 if (Src.isUndef())
7650 return Chain;
7651
7652 // Expand memmove to a series of load and store ops if the size operand falls
7653 // below a certain threshold.
7654 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7655 const DataLayout &DL = DAG.getDataLayout();
7656 LLVMContext &C = *DAG.getContext();
7657 std::vector<EVT> MemOps;
7658 bool DstAlignCanChange = false;
7659 MachineFunction &MF = DAG.getMachineFunction();
7660 MachineFrameInfo &MFI = MF.getFrameInfo();
7661 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7662 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7663 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7664 DstAlignCanChange = true;
7665 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
7666 if (!SrcAlign || Alignment > *SrcAlign)
7667 SrcAlign = Alignment;
7668 assert(SrcAlign && "SrcAlign must be set");
7669 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
7670 if (!TLI.findOptimalMemOpLowering(
7671 MemOps, Limit,
7672 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
7673 /*IsVolatile*/ true),
7674 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
7675 MF.getFunction().getAttributes()))
7676 return SDValue();
7677
7678 if (DstAlignCanChange) {
7679 Type *Ty = MemOps[0].getTypeForEVT(C);
7680 Align NewAlign = DL.getABITypeAlign(Ty);
7681
7682 // Don't promote to an alignment that would require dynamic stack
7683 // realignment which may conflict with optimizations such as tail call
7684 // optimization.
7685 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7686 if (!TRI->hasStackRealignment(MF))
7687 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7688 NewAlign = NewAlign.previous();
7689
7690 if (NewAlign > Alignment) {
7691 // Give the stack frame object a larger alignment if needed.
7692 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7693 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7694 Alignment = NewAlign;
7695 }
7696 }
7697
7698 // Prepare AAInfo for loads/stores after lowering this memmove.
7699 AAMDNodes NewAAInfo = AAInfo;
7700 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7701
7702 MachineMemOperand::Flags MMOFlags =
7703 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
7704 uint64_t SrcOff = 0, DstOff = 0;
7705 SmallVector<SDValue, 8> LoadValues;
7706 SmallVector<SDValue, 8> LoadChains;
7707 SmallVector<SDValue, 8> OutChains;
7708 unsigned NumMemOps = MemOps.size();
7709 for (unsigned i = 0; i < NumMemOps; i++) {
7710 EVT VT = MemOps[i];
7711 unsigned VTSize = VT.getSizeInBits() / 8;
7712 SDValue Value;
7713
7714 bool isDereferenceable =
7715 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
7716 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7717 if (isDereferenceable)
7718 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
7719
7720 Value = DAG.getLoad(
7721 VT, dl, Chain,
7722 DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
7723 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
7724 LoadValues.push_back(Value);
7725 LoadChains.push_back(Value.getValue(1));
7726 SrcOff += VTSize;
7727 }
7728 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
7729 OutChains.clear();
7730 for (unsigned i = 0; i < NumMemOps; i++) {
7731 EVT VT = MemOps[i];
7732 unsigned VTSize = VT.getSizeInBits() / 8;
7733 SDValue Store;
7734
7735 Store = DAG.getStore(
7736 Chain, dl, LoadValues[i],
7737 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7738 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7739 OutChains.push_back(Store);
7740 DstOff += VTSize;
7741 }
7742
7743 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7744 }
7745
7746 /// Lower the call to 'memset' intrinsic function into a series of store
7747 /// operations.
7748 ///
7749 /// \param DAG Selection DAG where lowered code is placed.
7750 /// \param dl Link to corresponding IR location.
7751 /// \param Chain Control flow dependency.
7752 /// \param Dst Pointer to destination memory location.
7753 /// \param Src Value of byte to write into the memory.
7754 /// \param Size Number of bytes to write.
7755 /// \param Alignment Alignment of the destination in bytes.
7756 /// \param isVol True if destination is volatile.
7757 /// \param AlwaysInline Makes sure no function call is generated.
7758 /// \param DstPtrInfo IR information on the memory pointer.
7759 /// \returns New head in the control flow, if lowering was successful, empty
7760 /// SDValue otherwise.
7761 ///
7762 /// The function tries to replace 'llvm.memset' intrinsic with several store
7763 /// operations and value calculation code. This is usually profitable for small
7764 /// memory size or when the semantic requires inlining.
getMemsetStores(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Dst,SDValue Src,uint64_t Size,Align Alignment,bool isVol,bool AlwaysInline,MachinePointerInfo DstPtrInfo,const AAMDNodes & AAInfo)7765 static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
7766 SDValue Chain, SDValue Dst, SDValue Src,
7767 uint64_t Size, Align Alignment, bool isVol,
7768 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
7769 const AAMDNodes &AAInfo) {
7770 // Turn a memset of undef to nop.
7771 // FIXME: We need to honor volatile even is Src is undef.
7772 if (Src.isUndef())
7773 return Chain;
7774
7775 // Expand memset to a series of load/store ops if the size operand
7776 // falls below a certain threshold.
7777 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7778 std::vector<EVT> MemOps;
7779 bool DstAlignCanChange = false;
7780 MachineFunction &MF = DAG.getMachineFunction();
7781 MachineFrameInfo &MFI = MF.getFrameInfo();
7782 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7783 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7784 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7785 DstAlignCanChange = true;
7786 bool IsZeroVal = isNullConstant(Src);
7787 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
7788
7789 if (!TLI.findOptimalMemOpLowering(
7790 MemOps, Limit,
7791 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
7792 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
7793 return SDValue();
7794
7795 if (DstAlignCanChange) {
7796 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
7797 const DataLayout &DL = DAG.getDataLayout();
7798 Align NewAlign = DL.getABITypeAlign(Ty);
7799
7800 // Don't promote to an alignment that would require dynamic stack
7801 // realignment which may conflict with optimizations such as tail call
7802 // optimization.
7803 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7804 if (!TRI->hasStackRealignment(MF))
7805 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7806 NewAlign = NewAlign.previous();
7807
7808 if (NewAlign > Alignment) {
7809 // Give the stack frame object a larger alignment if needed.
7810 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7811 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7812 Alignment = NewAlign;
7813 }
7814 }
7815
7816 SmallVector<SDValue, 8> OutChains;
7817 uint64_t DstOff = 0;
7818 unsigned NumMemOps = MemOps.size();
7819
7820 // Find the largest store and generate the bit pattern for it.
7821 EVT LargestVT = MemOps[0];
7822 for (unsigned i = 1; i < NumMemOps; i++)
7823 if (MemOps[i].bitsGT(LargestVT))
7824 LargestVT = MemOps[i];
7825 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
7826
7827 // Prepare AAInfo for loads/stores after lowering this memset.
7828 AAMDNodes NewAAInfo = AAInfo;
7829 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7830
7831 for (unsigned i = 0; i < NumMemOps; i++) {
7832 EVT VT = MemOps[i];
7833 unsigned VTSize = VT.getSizeInBits() / 8;
7834 if (VTSize > Size) {
7835 // Issuing an unaligned load / store pair that overlaps with the previous
7836 // pair. Adjust the offset accordingly.
7837 assert(i == NumMemOps-1 && i != 0);
7838 DstOff -= VTSize - Size;
7839 }
7840
7841 // If this store is smaller than the largest store see whether we can get
7842 // the smaller value for free with a truncate or extract vector element and
7843 // then store.
7844 SDValue Value = MemSetValue;
7845 if (VT.bitsLT(LargestVT)) {
7846 unsigned Index;
7847 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
7848 EVT SVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), NElts);
7849 if (!LargestVT.isVector() && !VT.isVector() &&
7850 TLI.isTruncateFree(LargestVT, VT))
7851 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
7852 else if (LargestVT.isVector() && !VT.isVector() &&
7853 TLI.shallExtractConstSplatVectorElementToStore(
7854 LargestVT.getTypeForEVT(*DAG.getContext()),
7855 VT.getSizeInBits(), Index) &&
7856 TLI.isTypeLegal(SVT) &&
7857 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
7858 // Target which can combine store(extractelement VectorTy, Idx) can get
7859 // the smaller value for free.
7860 SDValue TailValue = DAG.getNode(ISD::BITCAST, dl, SVT, MemSetValue);
7861 Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, TailValue,
7862 DAG.getVectorIdxConstant(Index, dl));
7863 } else
7864 Value = getMemsetValue(Src, VT, DAG, dl);
7865 }
7866 assert(Value.getValueType() == VT && "Value with wrong type.");
7867 SDValue Store = DAG.getStore(
7868 Chain, dl, Value,
7869 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7870 DstPtrInfo.getWithOffset(DstOff), Alignment,
7871 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
7872 NewAAInfo);
7873 OutChains.push_back(Store);
7874 DstOff += VT.getSizeInBits() / 8;
7875 Size -= VTSize;
7876 }
7877
7878 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7879 }
7880
checkAddrSpaceIsValidForLibcall(const TargetLowering * TLI,unsigned AS)7881 static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
7882 unsigned AS) {
7883 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
7884 // pointer operands can be losslessly bitcasted to pointers of address space 0
7885 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
7886 report_fatal_error("cannot lower memory intrinsic in address space " +
7887 Twine(AS));
7888 }
7889 }
7890
getMemcpy(SDValue Chain,const SDLoc & dl,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVol,bool AlwaysInline,bool isTailCall,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo,const AAMDNodes & AAInfo,AAResults * AA)7891 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
7892 SDValue Src, SDValue Size, Align Alignment,
7893 bool isVol, bool AlwaysInline, bool isTailCall,
7894 MachinePointerInfo DstPtrInfo,
7895 MachinePointerInfo SrcPtrInfo,
7896 const AAMDNodes &AAInfo, AAResults *AA) {
7897 // Check to see if we should lower the memcpy to loads and stores first.
7898 // For cases within the target-specified limits, this is the best choice.
7899 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
7900 if (ConstantSize) {
7901 // Memcpy with size zero? Just return the original chain.
7902 if (ConstantSize->isZero())
7903 return Chain;
7904
7905 SDValue Result = getMemcpyLoadsAndStores(
7906 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
7907 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
7908 if (Result.getNode())
7909 return Result;
7910 }
7911
7912 // Then check to see if we should lower the memcpy with target-specific
7913 // code. If the target chooses to do this, this is the next best.
7914 if (TSI) {
7915 SDValue Result = TSI->EmitTargetCodeForMemcpy(
7916 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
7917 DstPtrInfo, SrcPtrInfo);
7918 if (Result.getNode())
7919 return Result;
7920 }
7921
7922 // If we really need inline code and the target declined to provide it,
7923 // use a (potentially long) sequence of loads and stores.
7924 if (AlwaysInline) {
7925 assert(ConstantSize && "AlwaysInline requires a constant size!");
7926 return getMemcpyLoadsAndStores(
7927 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
7928 isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
7929 }
7930
7931 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
7932 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
7933
7934 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
7935 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
7936 // respect volatile, so they may do things like read or write memory
7937 // beyond the given memory regions. But fixing this isn't easy, and most
7938 // people don't care.
7939
7940 // Emit a library call.
7941 TargetLowering::ArgListTy Args;
7942 TargetLowering::ArgListEntry Entry;
7943 Entry.Ty = PointerType::getUnqual(*getContext());
7944 Entry.Node = Dst; Args.push_back(Entry);
7945 Entry.Node = Src; Args.push_back(Entry);
7946
7947 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7948 Entry.Node = Size; Args.push_back(Entry);
7949 // FIXME: pass in SDLoc
7950 TargetLowering::CallLoweringInfo CLI(*this);
7951 CLI.setDebugLoc(dl)
7952 .setChain(Chain)
7953 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
7954 Dst.getValueType().getTypeForEVT(*getContext()),
7955 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
7956 TLI->getPointerTy(getDataLayout())),
7957 std::move(Args))
7958 .setDiscardResult()
7959 .setTailCall(isTailCall);
7960
7961 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
7962 return CallResult.second;
7963 }
7964
getAtomicMemcpy(SDValue Chain,const SDLoc & dl,SDValue Dst,SDValue Src,SDValue Size,Type * SizeTy,unsigned ElemSz,bool isTailCall,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo)7965 SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
7966 SDValue Dst, SDValue Src, SDValue Size,
7967 Type *SizeTy, unsigned ElemSz,
7968 bool isTailCall,
7969 MachinePointerInfo DstPtrInfo,
7970 MachinePointerInfo SrcPtrInfo) {
7971 // Emit a library call.
7972 TargetLowering::ArgListTy Args;
7973 TargetLowering::ArgListEntry Entry;
7974 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7975 Entry.Node = Dst;
7976 Args.push_back(Entry);
7977
7978 Entry.Node = Src;
7979 Args.push_back(Entry);
7980
7981 Entry.Ty = SizeTy;
7982 Entry.Node = Size;
7983 Args.push_back(Entry);
7984
7985 RTLIB::Libcall LibraryCall =
7986 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);
7987 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
7988 report_fatal_error("Unsupported element size");
7989
7990 TargetLowering::CallLoweringInfo CLI(*this);
7991 CLI.setDebugLoc(dl)
7992 .setChain(Chain)
7993 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
7994 Type::getVoidTy(*getContext()),
7995 getExternalSymbol(TLI->getLibcallName(LibraryCall),
7996 TLI->getPointerTy(getDataLayout())),
7997 std::move(Args))
7998 .setDiscardResult()
7999 .setTailCall(isTailCall);
8000
8001 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8002 return CallResult.second;
8003 }
8004
getMemmove(SDValue Chain,const SDLoc & dl,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVol,bool isTailCall,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo,const AAMDNodes & AAInfo,AAResults * AA)8005 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
8006 SDValue Src, SDValue Size, Align Alignment,
8007 bool isVol, bool isTailCall,
8008 MachinePointerInfo DstPtrInfo,
8009 MachinePointerInfo SrcPtrInfo,
8010 const AAMDNodes &AAInfo, AAResults *AA) {
8011 // Check to see if we should lower the memmove to loads and stores first.
8012 // For cases within the target-specified limits, this is the best choice.
8013 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8014 if (ConstantSize) {
8015 // Memmove with size zero? Just return the original chain.
8016 if (ConstantSize->isZero())
8017 return Chain;
8018
8019 SDValue Result = getMemmoveLoadsAndStores(
8020 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8021 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
8022 if (Result.getNode())
8023 return Result;
8024 }
8025
8026 // Then check to see if we should lower the memmove with target-specific
8027 // code. If the target chooses to do this, this is the next best.
8028 if (TSI) {
8029 SDValue Result =
8030 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
8031 Alignment, isVol, DstPtrInfo, SrcPtrInfo);
8032 if (Result.getNode())
8033 return Result;
8034 }
8035
8036 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
8037 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
8038
8039 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
8040 // not be safe. See memcpy above for more details.
8041
8042 // Emit a library call.
8043 TargetLowering::ArgListTy Args;
8044 TargetLowering::ArgListEntry Entry;
8045 Entry.Ty = PointerType::getUnqual(*getContext());
8046 Entry.Node = Dst; Args.push_back(Entry);
8047 Entry.Node = Src; Args.push_back(Entry);
8048
8049 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8050 Entry.Node = Size; Args.push_back(Entry);
8051 // FIXME: pass in SDLoc
8052 TargetLowering::CallLoweringInfo CLI(*this);
8053 CLI.setDebugLoc(dl)
8054 .setChain(Chain)
8055 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
8056 Dst.getValueType().getTypeForEVT(*getContext()),
8057 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
8058 TLI->getPointerTy(getDataLayout())),
8059 std::move(Args))
8060 .setDiscardResult()
8061 .setTailCall(isTailCall);
8062
8063 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8064 return CallResult.second;
8065 }
8066
getAtomicMemmove(SDValue Chain,const SDLoc & dl,SDValue Dst,SDValue Src,SDValue Size,Type * SizeTy,unsigned ElemSz,bool isTailCall,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo)8067 SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
8068 SDValue Dst, SDValue Src, SDValue Size,
8069 Type *SizeTy, unsigned ElemSz,
8070 bool isTailCall,
8071 MachinePointerInfo DstPtrInfo,
8072 MachinePointerInfo SrcPtrInfo) {
8073 // Emit a library call.
8074 TargetLowering::ArgListTy Args;
8075 TargetLowering::ArgListEntry Entry;
8076 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8077 Entry.Node = Dst;
8078 Args.push_back(Entry);
8079
8080 Entry.Node = Src;
8081 Args.push_back(Entry);
8082
8083 Entry.Ty = SizeTy;
8084 Entry.Node = Size;
8085 Args.push_back(Entry);
8086
8087 RTLIB::Libcall LibraryCall =
8088 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);
8089 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8090 report_fatal_error("Unsupported element size");
8091
8092 TargetLowering::CallLoweringInfo CLI(*this);
8093 CLI.setDebugLoc(dl)
8094 .setChain(Chain)
8095 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8096 Type::getVoidTy(*getContext()),
8097 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8098 TLI->getPointerTy(getDataLayout())),
8099 std::move(Args))
8100 .setDiscardResult()
8101 .setTailCall(isTailCall);
8102
8103 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8104 return CallResult.second;
8105 }
8106
getMemset(SDValue Chain,const SDLoc & dl,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVol,bool AlwaysInline,bool isTailCall,MachinePointerInfo DstPtrInfo,const AAMDNodes & AAInfo)8107 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
8108 SDValue Src, SDValue Size, Align Alignment,
8109 bool isVol, bool AlwaysInline, bool isTailCall,
8110 MachinePointerInfo DstPtrInfo,
8111 const AAMDNodes &AAInfo) {
8112 // Check to see if we should lower the memset to stores first.
8113 // For cases within the target-specified limits, this is the best choice.
8114 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8115 if (ConstantSize) {
8116 // Memset with size zero? Just return the original chain.
8117 if (ConstantSize->isZero())
8118 return Chain;
8119
8120 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8121 ConstantSize->getZExtValue(), Alignment,
8122 isVol, false, DstPtrInfo, AAInfo);
8123
8124 if (Result.getNode())
8125 return Result;
8126 }
8127
8128 // Then check to see if we should lower the memset with target-specific
8129 // code. If the target chooses to do this, this is the next best.
8130 if (TSI) {
8131 SDValue Result = TSI->EmitTargetCodeForMemset(
8132 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, DstPtrInfo);
8133 if (Result.getNode())
8134 return Result;
8135 }
8136
8137 // If we really need inline code and the target declined to provide it,
8138 // use a (potentially long) sequence of loads and stores.
8139 if (AlwaysInline) {
8140 assert(ConstantSize && "AlwaysInline requires a constant size!");
8141 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8142 ConstantSize->getZExtValue(), Alignment,
8143 isVol, true, DstPtrInfo, AAInfo);
8144 assert(Result &&
8145 "getMemsetStores must return a valid sequence when AlwaysInline");
8146 return Result;
8147 }
8148
8149 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
8150
8151 // Emit a library call.
8152 auto &Ctx = *getContext();
8153 const auto& DL = getDataLayout();
8154
8155 TargetLowering::CallLoweringInfo CLI(*this);
8156 // FIXME: pass in SDLoc
8157 CLI.setDebugLoc(dl).setChain(Chain);
8158
8159 const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO);
8160
8161 // Helper function to create an Entry from Node and Type.
8162 const auto CreateEntry = [](SDValue Node, Type *Ty) {
8163 TargetLowering::ArgListEntry Entry;
8164 Entry.Node = Node;
8165 Entry.Ty = Ty;
8166 return Entry;
8167 };
8168
8169 // If zeroing out and bzero is present, use it.
8170 if (isNullConstant(Src) && BzeroName) {
8171 TargetLowering::ArgListTy Args;
8172 Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8173 Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8174 CLI.setLibCallee(
8175 TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx),
8176 getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args));
8177 } else {
8178 TargetLowering::ArgListTy Args;
8179 Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8180 Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx)));
8181 Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8182 CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
8183 Dst.getValueType().getTypeForEVT(Ctx),
8184 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
8185 TLI->getPointerTy(DL)),
8186 std::move(Args));
8187 }
8188
8189 CLI.setDiscardResult().setTailCall(isTailCall);
8190
8191 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8192 return CallResult.second;
8193 }
8194
getAtomicMemset(SDValue Chain,const SDLoc & dl,SDValue Dst,SDValue Value,SDValue Size,Type * SizeTy,unsigned ElemSz,bool isTailCall,MachinePointerInfo DstPtrInfo)8195 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
8196 SDValue Dst, SDValue Value, SDValue Size,
8197 Type *SizeTy, unsigned ElemSz,
8198 bool isTailCall,
8199 MachinePointerInfo DstPtrInfo) {
8200 // Emit a library call.
8201 TargetLowering::ArgListTy Args;
8202 TargetLowering::ArgListEntry Entry;
8203 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8204 Entry.Node = Dst;
8205 Args.push_back(Entry);
8206
8207 Entry.Ty = Type::getInt8Ty(*getContext());
8208 Entry.Node = Value;
8209 Args.push_back(Entry);
8210
8211 Entry.Ty = SizeTy;
8212 Entry.Node = Size;
8213 Args.push_back(Entry);
8214
8215 RTLIB::Libcall LibraryCall =
8216 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);
8217 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8218 report_fatal_error("Unsupported element size");
8219
8220 TargetLowering::CallLoweringInfo CLI(*this);
8221 CLI.setDebugLoc(dl)
8222 .setChain(Chain)
8223 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8224 Type::getVoidTy(*getContext()),
8225 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8226 TLI->getPointerTy(getDataLayout())),
8227 std::move(Args))
8228 .setDiscardResult()
8229 .setTailCall(isTailCall);
8230
8231 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8232 return CallResult.second;
8233 }
8234
getAtomic(unsigned Opcode,const SDLoc & dl,EVT MemVT,SDVTList VTList,ArrayRef<SDValue> Ops,MachineMemOperand * MMO)8235 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8236 SDVTList VTList, ArrayRef<SDValue> Ops,
8237 MachineMemOperand *MMO) {
8238 FoldingSetNodeID ID;
8239 ID.AddInteger(MemVT.getRawBits());
8240 AddNodeIDNode(ID, Opcode, VTList, Ops);
8241 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8242 ID.AddInteger(MMO->getFlags());
8243 void* IP = nullptr;
8244 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8245 cast<AtomicSDNode>(E)->refineAlignment(MMO);
8246 return SDValue(E, 0);
8247 }
8248
8249 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8250 VTList, MemVT, MMO);
8251 createOperands(N, Ops);
8252
8253 CSEMap.InsertNode(N, IP);
8254 InsertNode(N);
8255 return SDValue(N, 0);
8256 }
8257
getAtomicCmpSwap(unsigned Opcode,const SDLoc & dl,EVT MemVT,SDVTList VTs,SDValue Chain,SDValue Ptr,SDValue Cmp,SDValue Swp,MachineMemOperand * MMO)8258 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
8259 EVT MemVT, SDVTList VTs, SDValue Chain,
8260 SDValue Ptr, SDValue Cmp, SDValue Swp,
8261 MachineMemOperand *MMO) {
8262 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
8263 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
8264 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
8265
8266 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
8267 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8268 }
8269
getAtomic(unsigned Opcode,const SDLoc & dl,EVT MemVT,SDValue Chain,SDValue Ptr,SDValue Val,MachineMemOperand * MMO)8270 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8271 SDValue Chain, SDValue Ptr, SDValue Val,
8272 MachineMemOperand *MMO) {
8273 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
8274 Opcode == ISD::ATOMIC_LOAD_SUB ||
8275 Opcode == ISD::ATOMIC_LOAD_AND ||
8276 Opcode == ISD::ATOMIC_LOAD_CLR ||
8277 Opcode == ISD::ATOMIC_LOAD_OR ||
8278 Opcode == ISD::ATOMIC_LOAD_XOR ||
8279 Opcode == ISD::ATOMIC_LOAD_NAND ||
8280 Opcode == ISD::ATOMIC_LOAD_MIN ||
8281 Opcode == ISD::ATOMIC_LOAD_MAX ||
8282 Opcode == ISD::ATOMIC_LOAD_UMIN ||
8283 Opcode == ISD::ATOMIC_LOAD_UMAX ||
8284 Opcode == ISD::ATOMIC_LOAD_FADD ||
8285 Opcode == ISD::ATOMIC_LOAD_FSUB ||
8286 Opcode == ISD::ATOMIC_LOAD_FMAX ||
8287 Opcode == ISD::ATOMIC_LOAD_FMIN ||
8288 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
8289 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
8290 Opcode == ISD::ATOMIC_SWAP ||
8291 Opcode == ISD::ATOMIC_STORE) &&
8292 "Invalid Atomic Op");
8293
8294 EVT VT = Val.getValueType();
8295
8296 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
8297 getVTList(VT, MVT::Other);
8298 SDValue Ops[] = {Chain, Ptr, Val};
8299 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8300 }
8301
getAtomic(unsigned Opcode,const SDLoc & dl,EVT MemVT,EVT VT,SDValue Chain,SDValue Ptr,MachineMemOperand * MMO)8302 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8303 EVT VT, SDValue Chain, SDValue Ptr,
8304 MachineMemOperand *MMO) {
8305 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
8306
8307 SDVTList VTs = getVTList(VT, MVT::Other);
8308 SDValue Ops[] = {Chain, Ptr};
8309 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8310 }
8311
8312 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
getMergeValues(ArrayRef<SDValue> Ops,const SDLoc & dl)8313 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
8314 if (Ops.size() == 1)
8315 return Ops[0];
8316
8317 SmallVector<EVT, 4> VTs;
8318 VTs.reserve(Ops.size());
8319 for (const SDValue &Op : Ops)
8320 VTs.push_back(Op.getValueType());
8321 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
8322 }
8323
getMemIntrinsicNode(unsigned Opcode,const SDLoc & dl,SDVTList VTList,ArrayRef<SDValue> Ops,EVT MemVT,MachinePointerInfo PtrInfo,Align Alignment,MachineMemOperand::Flags Flags,uint64_t Size,const AAMDNodes & AAInfo)8324 SDValue SelectionDAG::getMemIntrinsicNode(
8325 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
8326 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
8327 MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
8328 if (!Size && MemVT.isScalableVector())
8329 Size = MemoryLocation::UnknownSize;
8330 else if (!Size)
8331 Size = MemVT.getStoreSize();
8332
8333 MachineFunction &MF = getMachineFunction();
8334 MachineMemOperand *MMO =
8335 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
8336
8337 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
8338 }
8339
getMemIntrinsicNode(unsigned Opcode,const SDLoc & dl,SDVTList VTList,ArrayRef<SDValue> Ops,EVT MemVT,MachineMemOperand * MMO)8340 SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
8341 SDVTList VTList,
8342 ArrayRef<SDValue> Ops, EVT MemVT,
8343 MachineMemOperand *MMO) {
8344 assert((Opcode == ISD::INTRINSIC_VOID ||
8345 Opcode == ISD::INTRINSIC_W_CHAIN ||
8346 Opcode == ISD::PREFETCH ||
8347 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
8348 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
8349 "Opcode is not a memory-accessing opcode!");
8350
8351 // Memoize the node unless it returns a glue result.
8352 MemIntrinsicSDNode *N;
8353 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
8354 FoldingSetNodeID ID;
8355 AddNodeIDNode(ID, Opcode, VTList, Ops);
8356 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
8357 Opcode, dl.getIROrder(), VTList, MemVT, MMO));
8358 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8359 ID.AddInteger(MMO->getFlags());
8360 ID.AddInteger(MemVT.getRawBits());
8361 void *IP = nullptr;
8362 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8363 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
8364 return SDValue(E, 0);
8365 }
8366
8367 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8368 VTList, MemVT, MMO);
8369 createOperands(N, Ops);
8370
8371 CSEMap.InsertNode(N, IP);
8372 } else {
8373 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8374 VTList, MemVT, MMO);
8375 createOperands(N, Ops);
8376 }
8377 InsertNode(N);
8378 SDValue V(N, 0);
8379 NewSDValueDbgMsg(V, "Creating new node: ", this);
8380 return V;
8381 }
8382
getLifetimeNode(bool IsStart,const SDLoc & dl,SDValue Chain,int FrameIndex,int64_t Size,int64_t Offset)8383 SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
8384 SDValue Chain, int FrameIndex,
8385 int64_t Size, int64_t Offset) {
8386 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
8387 const auto VTs = getVTList(MVT::Other);
8388 SDValue Ops[2] = {
8389 Chain,
8390 getFrameIndex(FrameIndex,
8391 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
8392 true)};
8393
8394 FoldingSetNodeID ID;
8395 AddNodeIDNode(ID, Opcode, VTs, Ops);
8396 ID.AddInteger(FrameIndex);
8397 ID.AddInteger(Size);
8398 ID.AddInteger(Offset);
8399 void *IP = nullptr;
8400 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8401 return SDValue(E, 0);
8402
8403 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
8404 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
8405 createOperands(N, Ops);
8406 CSEMap.InsertNode(N, IP);
8407 InsertNode(N);
8408 SDValue V(N, 0);
8409 NewSDValueDbgMsg(V, "Creating new node: ", this);
8410 return V;
8411 }
8412
getPseudoProbeNode(const SDLoc & Dl,SDValue Chain,uint64_t Guid,uint64_t Index,uint32_t Attr)8413 SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
8414 uint64_t Guid, uint64_t Index,
8415 uint32_t Attr) {
8416 const unsigned Opcode = ISD::PSEUDO_PROBE;
8417 const auto VTs = getVTList(MVT::Other);
8418 SDValue Ops[] = {Chain};
8419 FoldingSetNodeID ID;
8420 AddNodeIDNode(ID, Opcode, VTs, Ops);
8421 ID.AddInteger(Guid);
8422 ID.AddInteger(Index);
8423 void *IP = nullptr;
8424 if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP))
8425 return SDValue(E, 0);
8426
8427 auto *N = newSDNode<PseudoProbeSDNode>(
8428 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
8429 createOperands(N, Ops);
8430 CSEMap.InsertNode(N, IP);
8431 InsertNode(N);
8432 SDValue V(N, 0);
8433 NewSDValueDbgMsg(V, "Creating new node: ", this);
8434 return V;
8435 }
8436
8437 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8438 /// MachinePointerInfo record from it. This is particularly useful because the
8439 /// code generator has many cases where it doesn't bother passing in a
8440 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
InferPointerInfo(const MachinePointerInfo & Info,SelectionDAG & DAG,SDValue Ptr,int64_t Offset=0)8441 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
8442 SelectionDAG &DAG, SDValue Ptr,
8443 int64_t Offset = 0) {
8444 // If this is FI+Offset, we can model it.
8445 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
8446 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
8447 FI->getIndex(), Offset);
8448
8449 // If this is (FI+Offset1)+Offset2, we can model it.
8450 if (Ptr.getOpcode() != ISD::ADD ||
8451 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
8452 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
8453 return Info;
8454
8455 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
8456 return MachinePointerInfo::getFixedStack(
8457 DAG.getMachineFunction(), FI,
8458 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
8459 }
8460
8461 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8462 /// MachinePointerInfo record from it. This is particularly useful because the
8463 /// code generator has many cases where it doesn't bother passing in a
8464 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
InferPointerInfo(const MachinePointerInfo & Info,SelectionDAG & DAG,SDValue Ptr,SDValue OffsetOp)8465 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
8466 SelectionDAG &DAG, SDValue Ptr,
8467 SDValue OffsetOp) {
8468 // If the 'Offset' value isn't a constant, we can't handle this.
8469 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
8470 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
8471 if (OffsetOp.isUndef())
8472 return InferPointerInfo(Info, DAG, Ptr);
8473 return Info;
8474 }
8475
getLoad(ISD::MemIndexedMode AM,ISD::LoadExtType ExtType,EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,SDValue Offset,MachinePointerInfo PtrInfo,EVT MemVT,Align Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,const MDNode * Ranges)8476 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
8477 EVT VT, const SDLoc &dl, SDValue Chain,
8478 SDValue Ptr, SDValue Offset,
8479 MachinePointerInfo PtrInfo, EVT MemVT,
8480 Align Alignment,
8481 MachineMemOperand::Flags MMOFlags,
8482 const AAMDNodes &AAInfo, const MDNode *Ranges) {
8483 assert(Chain.getValueType() == MVT::Other &&
8484 "Invalid chain type");
8485
8486 MMOFlags |= MachineMemOperand::MOLoad;
8487 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8488 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8489 // clients.
8490 if (PtrInfo.V.isNull())
8491 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
8492
8493 uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
8494 MachineFunction &MF = getMachineFunction();
8495 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
8496 Alignment, AAInfo, Ranges);
8497 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
8498 }
8499
getLoad(ISD::MemIndexedMode AM,ISD::LoadExtType ExtType,EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,SDValue Offset,EVT MemVT,MachineMemOperand * MMO)8500 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
8501 EVT VT, const SDLoc &dl, SDValue Chain,
8502 SDValue Ptr, SDValue Offset, EVT MemVT,
8503 MachineMemOperand *MMO) {
8504 if (VT == MemVT) {
8505 ExtType = ISD::NON_EXTLOAD;
8506 } else if (ExtType == ISD::NON_EXTLOAD) {
8507 assert(VT == MemVT && "Non-extending load from different memory type!");
8508 } else {
8509 // Extending load.
8510 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
8511 "Should only be an extending load, not truncating!");
8512 assert(VT.isInteger() == MemVT.isInteger() &&
8513 "Cannot convert from FP to Int or Int -> FP!");
8514 assert(VT.isVector() == MemVT.isVector() &&
8515 "Cannot use an ext load to convert to or from a vector!");
8516 assert((!VT.isVector() ||
8517 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
8518 "Cannot use an ext load to change the number of vector elements!");
8519 }
8520
8521 bool Indexed = AM != ISD::UNINDEXED;
8522 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8523
8524 SDVTList VTs = Indexed ?
8525 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
8526 SDValue Ops[] = { Chain, Ptr, Offset };
8527 FoldingSetNodeID ID;
8528 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
8529 ID.AddInteger(MemVT.getRawBits());
8530 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
8531 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
8532 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8533 ID.AddInteger(MMO->getFlags());
8534 void *IP = nullptr;
8535 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8536 cast<LoadSDNode>(E)->refineAlignment(MMO);
8537 return SDValue(E, 0);
8538 }
8539 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8540 ExtType, MemVT, MMO);
8541 createOperands(N, Ops);
8542
8543 CSEMap.InsertNode(N, IP);
8544 InsertNode(N);
8545 SDValue V(N, 0);
8546 NewSDValueDbgMsg(V, "Creating new node: ", this);
8547 return V;
8548 }
8549
getLoad(EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,MachinePointerInfo PtrInfo,MaybeAlign Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,const MDNode * Ranges)8550 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
8551 SDValue Ptr, MachinePointerInfo PtrInfo,
8552 MaybeAlign Alignment,
8553 MachineMemOperand::Flags MMOFlags,
8554 const AAMDNodes &AAInfo, const MDNode *Ranges) {
8555 SDValue Undef = getUNDEF(Ptr.getValueType());
8556 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8557 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
8558 }
8559
getLoad(EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,MachineMemOperand * MMO)8560 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
8561 SDValue Ptr, MachineMemOperand *MMO) {
8562 SDValue Undef = getUNDEF(Ptr.getValueType());
8563 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8564 VT, MMO);
8565 }
8566
getExtLoad(ISD::LoadExtType ExtType,const SDLoc & dl,EVT VT,SDValue Chain,SDValue Ptr,MachinePointerInfo PtrInfo,EVT MemVT,MaybeAlign Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo)8567 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
8568 EVT VT, SDValue Chain, SDValue Ptr,
8569 MachinePointerInfo PtrInfo, EVT MemVT,
8570 MaybeAlign Alignment,
8571 MachineMemOperand::Flags MMOFlags,
8572 const AAMDNodes &AAInfo) {
8573 SDValue Undef = getUNDEF(Ptr.getValueType());
8574 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
8575 MemVT, Alignment, MMOFlags, AAInfo);
8576 }
8577
getExtLoad(ISD::LoadExtType ExtType,const SDLoc & dl,EVT VT,SDValue Chain,SDValue Ptr,EVT MemVT,MachineMemOperand * MMO)8578 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
8579 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
8580 MachineMemOperand *MMO) {
8581 SDValue Undef = getUNDEF(Ptr.getValueType());
8582 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
8583 MemVT, MMO);
8584 }
8585
getIndexedLoad(SDValue OrigLoad,const SDLoc & dl,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)8586 SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
8587 SDValue Base, SDValue Offset,
8588 ISD::MemIndexedMode AM) {
8589 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
8590 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8591 // Don't propagate the invariant or dereferenceable flags.
8592 auto MMOFlags =
8593 LD->getMemOperand()->getFlags() &
8594 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
8595 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
8596 LD->getChain(), Base, Offset, LD->getPointerInfo(),
8597 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
8598 }
8599
getStore(SDValue Chain,const SDLoc & dl,SDValue Val,SDValue Ptr,MachinePointerInfo PtrInfo,Align Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo)8600 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8601 SDValue Ptr, MachinePointerInfo PtrInfo,
8602 Align Alignment,
8603 MachineMemOperand::Flags MMOFlags,
8604 const AAMDNodes &AAInfo) {
8605 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8606
8607 MMOFlags |= MachineMemOperand::MOStore;
8608 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8609
8610 if (PtrInfo.V.isNull())
8611 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8612
8613 MachineFunction &MF = getMachineFunction();
8614 uint64_t Size =
8615 MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
8616 MachineMemOperand *MMO =
8617 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
8618 return getStore(Chain, dl, Val, Ptr, MMO);
8619 }
8620
getStore(SDValue Chain,const SDLoc & dl,SDValue Val,SDValue Ptr,MachineMemOperand * MMO)8621 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8622 SDValue Ptr, MachineMemOperand *MMO) {
8623 assert(Chain.getValueType() == MVT::Other &&
8624 "Invalid chain type");
8625 EVT VT = Val.getValueType();
8626 SDVTList VTs = getVTList(MVT::Other);
8627 SDValue Undef = getUNDEF(Ptr.getValueType());
8628 SDValue Ops[] = { Chain, Val, Ptr, Undef };
8629 FoldingSetNodeID ID;
8630 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8631 ID.AddInteger(VT.getRawBits());
8632 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
8633 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
8634 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8635 ID.AddInteger(MMO->getFlags());
8636 void *IP = nullptr;
8637 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8638 cast<StoreSDNode>(E)->refineAlignment(MMO);
8639 return SDValue(E, 0);
8640 }
8641 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8642 ISD::UNINDEXED, false, VT, MMO);
8643 createOperands(N, Ops);
8644
8645 CSEMap.InsertNode(N, IP);
8646 InsertNode(N);
8647 SDValue V(N, 0);
8648 NewSDValueDbgMsg(V, "Creating new node: ", this);
8649 return V;
8650 }
8651
getTruncStore(SDValue Chain,const SDLoc & dl,SDValue Val,SDValue Ptr,MachinePointerInfo PtrInfo,EVT SVT,Align Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo)8652 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8653 SDValue Ptr, MachinePointerInfo PtrInfo,
8654 EVT SVT, Align Alignment,
8655 MachineMemOperand::Flags MMOFlags,
8656 const AAMDNodes &AAInfo) {
8657 assert(Chain.getValueType() == MVT::Other &&
8658 "Invalid chain type");
8659
8660 MMOFlags |= MachineMemOperand::MOStore;
8661 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8662
8663 if (PtrInfo.V.isNull())
8664 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8665
8666 MachineFunction &MF = getMachineFunction();
8667 MachineMemOperand *MMO = MF.getMachineMemOperand(
8668 PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
8669 Alignment, AAInfo);
8670 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
8671 }
8672
getTruncStore(SDValue Chain,const SDLoc & dl,SDValue Val,SDValue Ptr,EVT SVT,MachineMemOperand * MMO)8673 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8674 SDValue Ptr, EVT SVT,
8675 MachineMemOperand *MMO) {
8676 EVT VT = Val.getValueType();
8677
8678 assert(Chain.getValueType() == MVT::Other &&
8679 "Invalid chain type");
8680 if (VT == SVT)
8681 return getStore(Chain, dl, Val, Ptr, MMO);
8682
8683 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
8684 "Should only be a truncating store, not extending!");
8685 assert(VT.isInteger() == SVT.isInteger() &&
8686 "Can't do FP-INT conversion!");
8687 assert(VT.isVector() == SVT.isVector() &&
8688 "Cannot use trunc store to convert to or from a vector!");
8689 assert((!VT.isVector() ||
8690 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
8691 "Cannot use trunc store to change the number of vector elements!");
8692
8693 SDVTList VTs = getVTList(MVT::Other);
8694 SDValue Undef = getUNDEF(Ptr.getValueType());
8695 SDValue Ops[] = { Chain, Val, Ptr, Undef };
8696 FoldingSetNodeID ID;
8697 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8698 ID.AddInteger(SVT.getRawBits());
8699 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
8700 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
8701 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8702 ID.AddInteger(MMO->getFlags());
8703 void *IP = nullptr;
8704 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8705 cast<StoreSDNode>(E)->refineAlignment(MMO);
8706 return SDValue(E, 0);
8707 }
8708 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8709 ISD::UNINDEXED, true, SVT, MMO);
8710 createOperands(N, Ops);
8711
8712 CSEMap.InsertNode(N, IP);
8713 InsertNode(N);
8714 SDValue V(N, 0);
8715 NewSDValueDbgMsg(V, "Creating new node: ", this);
8716 return V;
8717 }
8718
getIndexedStore(SDValue OrigStore,const SDLoc & dl,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)8719 SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
8720 SDValue Base, SDValue Offset,
8721 ISD::MemIndexedMode AM) {
8722 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
8723 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
8724 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
8725 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
8726 FoldingSetNodeID ID;
8727 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8728 ID.AddInteger(ST->getMemoryVT().getRawBits());
8729 ID.AddInteger(ST->getRawSubclassData());
8730 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
8731 ID.AddInteger(ST->getMemOperand()->getFlags());
8732 void *IP = nullptr;
8733 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8734 return SDValue(E, 0);
8735
8736 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8737 ST->isTruncatingStore(), ST->getMemoryVT(),
8738 ST->getMemOperand());
8739 createOperands(N, Ops);
8740
8741 CSEMap.InsertNode(N, IP);
8742 InsertNode(N);
8743 SDValue V(N, 0);
8744 NewSDValueDbgMsg(V, "Creating new node: ", this);
8745 return V;
8746 }
8747
getLoadVP(ISD::MemIndexedMode AM,ISD::LoadExtType ExtType,EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,SDValue Offset,SDValue Mask,SDValue EVL,MachinePointerInfo PtrInfo,EVT MemVT,Align Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,const MDNode * Ranges,bool IsExpanding)8748 SDValue SelectionDAG::getLoadVP(
8749 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
8750 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
8751 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
8752 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
8753 const MDNode *Ranges, bool IsExpanding) {
8754 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8755
8756 MMOFlags |= MachineMemOperand::MOLoad;
8757 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8758 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8759 // clients.
8760 if (PtrInfo.V.isNull())
8761 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
8762
8763 uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
8764 MachineFunction &MF = getMachineFunction();
8765 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
8766 Alignment, AAInfo, Ranges);
8767 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
8768 MMO, IsExpanding);
8769 }
8770
getLoadVP(ISD::MemIndexedMode AM,ISD::LoadExtType ExtType,EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,SDValue Offset,SDValue Mask,SDValue EVL,EVT MemVT,MachineMemOperand * MMO,bool IsExpanding)8771 SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
8772 ISD::LoadExtType ExtType, EVT VT,
8773 const SDLoc &dl, SDValue Chain, SDValue Ptr,
8774 SDValue Offset, SDValue Mask, SDValue EVL,
8775 EVT MemVT, MachineMemOperand *MMO,
8776 bool IsExpanding) {
8777 bool Indexed = AM != ISD::UNINDEXED;
8778 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8779
8780 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
8781 : getVTList(VT, MVT::Other);
8782 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
8783 FoldingSetNodeID ID;
8784 AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops);
8785 ID.AddInteger(MemVT.getRawBits());
8786 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
8787 dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
8788 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8789 ID.AddInteger(MMO->getFlags());
8790 void *IP = nullptr;
8791 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8792 cast<VPLoadSDNode>(E)->refineAlignment(MMO);
8793 return SDValue(E, 0);
8794 }
8795 auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8796 ExtType, IsExpanding, MemVT, MMO);
8797 createOperands(N, Ops);
8798
8799 CSEMap.InsertNode(N, IP);
8800 InsertNode(N);
8801 SDValue V(N, 0);
8802 NewSDValueDbgMsg(V, "Creating new node: ", this);
8803 return V;
8804 }
8805
getLoadVP(EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,SDValue Mask,SDValue EVL,MachinePointerInfo PtrInfo,MaybeAlign Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,const MDNode * Ranges,bool IsExpanding)8806 SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
8807 SDValue Ptr, SDValue Mask, SDValue EVL,
8808 MachinePointerInfo PtrInfo,
8809 MaybeAlign Alignment,
8810 MachineMemOperand::Flags MMOFlags,
8811 const AAMDNodes &AAInfo, const MDNode *Ranges,
8812 bool IsExpanding) {
8813 SDValue Undef = getUNDEF(Ptr.getValueType());
8814 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8815 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
8816 IsExpanding);
8817 }
8818
getLoadVP(EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,SDValue Mask,SDValue EVL,MachineMemOperand * MMO,bool IsExpanding)8819 SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
8820 SDValue Ptr, SDValue Mask, SDValue EVL,
8821 MachineMemOperand *MMO, bool IsExpanding) {
8822 SDValue Undef = getUNDEF(Ptr.getValueType());
8823 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8824 Mask, EVL, VT, MMO, IsExpanding);
8825 }
8826
getExtLoadVP(ISD::LoadExtType ExtType,const SDLoc & dl,EVT VT,SDValue Chain,SDValue Ptr,SDValue Mask,SDValue EVL,MachinePointerInfo PtrInfo,EVT MemVT,MaybeAlign Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,bool IsExpanding)8827 SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
8828 EVT VT, SDValue Chain, SDValue Ptr,
8829 SDValue Mask, SDValue EVL,
8830 MachinePointerInfo PtrInfo, EVT MemVT,
8831 MaybeAlign Alignment,
8832 MachineMemOperand::Flags MMOFlags,
8833 const AAMDNodes &AAInfo, bool IsExpanding) {
8834 SDValue Undef = getUNDEF(Ptr.getValueType());
8835 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
8836 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr,
8837 IsExpanding);
8838 }
8839
getExtLoadVP(ISD::LoadExtType ExtType,const SDLoc & dl,EVT VT,SDValue Chain,SDValue Ptr,SDValue Mask,SDValue EVL,EVT MemVT,MachineMemOperand * MMO,bool IsExpanding)8840 SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
8841 EVT VT, SDValue Chain, SDValue Ptr,
8842 SDValue Mask, SDValue EVL, EVT MemVT,
8843 MachineMemOperand *MMO, bool IsExpanding) {
8844 SDValue Undef = getUNDEF(Ptr.getValueType());
8845 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
8846 EVL, MemVT, MMO, IsExpanding);
8847 }
8848
getIndexedLoadVP(SDValue OrigLoad,const SDLoc & dl,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)8849 SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
8850 SDValue Base, SDValue Offset,
8851 ISD::MemIndexedMode AM) {
8852 auto *LD = cast<VPLoadSDNode>(OrigLoad);
8853 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8854 // Don't propagate the invariant or dereferenceable flags.
8855 auto MMOFlags =
8856 LD->getMemOperand()->getFlags() &
8857 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
8858 return getLoadVP(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
8859 LD->getChain(), Base, Offset, LD->getMask(),
8860 LD->getVectorLength(), LD->getPointerInfo(),
8861 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(),
8862 nullptr, LD->isExpandingLoad());
8863 }
8864
getStoreVP(SDValue Chain,const SDLoc & dl,SDValue Val,SDValue Ptr,SDValue Offset,SDValue Mask,SDValue EVL,EVT MemVT,MachineMemOperand * MMO,ISD::MemIndexedMode AM,bool IsTruncating,bool IsCompressing)8865 SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
8866 SDValue Ptr, SDValue Offset, SDValue Mask,
8867 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
8868 ISD::MemIndexedMode AM, bool IsTruncating,
8869 bool IsCompressing) {
8870 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8871 bool Indexed = AM != ISD::UNINDEXED;
8872 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
8873 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
8874 : getVTList(MVT::Other);
8875 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
8876 FoldingSetNodeID ID;
8877 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
8878 ID.AddInteger(MemVT.getRawBits());
8879 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
8880 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
8881 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8882 ID.AddInteger(MMO->getFlags());
8883 void *IP = nullptr;
8884 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8885 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
8886 return SDValue(E, 0);
8887 }
8888 auto *N = newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8889 IsTruncating, IsCompressing, MemVT, MMO);
8890 createOperands(N, Ops);
8891
8892 CSEMap.InsertNode(N, IP);
8893 InsertNode(N);
8894 SDValue V(N, 0);
8895 NewSDValueDbgMsg(V, "Creating new node: ", this);
8896 return V;
8897 }
8898
getTruncStoreVP(SDValue Chain,const SDLoc & dl,SDValue Val,SDValue Ptr,SDValue Mask,SDValue EVL,MachinePointerInfo PtrInfo,EVT SVT,Align Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,bool IsCompressing)8899 SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
8900 SDValue Val, SDValue Ptr, SDValue Mask,
8901 SDValue EVL, MachinePointerInfo PtrInfo,
8902 EVT SVT, Align Alignment,
8903 MachineMemOperand::Flags MMOFlags,
8904 const AAMDNodes &AAInfo,
8905 bool IsCompressing) {
8906 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8907
8908 MMOFlags |= MachineMemOperand::MOStore;
8909 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8910
8911 if (PtrInfo.V.isNull())
8912 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8913
8914 MachineFunction &MF = getMachineFunction();
8915 MachineMemOperand *MMO = MF.getMachineMemOperand(
8916 PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
8917 Alignment, AAInfo);
8918 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
8919 IsCompressing);
8920 }
8921
getTruncStoreVP(SDValue Chain,const SDLoc & dl,SDValue Val,SDValue Ptr,SDValue Mask,SDValue EVL,EVT SVT,MachineMemOperand * MMO,bool IsCompressing)8922 SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
8923 SDValue Val, SDValue Ptr, SDValue Mask,
8924 SDValue EVL, EVT SVT,
8925 MachineMemOperand *MMO,
8926 bool IsCompressing) {
8927 EVT VT = Val.getValueType();
8928
8929 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8930 if (VT == SVT)
8931 return getStoreVP(Chain, dl, Val, Ptr, getUNDEF(Ptr.getValueType()), Mask,
8932 EVL, VT, MMO, ISD::UNINDEXED,
8933 /*IsTruncating*/ false, IsCompressing);
8934
8935 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
8936 "Should only be a truncating store, not extending!");
8937 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
8938 assert(VT.isVector() == SVT.isVector() &&
8939 "Cannot use trunc store to convert to or from a vector!");
8940 assert((!VT.isVector() ||
8941 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
8942 "Cannot use trunc store to change the number of vector elements!");
8943
8944 SDVTList VTs = getVTList(MVT::Other);
8945 SDValue Undef = getUNDEF(Ptr.getValueType());
8946 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
8947 FoldingSetNodeID ID;
8948 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
8949 ID.AddInteger(SVT.getRawBits());
8950 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
8951 dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
8952 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8953 ID.AddInteger(MMO->getFlags());
8954 void *IP = nullptr;
8955 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8956 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
8957 return SDValue(E, 0);
8958 }
8959 auto *N =
8960 newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8961 ISD::UNINDEXED, true, IsCompressing, SVT, MMO);
8962 createOperands(N, Ops);
8963
8964 CSEMap.InsertNode(N, IP);
8965 InsertNode(N);
8966 SDValue V(N, 0);
8967 NewSDValueDbgMsg(V, "Creating new node: ", this);
8968 return V;
8969 }
8970
getIndexedStoreVP(SDValue OrigStore,const SDLoc & dl,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)8971 SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
8972 SDValue Base, SDValue Offset,
8973 ISD::MemIndexedMode AM) {
8974 auto *ST = cast<VPStoreSDNode>(OrigStore);
8975 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
8976 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
8977 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
8978 Offset, ST->getMask(), ST->getVectorLength()};
8979 FoldingSetNodeID ID;
8980 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
8981 ID.AddInteger(ST->getMemoryVT().getRawBits());
8982 ID.AddInteger(ST->getRawSubclassData());
8983 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
8984 ID.AddInteger(ST->getMemOperand()->getFlags());
8985 void *IP = nullptr;
8986 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8987 return SDValue(E, 0);
8988
8989 auto *N = newSDNode<VPStoreSDNode>(
8990 dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ST->isTruncatingStore(),
8991 ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand());
8992 createOperands(N, Ops);
8993
8994 CSEMap.InsertNode(N, IP);
8995 InsertNode(N);
8996 SDValue V(N, 0);
8997 NewSDValueDbgMsg(V, "Creating new node: ", this);
8998 return V;
8999 }
9000
getStridedLoadVP(ISD::MemIndexedMode AM,ISD::LoadExtType ExtType,EVT VT,const SDLoc & DL,SDValue Chain,SDValue Ptr,SDValue Offset,SDValue Stride,SDValue Mask,SDValue EVL,MachinePointerInfo PtrInfo,EVT MemVT,Align Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,const MDNode * Ranges,bool IsExpanding)9001 SDValue SelectionDAG::getStridedLoadVP(
9002 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9003 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9004 SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
9005 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9006 const MDNode *Ranges, bool IsExpanding) {
9007 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9008
9009 MMOFlags |= MachineMemOperand::MOLoad;
9010 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
9011 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
9012 // clients.
9013 if (PtrInfo.V.isNull())
9014 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
9015
9016 uint64_t Size = MemoryLocation::UnknownSize;
9017 MachineFunction &MF = getMachineFunction();
9018 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
9019 Alignment, AAInfo, Ranges);
9020 return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, Mask,
9021 EVL, MemVT, MMO, IsExpanding);
9022 }
9023
getStridedLoadVP(ISD::MemIndexedMode AM,ISD::LoadExtType ExtType,EVT VT,const SDLoc & DL,SDValue Chain,SDValue Ptr,SDValue Offset,SDValue Stride,SDValue Mask,SDValue EVL,EVT MemVT,MachineMemOperand * MMO,bool IsExpanding)9024 SDValue SelectionDAG::getStridedLoadVP(
9025 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9026 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9027 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
9028 bool Indexed = AM != ISD::UNINDEXED;
9029 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9030
9031 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
9032 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
9033 : getVTList(VT, MVT::Other);
9034 FoldingSetNodeID ID;
9035 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
9036 ID.AddInteger(VT.getRawBits());
9037 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9038 DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9039 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9040
9041 void *IP = nullptr;
9042 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9043 cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);
9044 return SDValue(E, 0);
9045 }
9046
9047 auto *N =
9048 newSDNode<VPStridedLoadSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, AM,
9049 ExtType, IsExpanding, MemVT, MMO);
9050 createOperands(N, Ops);
9051 CSEMap.InsertNode(N, IP);
9052 InsertNode(N);
9053 SDValue V(N, 0);
9054 NewSDValueDbgMsg(V, "Creating new node: ", this);
9055 return V;
9056 }
9057
getStridedLoadVP(EVT VT,const SDLoc & DL,SDValue Chain,SDValue Ptr,SDValue Stride,SDValue Mask,SDValue EVL,MachinePointerInfo PtrInfo,MaybeAlign Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,const MDNode * Ranges,bool IsExpanding)9058 SDValue SelectionDAG::getStridedLoadVP(
9059 EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Stride,
9060 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, MaybeAlign Alignment,
9061 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9062 const MDNode *Ranges, bool IsExpanding) {
9063 SDValue Undef = getUNDEF(Ptr.getValueType());
9064 return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
9065 Undef, Stride, Mask, EVL, PtrInfo, VT, Alignment,
9066 MMOFlags, AAInfo, Ranges, IsExpanding);
9067 }
9068
getStridedLoadVP(EVT VT,const SDLoc & DL,SDValue Chain,SDValue Ptr,SDValue Stride,SDValue Mask,SDValue EVL,MachineMemOperand * MMO,bool IsExpanding)9069 SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
9070 SDValue Ptr, SDValue Stride,
9071 SDValue Mask, SDValue EVL,
9072 MachineMemOperand *MMO,
9073 bool IsExpanding) {
9074 SDValue Undef = getUNDEF(Ptr.getValueType());
9075 return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
9076 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
9077 }
9078
getExtStridedLoadVP(ISD::LoadExtType ExtType,const SDLoc & DL,EVT VT,SDValue Chain,SDValue Ptr,SDValue Stride,SDValue Mask,SDValue EVL,MachinePointerInfo PtrInfo,EVT MemVT,MaybeAlign Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,bool IsExpanding)9079 SDValue SelectionDAG::getExtStridedLoadVP(
9080 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9081 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL,
9082 MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment,
9083 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9084 bool IsExpanding) {
9085 SDValue Undef = getUNDEF(Ptr.getValueType());
9086 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,
9087 Stride, Mask, EVL, PtrInfo, MemVT, Alignment,
9088 MMOFlags, AAInfo, nullptr, IsExpanding);
9089 }
9090
getExtStridedLoadVP(ISD::LoadExtType ExtType,const SDLoc & DL,EVT VT,SDValue Chain,SDValue Ptr,SDValue Stride,SDValue Mask,SDValue EVL,EVT MemVT,MachineMemOperand * MMO,bool IsExpanding)9091 SDValue SelectionDAG::getExtStridedLoadVP(
9092 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9093 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
9094 MachineMemOperand *MMO, bool IsExpanding) {
9095 SDValue Undef = getUNDEF(Ptr.getValueType());
9096 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,
9097 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9098 }
9099
getIndexedStridedLoadVP(SDValue OrigLoad,const SDLoc & DL,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)9100 SDValue SelectionDAG::getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL,
9101 SDValue Base, SDValue Offset,
9102 ISD::MemIndexedMode AM) {
9103 auto *SLD = cast<VPStridedLoadSDNode>(OrigLoad);
9104 assert(SLD->getOffset().isUndef() &&
9105 "Strided load is already a indexed load!");
9106 // Don't propagate the invariant or dereferenceable flags.
9107 auto MMOFlags =
9108 SLD->getMemOperand()->getFlags() &
9109 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
9110 return getStridedLoadVP(
9111 AM, SLD->getExtensionType(), OrigLoad.getValueType(), DL, SLD->getChain(),
9112 Base, Offset, SLD->getStride(), SLD->getMask(), SLD->getVectorLength(),
9113 SLD->getPointerInfo(), SLD->getMemoryVT(), SLD->getAlign(), MMOFlags,
9114 SLD->getAAInfo(), nullptr, SLD->isExpandingLoad());
9115 }
9116
getStridedStoreVP(SDValue Chain,const SDLoc & DL,SDValue Val,SDValue Ptr,SDValue Offset,SDValue Stride,SDValue Mask,SDValue EVL,EVT MemVT,MachineMemOperand * MMO,ISD::MemIndexedMode AM,bool IsTruncating,bool IsCompressing)9117 SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
9118 SDValue Val, SDValue Ptr,
9119 SDValue Offset, SDValue Stride,
9120 SDValue Mask, SDValue EVL, EVT MemVT,
9121 MachineMemOperand *MMO,
9122 ISD::MemIndexedMode AM,
9123 bool IsTruncating, bool IsCompressing) {
9124 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9125 bool Indexed = AM != ISD::UNINDEXED;
9126 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9127 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9128 : getVTList(MVT::Other);
9129 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
9130 FoldingSetNodeID ID;
9131 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9132 ID.AddInteger(MemVT.getRawBits());
9133 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9134 DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9135 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9136 void *IP = nullptr;
9137 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9138 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9139 return SDValue(E, 0);
9140 }
9141 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9142 VTs, AM, IsTruncating,
9143 IsCompressing, MemVT, MMO);
9144 createOperands(N, Ops);
9145
9146 CSEMap.InsertNode(N, IP);
9147 InsertNode(N);
9148 SDValue V(N, 0);
9149 NewSDValueDbgMsg(V, "Creating new node: ", this);
9150 return V;
9151 }
9152
getTruncStridedStoreVP(SDValue Chain,const SDLoc & DL,SDValue Val,SDValue Ptr,SDValue Stride,SDValue Mask,SDValue EVL,MachinePointerInfo PtrInfo,EVT SVT,Align Alignment,MachineMemOperand::Flags MMOFlags,const AAMDNodes & AAInfo,bool IsCompressing)9153 SDValue SelectionDAG::getTruncStridedStoreVP(
9154 SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride,
9155 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT,
9156 Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9157 bool IsCompressing) {
9158 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9159
9160 MMOFlags |= MachineMemOperand::MOStore;
9161 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9162
9163 if (PtrInfo.V.isNull())
9164 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
9165
9166 MachineFunction &MF = getMachineFunction();
9167 MachineMemOperand *MMO = MF.getMachineMemOperand(
9168 PtrInfo, MMOFlags, MemoryLocation::UnknownSize, Alignment, AAInfo);
9169 return getTruncStridedStoreVP(Chain, DL, Val, Ptr, Stride, Mask, EVL, SVT,
9170 MMO, IsCompressing);
9171 }
9172
getTruncStridedStoreVP(SDValue Chain,const SDLoc & DL,SDValue Val,SDValue Ptr,SDValue Stride,SDValue Mask,SDValue EVL,EVT SVT,MachineMemOperand * MMO,bool IsCompressing)9173 SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
9174 SDValue Val, SDValue Ptr,
9175 SDValue Stride, SDValue Mask,
9176 SDValue EVL, EVT SVT,
9177 MachineMemOperand *MMO,
9178 bool IsCompressing) {
9179 EVT VT = Val.getValueType();
9180
9181 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9182 if (VT == SVT)
9183 return getStridedStoreVP(Chain, DL, Val, Ptr, getUNDEF(Ptr.getValueType()),
9184 Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED,
9185 /*IsTruncating*/ false, IsCompressing);
9186
9187 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
9188 "Should only be a truncating store, not extending!");
9189 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9190 assert(VT.isVector() == SVT.isVector() &&
9191 "Cannot use trunc store to convert to or from a vector!");
9192 assert((!VT.isVector() ||
9193 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
9194 "Cannot use trunc store to change the number of vector elements!");
9195
9196 SDVTList VTs = getVTList(MVT::Other);
9197 SDValue Undef = getUNDEF(Ptr.getValueType());
9198 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
9199 FoldingSetNodeID ID;
9200 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9201 ID.AddInteger(SVT.getRawBits());
9202 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9203 DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
9204 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9205 void *IP = nullptr;
9206 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9207 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9208 return SDValue(E, 0);
9209 }
9210 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9211 VTs, ISD::UNINDEXED, true,
9212 IsCompressing, SVT, MMO);
9213 createOperands(N, Ops);
9214
9215 CSEMap.InsertNode(N, IP);
9216 InsertNode(N);
9217 SDValue V(N, 0);
9218 NewSDValueDbgMsg(V, "Creating new node: ", this);
9219 return V;
9220 }
9221
getIndexedStridedStoreVP(SDValue OrigStore,const SDLoc & DL,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)9222 SDValue SelectionDAG::getIndexedStridedStoreVP(SDValue OrigStore,
9223 const SDLoc &DL, SDValue Base,
9224 SDValue Offset,
9225 ISD::MemIndexedMode AM) {
9226 auto *SST = cast<VPStridedStoreSDNode>(OrigStore);
9227 assert(SST->getOffset().isUndef() &&
9228 "Strided store is already an indexed store!");
9229 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9230 SDValue Ops[] = {
9231 SST->getChain(), SST->getValue(), Base, Offset, SST->getStride(),
9232 SST->getMask(), SST->getVectorLength()};
9233 FoldingSetNodeID ID;
9234 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9235 ID.AddInteger(SST->getMemoryVT().getRawBits());
9236 ID.AddInteger(SST->getRawSubclassData());
9237 ID.AddInteger(SST->getPointerInfo().getAddrSpace());
9238 void *IP = nullptr;
9239 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
9240 return SDValue(E, 0);
9241
9242 auto *N = newSDNode<VPStridedStoreSDNode>(
9243 DL.getIROrder(), DL.getDebugLoc(), VTs, AM, SST->isTruncatingStore(),
9244 SST->isCompressingStore(), SST->getMemoryVT(), SST->getMemOperand());
9245 createOperands(N, Ops);
9246
9247 CSEMap.InsertNode(N, IP);
9248 InsertNode(N);
9249 SDValue V(N, 0);
9250 NewSDValueDbgMsg(V, "Creating new node: ", this);
9251 return V;
9252 }
9253
getGatherVP(SDVTList VTs,EVT VT,const SDLoc & dl,ArrayRef<SDValue> Ops,MachineMemOperand * MMO,ISD::MemIndexType IndexType)9254 SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
9255 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
9256 ISD::MemIndexType IndexType) {
9257 assert(Ops.size() == 6 && "Incompatible number of operands");
9258
9259 FoldingSetNodeID ID;
9260 AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops);
9261 ID.AddInteger(VT.getRawBits());
9262 ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
9263 dl.getIROrder(), VTs, VT, MMO, IndexType));
9264 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9265 ID.AddInteger(MMO->getFlags());
9266 void *IP = nullptr;
9267 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9268 cast<VPGatherSDNode>(E)->refineAlignment(MMO);
9269 return SDValue(E, 0);
9270 }
9271
9272 auto *N = newSDNode<VPGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9273 VT, MMO, IndexType);
9274 createOperands(N, Ops);
9275
9276 assert(N->getMask().getValueType().getVectorElementCount() ==
9277 N->getValueType(0).getVectorElementCount() &&
9278 "Vector width mismatch between mask and data");
9279 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9280 N->getValueType(0).getVectorElementCount().isScalable() &&
9281 "Scalable flags of index and data do not match");
9282 assert(ElementCount::isKnownGE(
9283 N->getIndex().getValueType().getVectorElementCount(),
9284 N->getValueType(0).getVectorElementCount()) &&
9285 "Vector width mismatch between index and data");
9286 assert(isa<ConstantSDNode>(N->getScale()) &&
9287 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9288 "Scale should be a constant power of 2");
9289
9290 CSEMap.InsertNode(N, IP);
9291 InsertNode(N);
9292 SDValue V(N, 0);
9293 NewSDValueDbgMsg(V, "Creating new node: ", this);
9294 return V;
9295 }
9296
getScatterVP(SDVTList VTs,EVT VT,const SDLoc & dl,ArrayRef<SDValue> Ops,MachineMemOperand * MMO,ISD::MemIndexType IndexType)9297 SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
9298 ArrayRef<SDValue> Ops,
9299 MachineMemOperand *MMO,
9300 ISD::MemIndexType IndexType) {
9301 assert(Ops.size() == 7 && "Incompatible number of operands");
9302
9303 FoldingSetNodeID ID;
9304 AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops);
9305 ID.AddInteger(VT.getRawBits());
9306 ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
9307 dl.getIROrder(), VTs, VT, MMO, IndexType));
9308 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9309 ID.AddInteger(MMO->getFlags());
9310 void *IP = nullptr;
9311 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9312 cast<VPScatterSDNode>(E)->refineAlignment(MMO);
9313 return SDValue(E, 0);
9314 }
9315 auto *N = newSDNode<VPScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9316 VT, MMO, IndexType);
9317 createOperands(N, Ops);
9318
9319 assert(N->getMask().getValueType().getVectorElementCount() ==
9320 N->getValue().getValueType().getVectorElementCount() &&
9321 "Vector width mismatch between mask and data");
9322 assert(
9323 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9324 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9325 "Scalable flags of index and data do not match");
9326 assert(ElementCount::isKnownGE(
9327 N->getIndex().getValueType().getVectorElementCount(),
9328 N->getValue().getValueType().getVectorElementCount()) &&
9329 "Vector width mismatch between index and data");
9330 assert(isa<ConstantSDNode>(N->getScale()) &&
9331 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9332 "Scale should be a constant power of 2");
9333
9334 CSEMap.InsertNode(N, IP);
9335 InsertNode(N);
9336 SDValue V(N, 0);
9337 NewSDValueDbgMsg(V, "Creating new node: ", this);
9338 return V;
9339 }
9340
getMaskedLoad(EVT VT,const SDLoc & dl,SDValue Chain,SDValue Base,SDValue Offset,SDValue Mask,SDValue PassThru,EVT MemVT,MachineMemOperand * MMO,ISD::MemIndexedMode AM,ISD::LoadExtType ExtTy,bool isExpanding)9341 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
9342 SDValue Base, SDValue Offset, SDValue Mask,
9343 SDValue PassThru, EVT MemVT,
9344 MachineMemOperand *MMO,
9345 ISD::MemIndexedMode AM,
9346 ISD::LoadExtType ExtTy, bool isExpanding) {
9347 bool Indexed = AM != ISD::UNINDEXED;
9348 assert((Indexed || Offset.isUndef()) &&
9349 "Unindexed masked load with an offset!");
9350 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
9351 : getVTList(VT, MVT::Other);
9352 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
9353 FoldingSetNodeID ID;
9354 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
9355 ID.AddInteger(MemVT.getRawBits());
9356 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9357 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
9358 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9359 ID.AddInteger(MMO->getFlags());
9360 void *IP = nullptr;
9361 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9362 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
9363 return SDValue(E, 0);
9364 }
9365 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9366 AM, ExtTy, isExpanding, MemVT, MMO);
9367 createOperands(N, Ops);
9368
9369 CSEMap.InsertNode(N, IP);
9370 InsertNode(N);
9371 SDValue V(N, 0);
9372 NewSDValueDbgMsg(V, "Creating new node: ", this);
9373 return V;
9374 }
9375
getIndexedMaskedLoad(SDValue OrigLoad,const SDLoc & dl,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)9376 SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
9377 SDValue Base, SDValue Offset,
9378 ISD::MemIndexedMode AM) {
9379 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
9380 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
9381 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
9382 Offset, LD->getMask(), LD->getPassThru(),
9383 LD->getMemoryVT(), LD->getMemOperand(), AM,
9384 LD->getExtensionType(), LD->isExpandingLoad());
9385 }
9386
getMaskedStore(SDValue Chain,const SDLoc & dl,SDValue Val,SDValue Base,SDValue Offset,SDValue Mask,EVT MemVT,MachineMemOperand * MMO,ISD::MemIndexedMode AM,bool IsTruncating,bool IsCompressing)9387 SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
9388 SDValue Val, SDValue Base, SDValue Offset,
9389 SDValue Mask, EVT MemVT,
9390 MachineMemOperand *MMO,
9391 ISD::MemIndexedMode AM, bool IsTruncating,
9392 bool IsCompressing) {
9393 assert(Chain.getValueType() == MVT::Other &&
9394 "Invalid chain type");
9395 bool Indexed = AM != ISD::UNINDEXED;
9396 assert((Indexed || Offset.isUndef()) &&
9397 "Unindexed masked store with an offset!");
9398 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
9399 : getVTList(MVT::Other);
9400 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
9401 FoldingSetNodeID ID;
9402 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
9403 ID.AddInteger(MemVT.getRawBits());
9404 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
9405 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9406 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9407 ID.AddInteger(MMO->getFlags());
9408 void *IP = nullptr;
9409 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9410 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
9411 return SDValue(E, 0);
9412 }
9413 auto *N =
9414 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9415 IsTruncating, IsCompressing, MemVT, MMO);
9416 createOperands(N, Ops);
9417
9418 CSEMap.InsertNode(N, IP);
9419 InsertNode(N);
9420 SDValue V(N, 0);
9421 NewSDValueDbgMsg(V, "Creating new node: ", this);
9422 return V;
9423 }
9424
getIndexedMaskedStore(SDValue OrigStore,const SDLoc & dl,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)9425 SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
9426 SDValue Base, SDValue Offset,
9427 ISD::MemIndexedMode AM) {
9428 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
9429 assert(ST->getOffset().isUndef() &&
9430 "Masked store is already a indexed store!");
9431 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
9432 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
9433 AM, ST->isTruncatingStore(), ST->isCompressingStore());
9434 }
9435
getMaskedGather(SDVTList VTs,EVT MemVT,const SDLoc & dl,ArrayRef<SDValue> Ops,MachineMemOperand * MMO,ISD::MemIndexType IndexType,ISD::LoadExtType ExtTy)9436 SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
9437 ArrayRef<SDValue> Ops,
9438 MachineMemOperand *MMO,
9439 ISD::MemIndexType IndexType,
9440 ISD::LoadExtType ExtTy) {
9441 assert(Ops.size() == 6 && "Incompatible number of operands");
9442
9443 FoldingSetNodeID ID;
9444 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
9445 ID.AddInteger(MemVT.getRawBits());
9446 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
9447 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
9448 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9449 ID.AddInteger(MMO->getFlags());
9450 void *IP = nullptr;
9451 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9452 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
9453 return SDValue(E, 0);
9454 }
9455
9456 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
9457 VTs, MemVT, MMO, IndexType, ExtTy);
9458 createOperands(N, Ops);
9459
9460 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
9461 "Incompatible type of the PassThru value in MaskedGatherSDNode");
9462 assert(N->getMask().getValueType().getVectorElementCount() ==
9463 N->getValueType(0).getVectorElementCount() &&
9464 "Vector width mismatch between mask and data");
9465 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9466 N->getValueType(0).getVectorElementCount().isScalable() &&
9467 "Scalable flags of index and data do not match");
9468 assert(ElementCount::isKnownGE(
9469 N->getIndex().getValueType().getVectorElementCount(),
9470 N->getValueType(0).getVectorElementCount()) &&
9471 "Vector width mismatch between index and data");
9472 assert(isa<ConstantSDNode>(N->getScale()) &&
9473 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9474 "Scale should be a constant power of 2");
9475
9476 CSEMap.InsertNode(N, IP);
9477 InsertNode(N);
9478 SDValue V(N, 0);
9479 NewSDValueDbgMsg(V, "Creating new node: ", this);
9480 return V;
9481 }
9482
getMaskedScatter(SDVTList VTs,EVT MemVT,const SDLoc & dl,ArrayRef<SDValue> Ops,MachineMemOperand * MMO,ISD::MemIndexType IndexType,bool IsTrunc)9483 SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
9484 ArrayRef<SDValue> Ops,
9485 MachineMemOperand *MMO,
9486 ISD::MemIndexType IndexType,
9487 bool IsTrunc) {
9488 assert(Ops.size() == 6 && "Incompatible number of operands");
9489
9490 FoldingSetNodeID ID;
9491 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
9492 ID.AddInteger(MemVT.getRawBits());
9493 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
9494 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
9495 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9496 ID.AddInteger(MMO->getFlags());
9497 void *IP = nullptr;
9498 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9499 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
9500 return SDValue(E, 0);
9501 }
9502
9503 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
9504 VTs, MemVT, MMO, IndexType, IsTrunc);
9505 createOperands(N, Ops);
9506
9507 assert(N->getMask().getValueType().getVectorElementCount() ==
9508 N->getValue().getValueType().getVectorElementCount() &&
9509 "Vector width mismatch between mask and data");
9510 assert(
9511 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9512 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9513 "Scalable flags of index and data do not match");
9514 assert(ElementCount::isKnownGE(
9515 N->getIndex().getValueType().getVectorElementCount(),
9516 N->getValue().getValueType().getVectorElementCount()) &&
9517 "Vector width mismatch between index and data");
9518 assert(isa<ConstantSDNode>(N->getScale()) &&
9519 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9520 "Scale should be a constant power of 2");
9521
9522 CSEMap.InsertNode(N, IP);
9523 InsertNode(N);
9524 SDValue V(N, 0);
9525 NewSDValueDbgMsg(V, "Creating new node: ", this);
9526 return V;
9527 }
9528
getGetFPEnv(SDValue Chain,const SDLoc & dl,SDValue Ptr,EVT MemVT,MachineMemOperand * MMO)9529 SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
9530 EVT MemVT, MachineMemOperand *MMO) {
9531 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9532 SDVTList VTs = getVTList(MVT::Other);
9533 SDValue Ops[] = {Chain, Ptr};
9534 FoldingSetNodeID ID;
9535 AddNodeIDNode(ID, ISD::GET_FPENV_MEM, VTs, Ops);
9536 ID.AddInteger(MemVT.getRawBits());
9537 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9538 ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
9539 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9540 ID.AddInteger(MMO->getFlags());
9541 void *IP = nullptr;
9542 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9543 return SDValue(E, 0);
9544
9545 auto *N = newSDNode<FPStateAccessSDNode>(ISD::GET_FPENV_MEM, dl.getIROrder(),
9546 dl.getDebugLoc(), VTs, MemVT, MMO);
9547 createOperands(N, Ops);
9548
9549 CSEMap.InsertNode(N, IP);
9550 InsertNode(N);
9551 SDValue V(N, 0);
9552 NewSDValueDbgMsg(V, "Creating new node: ", this);
9553 return V;
9554 }
9555
getSetFPEnv(SDValue Chain,const SDLoc & dl,SDValue Ptr,EVT MemVT,MachineMemOperand * MMO)9556 SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
9557 EVT MemVT, MachineMemOperand *MMO) {
9558 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9559 SDVTList VTs = getVTList(MVT::Other);
9560 SDValue Ops[] = {Chain, Ptr};
9561 FoldingSetNodeID ID;
9562 AddNodeIDNode(ID, ISD::SET_FPENV_MEM, VTs, Ops);
9563 ID.AddInteger(MemVT.getRawBits());
9564 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9565 ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
9566 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9567 ID.AddInteger(MMO->getFlags());
9568 void *IP = nullptr;
9569 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9570 return SDValue(E, 0);
9571
9572 auto *N = newSDNode<FPStateAccessSDNode>(ISD::SET_FPENV_MEM, dl.getIROrder(),
9573 dl.getDebugLoc(), VTs, MemVT, MMO);
9574 createOperands(N, Ops);
9575
9576 CSEMap.InsertNode(N, IP);
9577 InsertNode(N);
9578 SDValue V(N, 0);
9579 NewSDValueDbgMsg(V, "Creating new node: ", this);
9580 return V;
9581 }
9582
simplifySelect(SDValue Cond,SDValue T,SDValue F)9583 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
9584 // select undef, T, F --> T (if T is a constant), otherwise F
9585 // select, ?, undef, F --> F
9586 // select, ?, T, undef --> T
9587 if (Cond.isUndef())
9588 return isConstantValueOfAnyType(T) ? T : F;
9589 if (T.isUndef())
9590 return F;
9591 if (F.isUndef())
9592 return T;
9593
9594 // select true, T, F --> T
9595 // select false, T, F --> F
9596 if (auto *CondC = dyn_cast<ConstantSDNode>(Cond))
9597 return CondC->isZero() ? F : T;
9598
9599 // TODO: This should simplify VSELECT with non-zero constant condition using
9600 // something like this (but check boolean contents to be complete?):
9601 if (ConstantSDNode *CondC = isConstOrConstSplat(Cond, /*AllowUndefs*/ false,
9602 /*AllowTruncation*/ true))
9603 if (CondC->isZero())
9604 return F;
9605
9606 // select ?, T, T --> T
9607 if (T == F)
9608 return T;
9609
9610 return SDValue();
9611 }
9612
simplifyShift(SDValue X,SDValue Y)9613 SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
9614 // shift undef, Y --> 0 (can always assume that the undef value is 0)
9615 if (X.isUndef())
9616 return getConstant(0, SDLoc(X.getNode()), X.getValueType());
9617 // shift X, undef --> undef (because it may shift by the bitwidth)
9618 if (Y.isUndef())
9619 return getUNDEF(X.getValueType());
9620
9621 // shift 0, Y --> 0
9622 // shift X, 0 --> X
9623 if (isNullOrNullSplat(X) || isNullOrNullSplat(Y))
9624 return X;
9625
9626 // shift X, C >= bitwidth(X) --> undef
9627 // All vector elements must be too big (or undef) to avoid partial undefs.
9628 auto isShiftTooBig = [X](ConstantSDNode *Val) {
9629 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
9630 };
9631 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
9632 return getUNDEF(X.getValueType());
9633
9634 return SDValue();
9635 }
9636
simplifyFPBinop(unsigned Opcode,SDValue X,SDValue Y,SDNodeFlags Flags)9637 SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
9638 SDNodeFlags Flags) {
9639 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
9640 // (an undef operand can be chosen to be Nan/Inf), then the result of this
9641 // operation is poison. That result can be relaxed to undef.
9642 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
9643 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
9644 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
9645 (YC && YC->getValueAPF().isNaN());
9646 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
9647 (YC && YC->getValueAPF().isInfinity());
9648
9649 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
9650 return getUNDEF(X.getValueType());
9651
9652 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
9653 return getUNDEF(X.getValueType());
9654
9655 if (!YC)
9656 return SDValue();
9657
9658 // X + -0.0 --> X
9659 if (Opcode == ISD::FADD)
9660 if (YC->getValueAPF().isNegZero())
9661 return X;
9662
9663 // X - +0.0 --> X
9664 if (Opcode == ISD::FSUB)
9665 if (YC->getValueAPF().isPosZero())
9666 return X;
9667
9668 // X * 1.0 --> X
9669 // X / 1.0 --> X
9670 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
9671 if (YC->getValueAPF().isExactlyValue(1.0))
9672 return X;
9673
9674 // X * 0.0 --> 0.0
9675 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
9676 if (YC->getValueAPF().isZero())
9677 return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
9678
9679 return SDValue();
9680 }
9681
getVAArg(EVT VT,const SDLoc & dl,SDValue Chain,SDValue Ptr,SDValue SV,unsigned Align)9682 SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
9683 SDValue Ptr, SDValue SV, unsigned Align) {
9684 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
9685 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
9686 }
9687
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,ArrayRef<SDUse> Ops)9688 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9689 ArrayRef<SDUse> Ops) {
9690 switch (Ops.size()) {
9691 case 0: return getNode(Opcode, DL, VT);
9692 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
9693 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
9694 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
9695 default: break;
9696 }
9697
9698 // Copy from an SDUse array into an SDValue array for use with
9699 // the regular getNode logic.
9700 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
9701 return getNode(Opcode, DL, VT, NewOps);
9702 }
9703
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,ArrayRef<SDValue> Ops)9704 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9705 ArrayRef<SDValue> Ops) {
9706 SDNodeFlags Flags;
9707 if (Inserter)
9708 Flags = Inserter->getFlags();
9709 return getNode(Opcode, DL, VT, Ops, Flags);
9710 }
9711
getNode(unsigned Opcode,const SDLoc & DL,EVT VT,ArrayRef<SDValue> Ops,const SDNodeFlags Flags)9712 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9713 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9714 unsigned NumOps = Ops.size();
9715 switch (NumOps) {
9716 case 0: return getNode(Opcode, DL, VT);
9717 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
9718 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
9719 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
9720 default: break;
9721 }
9722
9723 #ifndef NDEBUG
9724 for (const auto &Op : Ops)
9725 assert(Op.getOpcode() != ISD::DELETED_NODE &&
9726 "Operand is DELETED_NODE!");
9727 #endif
9728
9729 switch (Opcode) {
9730 default: break;
9731 case ISD::BUILD_VECTOR:
9732 // Attempt to simplify BUILD_VECTOR.
9733 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
9734 return V;
9735 break;
9736 case ISD::CONCAT_VECTORS:
9737 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
9738 return V;
9739 break;
9740 case ISD::SELECT_CC:
9741 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
9742 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
9743 "LHS and RHS of condition must have same type!");
9744 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9745 "True and False arms of SelectCC must have same type!");
9746 assert(Ops[2].getValueType() == VT &&
9747 "select_cc node must be of same type as true and false value!");
9748 assert((!Ops[0].getValueType().isVector() ||
9749 Ops[0].getValueType().getVectorElementCount() ==
9750 VT.getVectorElementCount()) &&
9751 "Expected select_cc with vector result to have the same sized "
9752 "comparison type!");
9753 break;
9754 case ISD::BR_CC:
9755 assert(NumOps == 5 && "BR_CC takes 5 operands!");
9756 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9757 "LHS/RHS of comparison should match types!");
9758 break;
9759 case ISD::VP_ADD:
9760 case ISD::VP_SUB:
9761 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
9762 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9763 Opcode = ISD::VP_XOR;
9764 break;
9765 case ISD::VP_MUL:
9766 // If it is VP_MUL mask operation then turn it to VP_AND
9767 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9768 Opcode = ISD::VP_AND;
9769 break;
9770 case ISD::VP_REDUCE_MUL:
9771 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
9772 if (VT == MVT::i1)
9773 Opcode = ISD::VP_REDUCE_AND;
9774 break;
9775 case ISD::VP_REDUCE_ADD:
9776 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
9777 if (VT == MVT::i1)
9778 Opcode = ISD::VP_REDUCE_XOR;
9779 break;
9780 case ISD::VP_REDUCE_SMAX:
9781 case ISD::VP_REDUCE_UMIN:
9782 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
9783 // VP_REDUCE_AND.
9784 if (VT == MVT::i1)
9785 Opcode = ISD::VP_REDUCE_AND;
9786 break;
9787 case ISD::VP_REDUCE_SMIN:
9788 case ISD::VP_REDUCE_UMAX:
9789 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
9790 // VP_REDUCE_OR.
9791 if (VT == MVT::i1)
9792 Opcode = ISD::VP_REDUCE_OR;
9793 break;
9794 }
9795
9796 // Memoize nodes.
9797 SDNode *N;
9798 SDVTList VTs = getVTList(VT);
9799
9800 if (VT != MVT::Glue) {
9801 FoldingSetNodeID ID;
9802 AddNodeIDNode(ID, Opcode, VTs, Ops);
9803 void *IP = nullptr;
9804
9805 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
9806 return SDValue(E, 0);
9807
9808 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9809 createOperands(N, Ops);
9810
9811 CSEMap.InsertNode(N, IP);
9812 } else {
9813 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9814 createOperands(N, Ops);
9815 }
9816
9817 N->setFlags(Flags);
9818 InsertNode(N);
9819 SDValue V(N, 0);
9820 NewSDValueDbgMsg(V, "Creating new node: ", this);
9821 return V;
9822 }
9823
getNode(unsigned Opcode,const SDLoc & DL,ArrayRef<EVT> ResultTys,ArrayRef<SDValue> Ops)9824 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
9825 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
9826 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
9827 }
9828
getNode(unsigned Opcode,const SDLoc & DL,SDVTList VTList,ArrayRef<SDValue> Ops)9829 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9830 ArrayRef<SDValue> Ops) {
9831 SDNodeFlags Flags;
9832 if (Inserter)
9833 Flags = Inserter->getFlags();
9834 return getNode(Opcode, DL, VTList, Ops, Flags);
9835 }
9836
getNode(unsigned Opcode,const SDLoc & DL,SDVTList VTList,ArrayRef<SDValue> Ops,const SDNodeFlags Flags)9837 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9838 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9839 if (VTList.NumVTs == 1)
9840 return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags);
9841
9842 #ifndef NDEBUG
9843 for (const auto &Op : Ops)
9844 assert(Op.getOpcode() != ISD::DELETED_NODE &&
9845 "Operand is DELETED_NODE!");
9846 #endif
9847
9848 switch (Opcode) {
9849 case ISD::SADDO:
9850 case ISD::UADDO:
9851 case ISD::SSUBO:
9852 case ISD::USUBO: {
9853 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9854 "Invalid add/sub overflow op!");
9855 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
9856 Ops[0].getValueType() == Ops[1].getValueType() &&
9857 Ops[0].getValueType() == VTList.VTs[0] &&
9858 "Binary operator types must match!");
9859 SDValue N1 = Ops[0], N2 = Ops[1];
9860 canonicalizeCommutativeBinop(Opcode, N1, N2);
9861
9862 // (X +- 0) -> X with zero-overflow.
9863 ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false,
9864 /*AllowTruncation*/ true);
9865 if (N2CV && N2CV->isZero()) {
9866 SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]);
9867 return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);
9868 }
9869
9870 if (VTList.VTs[0].isVector() &&
9871 VTList.VTs[0].getVectorElementType() == MVT::i1 &&
9872 VTList.VTs[1].getVectorElementType() == MVT::i1) {
9873 SDValue F1 = getFreeze(N1);
9874 SDValue F2 = getFreeze(N2);
9875 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
9876 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
9877 return getNode(ISD::MERGE_VALUES, DL, VTList,
9878 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
9879 getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)},
9880 Flags);
9881 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
9882 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
9883 SDValue NotF1 = getNOT(DL, F1, VTList.VTs[0]);
9884 return getNode(ISD::MERGE_VALUES, DL, VTList,
9885 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
9886 getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)},
9887 Flags);
9888 }
9889 }
9890 break;
9891 }
9892 case ISD::SMUL_LOHI:
9893 case ISD::UMUL_LOHI: {
9894 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
9895 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
9896 VTList.VTs[0] == Ops[0].getValueType() &&
9897 VTList.VTs[0] == Ops[1].getValueType() &&
9898 "Binary operator types must match!");
9899 // Constant fold.
9900 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Ops[0]);
9901 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ops[1]);
9902 if (LHS && RHS) {
9903 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
9904 unsigned OutWidth = Width * 2;
9905 APInt Val = LHS->getAPIntValue();
9906 APInt Mul = RHS->getAPIntValue();
9907 if (Opcode == ISD::SMUL_LOHI) {
9908 Val = Val.sext(OutWidth);
9909 Mul = Mul.sext(OutWidth);
9910 } else {
9911 Val = Val.zext(OutWidth);
9912 Mul = Mul.zext(OutWidth);
9913 }
9914 Val *= Mul;
9915
9916 SDValue Hi =
9917 getConstant(Val.extractBits(Width, Width), DL, VTList.VTs[0]);
9918 SDValue Lo = getConstant(Val.trunc(Width), DL, VTList.VTs[0]);
9919 return getNode(ISD::MERGE_VALUES, DL, VTList, {Lo, Hi}, Flags);
9920 }
9921 break;
9922 }
9923 case ISD::FFREXP: {
9924 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
9925 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
9926 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
9927
9928 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Ops[0])) {
9929 int FrexpExp;
9930 APFloat FrexpMant =
9931 frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven);
9932 SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]);
9933 SDValue Result1 =
9934 getConstant(FrexpMant.isFinite() ? FrexpExp : 0, DL, VTList.VTs[1]);
9935 return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags);
9936 }
9937
9938 break;
9939 }
9940 case ISD::STRICT_FP_EXTEND:
9941 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9942 "Invalid STRICT_FP_EXTEND!");
9943 assert(VTList.VTs[0].isFloatingPoint() &&
9944 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
9945 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9946 "STRICT_FP_EXTEND result type should be vector iff the operand "
9947 "type is vector!");
9948 assert((!VTList.VTs[0].isVector() ||
9949 VTList.VTs[0].getVectorElementCount() ==
9950 Ops[1].getValueType().getVectorElementCount()) &&
9951 "Vector element count mismatch!");
9952 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
9953 "Invalid fpext node, dst <= src!");
9954 break;
9955 case ISD::STRICT_FP_ROUND:
9956 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
9957 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9958 "STRICT_FP_ROUND result type should be vector iff the operand "
9959 "type is vector!");
9960 assert((!VTList.VTs[0].isVector() ||
9961 VTList.VTs[0].getVectorElementCount() ==
9962 Ops[1].getValueType().getVectorElementCount()) &&
9963 "Vector element count mismatch!");
9964 assert(VTList.VTs[0].isFloatingPoint() &&
9965 Ops[1].getValueType().isFloatingPoint() &&
9966 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
9967 isa<ConstantSDNode>(Ops[2]) &&
9968 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
9969 "Invalid STRICT_FP_ROUND!");
9970 break;
9971 #if 0
9972 // FIXME: figure out how to safely handle things like
9973 // int foo(int x) { return 1 << (x & 255); }
9974 // int bar() { return foo(256); }
9975 case ISD::SRA_PARTS:
9976 case ISD::SRL_PARTS:
9977 case ISD::SHL_PARTS:
9978 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
9979 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
9980 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
9981 else if (N3.getOpcode() == ISD::AND)
9982 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
9983 // If the and is only masking out bits that cannot effect the shift,
9984 // eliminate the and.
9985 unsigned NumBits = VT.getScalarSizeInBits()*2;
9986 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
9987 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
9988 }
9989 break;
9990 #endif
9991 }
9992
9993 // Memoize the node unless it returns a glue result.
9994 SDNode *N;
9995 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
9996 FoldingSetNodeID ID;
9997 AddNodeIDNode(ID, Opcode, VTList, Ops);
9998 void *IP = nullptr;
9999 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
10000 return SDValue(E, 0);
10001
10002 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
10003 createOperands(N, Ops);
10004 CSEMap.InsertNode(N, IP);
10005 } else {
10006 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
10007 createOperands(N, Ops);
10008 }
10009
10010 N->setFlags(Flags);
10011 InsertNode(N);
10012 SDValue V(N, 0);
10013 NewSDValueDbgMsg(V, "Creating new node: ", this);
10014 return V;
10015 }
10016
getNode(unsigned Opcode,const SDLoc & DL,SDVTList VTList)10017 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10018 SDVTList VTList) {
10019 return getNode(Opcode, DL, VTList, std::nullopt);
10020 }
10021
getNode(unsigned Opcode,const SDLoc & DL,SDVTList VTList,SDValue N1)10022 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10023 SDValue N1) {
10024 SDValue Ops[] = { N1 };
10025 return getNode(Opcode, DL, VTList, Ops);
10026 }
10027
getNode(unsigned Opcode,const SDLoc & DL,SDVTList VTList,SDValue N1,SDValue N2)10028 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10029 SDValue N1, SDValue N2) {
10030 SDValue Ops[] = { N1, N2 };
10031 return getNode(Opcode, DL, VTList, Ops);
10032 }
10033
getNode(unsigned Opcode,const SDLoc & DL,SDVTList VTList,SDValue N1,SDValue N2,SDValue N3)10034 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10035 SDValue N1, SDValue N2, SDValue N3) {
10036 SDValue Ops[] = { N1, N2, N3 };
10037 return getNode(Opcode, DL, VTList, Ops);
10038 }
10039
getNode(unsigned Opcode,const SDLoc & DL,SDVTList VTList,SDValue N1,SDValue N2,SDValue N3,SDValue N4)10040 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10041 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
10042 SDValue Ops[] = { N1, N2, N3, N4 };
10043 return getNode(Opcode, DL, VTList, Ops);
10044 }
10045
getNode(unsigned Opcode,const SDLoc & DL,SDVTList VTList,SDValue N1,SDValue N2,SDValue N3,SDValue N4,SDValue N5)10046 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10047 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
10048 SDValue N5) {
10049 SDValue Ops[] = { N1, N2, N3, N4, N5 };
10050 return getNode(Opcode, DL, VTList, Ops);
10051 }
10052
getVTList(EVT VT)10053 SDVTList SelectionDAG::getVTList(EVT VT) {
10054 return makeVTList(SDNode::getValueTypeList(VT), 1);
10055 }
10056
getVTList(EVT VT1,EVT VT2)10057 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
10058 FoldingSetNodeID ID;
10059 ID.AddInteger(2U);
10060 ID.AddInteger(VT1.getRawBits());
10061 ID.AddInteger(VT2.getRawBits());
10062
10063 void *IP = nullptr;
10064 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10065 if (!Result) {
10066 EVT *Array = Allocator.Allocate<EVT>(2);
10067 Array[0] = VT1;
10068 Array[1] = VT2;
10069 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
10070 VTListMap.InsertNode(Result, IP);
10071 }
10072 return Result->getSDVTList();
10073 }
10074
getVTList(EVT VT1,EVT VT2,EVT VT3)10075 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
10076 FoldingSetNodeID ID;
10077 ID.AddInteger(3U);
10078 ID.AddInteger(VT1.getRawBits());
10079 ID.AddInteger(VT2.getRawBits());
10080 ID.AddInteger(VT3.getRawBits());
10081
10082 void *IP = nullptr;
10083 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10084 if (!Result) {
10085 EVT *Array = Allocator.Allocate<EVT>(3);
10086 Array[0] = VT1;
10087 Array[1] = VT2;
10088 Array[2] = VT3;
10089 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
10090 VTListMap.InsertNode(Result, IP);
10091 }
10092 return Result->getSDVTList();
10093 }
10094
getVTList(EVT VT1,EVT VT2,EVT VT3,EVT VT4)10095 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
10096 FoldingSetNodeID ID;
10097 ID.AddInteger(4U);
10098 ID.AddInteger(VT1.getRawBits());
10099 ID.AddInteger(VT2.getRawBits());
10100 ID.AddInteger(VT3.getRawBits());
10101 ID.AddInteger(VT4.getRawBits());
10102
10103 void *IP = nullptr;
10104 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10105 if (!Result) {
10106 EVT *Array = Allocator.Allocate<EVT>(4);
10107 Array[0] = VT1;
10108 Array[1] = VT2;
10109 Array[2] = VT3;
10110 Array[3] = VT4;
10111 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
10112 VTListMap.InsertNode(Result, IP);
10113 }
10114 return Result->getSDVTList();
10115 }
10116
getVTList(ArrayRef<EVT> VTs)10117 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
10118 unsigned NumVTs = VTs.size();
10119 FoldingSetNodeID ID;
10120 ID.AddInteger(NumVTs);
10121 for (unsigned index = 0; index < NumVTs; index++) {
10122 ID.AddInteger(VTs[index].getRawBits());
10123 }
10124
10125 void *IP = nullptr;
10126 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10127 if (!Result) {
10128 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
10129 llvm::copy(VTs, Array);
10130 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
10131 VTListMap.InsertNode(Result, IP);
10132 }
10133 return Result->getSDVTList();
10134 }
10135
10136
10137 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
10138 /// specified operands. If the resultant node already exists in the DAG,
10139 /// this does not modify the specified node, instead it returns the node that
10140 /// already exists. If the resultant node does not exist in the DAG, the
10141 /// input node is returned. As a degenerate case, if you specify the same
10142 /// input operands as the node already has, the input node is returned.
UpdateNodeOperands(SDNode * N,SDValue Op)10143 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
10144 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
10145
10146 // Check to see if there is no change.
10147 if (Op == N->getOperand(0)) return N;
10148
10149 // See if the modified node already exists.
10150 void *InsertPos = nullptr;
10151 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
10152 return Existing;
10153
10154 // Nope it doesn't. Remove the node from its current place in the maps.
10155 if (InsertPos)
10156 if (!RemoveNodeFromCSEMaps(N))
10157 InsertPos = nullptr;
10158
10159 // Now we update the operands.
10160 N->OperandList[0].set(Op);
10161
10162 updateDivergence(N);
10163 // If this gets put into a CSE map, add it.
10164 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10165 return N;
10166 }
10167
UpdateNodeOperands(SDNode * N,SDValue Op1,SDValue Op2)10168 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
10169 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
10170
10171 // Check to see if there is no change.
10172 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
10173 return N; // No operands changed, just return the input node.
10174
10175 // See if the modified node already exists.
10176 void *InsertPos = nullptr;
10177 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
10178 return Existing;
10179
10180 // Nope it doesn't. Remove the node from its current place in the maps.
10181 if (InsertPos)
10182 if (!RemoveNodeFromCSEMaps(N))
10183 InsertPos = nullptr;
10184
10185 // Now we update the operands.
10186 if (N->OperandList[0] != Op1)
10187 N->OperandList[0].set(Op1);
10188 if (N->OperandList[1] != Op2)
10189 N->OperandList[1].set(Op2);
10190
10191 updateDivergence(N);
10192 // If this gets put into a CSE map, add it.
10193 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10194 return N;
10195 }
10196
10197 SDNode *SelectionDAG::
UpdateNodeOperands(SDNode * N,SDValue Op1,SDValue Op2,SDValue Op3)10198 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
10199 SDValue Ops[] = { Op1, Op2, Op3 };
10200 return UpdateNodeOperands(N, Ops);
10201 }
10202
10203 SDNode *SelectionDAG::
UpdateNodeOperands(SDNode * N,SDValue Op1,SDValue Op2,SDValue Op3,SDValue Op4)10204 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
10205 SDValue Op3, SDValue Op4) {
10206 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10207 return UpdateNodeOperands(N, Ops);
10208 }
10209
10210 SDNode *SelectionDAG::
UpdateNodeOperands(SDNode * N,SDValue Op1,SDValue Op2,SDValue Op3,SDValue Op4,SDValue Op5)10211 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
10212 SDValue Op3, SDValue Op4, SDValue Op5) {
10213 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10214 return UpdateNodeOperands(N, Ops);
10215 }
10216
10217 SDNode *SelectionDAG::
UpdateNodeOperands(SDNode * N,ArrayRef<SDValue> Ops)10218 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
10219 unsigned NumOps = Ops.size();
10220 assert(N->getNumOperands() == NumOps &&
10221 "Update with wrong number of operands");
10222
10223 // If no operands changed just return the input node.
10224 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
10225 return N;
10226
10227 // See if the modified node already exists.
10228 void *InsertPos = nullptr;
10229 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
10230 return Existing;
10231
10232 // Nope it doesn't. Remove the node from its current place in the maps.
10233 if (InsertPos)
10234 if (!RemoveNodeFromCSEMaps(N))
10235 InsertPos = nullptr;
10236
10237 // Now we update the operands.
10238 for (unsigned i = 0; i != NumOps; ++i)
10239 if (N->OperandList[i] != Ops[i])
10240 N->OperandList[i].set(Ops[i]);
10241
10242 updateDivergence(N);
10243 // If this gets put into a CSE map, add it.
10244 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10245 return N;
10246 }
10247
10248 /// DropOperands - Release the operands and set this node to have
10249 /// zero operands.
DropOperands()10250 void SDNode::DropOperands() {
10251 // Unlike the code in MorphNodeTo that does this, we don't need to
10252 // watch for dead nodes here.
10253 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
10254 SDUse &Use = *I++;
10255 Use.set(SDValue());
10256 }
10257 }
10258
setNodeMemRefs(MachineSDNode * N,ArrayRef<MachineMemOperand * > NewMemRefs)10259 void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
10260 ArrayRef<MachineMemOperand *> NewMemRefs) {
10261 if (NewMemRefs.empty()) {
10262 N->clearMemRefs();
10263 return;
10264 }
10265
10266 // Check if we can avoid allocating by storing a single reference directly.
10267 if (NewMemRefs.size() == 1) {
10268 N->MemRefs = NewMemRefs[0];
10269 N->NumMemRefs = 1;
10270 return;
10271 }
10272
10273 MachineMemOperand **MemRefsBuffer =
10274 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
10275 llvm::copy(NewMemRefs, MemRefsBuffer);
10276 N->MemRefs = MemRefsBuffer;
10277 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
10278 }
10279
10280 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
10281 /// machine opcode.
10282 ///
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT)10283 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10284 EVT VT) {
10285 SDVTList VTs = getVTList(VT);
10286 return SelectNodeTo(N, MachineOpc, VTs, std::nullopt);
10287 }
10288
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT,SDValue Op1)10289 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10290 EVT VT, SDValue Op1) {
10291 SDVTList VTs = getVTList(VT);
10292 SDValue Ops[] = { Op1 };
10293 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10294 }
10295
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT,SDValue Op1,SDValue Op2)10296 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10297 EVT VT, SDValue Op1,
10298 SDValue Op2) {
10299 SDVTList VTs = getVTList(VT);
10300 SDValue Ops[] = { Op1, Op2 };
10301 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10302 }
10303
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT,SDValue Op1,SDValue Op2,SDValue Op3)10304 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10305 EVT VT, SDValue Op1,
10306 SDValue Op2, SDValue Op3) {
10307 SDVTList VTs = getVTList(VT);
10308 SDValue Ops[] = { Op1, Op2, Op3 };
10309 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10310 }
10311
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT,ArrayRef<SDValue> Ops)10312 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10313 EVT VT, ArrayRef<SDValue> Ops) {
10314 SDVTList VTs = getVTList(VT);
10315 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10316 }
10317
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,ArrayRef<SDValue> Ops)10318 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10319 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
10320 SDVTList VTs = getVTList(VT1, VT2);
10321 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10322 }
10323
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2)10324 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10325 EVT VT1, EVT VT2) {
10326 SDVTList VTs = getVTList(VT1, VT2);
10327 return SelectNodeTo(N, MachineOpc, VTs, std::nullopt);
10328 }
10329
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,EVT VT3,ArrayRef<SDValue> Ops)10330 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10331 EVT VT1, EVT VT2, EVT VT3,
10332 ArrayRef<SDValue> Ops) {
10333 SDVTList VTs = getVTList(VT1, VT2, VT3);
10334 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10335 }
10336
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,SDValue Op1,SDValue Op2)10337 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10338 EVT VT1, EVT VT2,
10339 SDValue Op1, SDValue Op2) {
10340 SDVTList VTs = getVTList(VT1, VT2);
10341 SDValue Ops[] = { Op1, Op2 };
10342 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10343 }
10344
SelectNodeTo(SDNode * N,unsigned MachineOpc,SDVTList VTs,ArrayRef<SDValue> Ops)10345 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10346 SDVTList VTs,ArrayRef<SDValue> Ops) {
10347 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
10348 // Reset the NodeID to -1.
10349 New->setNodeId(-1);
10350 if (New != N) {
10351 ReplaceAllUsesWith(N, New);
10352 RemoveDeadNode(N);
10353 }
10354 return New;
10355 }
10356
10357 /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
10358 /// the line number information on the merged node since it is not possible to
10359 /// preserve the information that operation is associated with multiple lines.
10360 /// This will make the debugger working better at -O0, were there is a higher
10361 /// probability having other instructions associated with that line.
10362 ///
10363 /// For IROrder, we keep the smaller of the two
UpdateSDLocOnMergeSDNode(SDNode * N,const SDLoc & OLoc)10364 SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
10365 DebugLoc NLoc = N->getDebugLoc();
10366 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
10367 N->setDebugLoc(DebugLoc());
10368 }
10369 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
10370 N->setIROrder(Order);
10371 return N;
10372 }
10373
10374 /// MorphNodeTo - This *mutates* the specified node to have the specified
10375 /// return type, opcode, and operands.
10376 ///
10377 /// Note that MorphNodeTo returns the resultant node. If there is already a
10378 /// node of the specified opcode and operands, it returns that node instead of
10379 /// the current one. Note that the SDLoc need not be the same.
10380 ///
10381 /// Using MorphNodeTo is faster than creating a new node and swapping it in
10382 /// with ReplaceAllUsesWith both because it often avoids allocating a new
10383 /// node, and because it doesn't require CSE recalculation for any of
10384 /// the node's users.
10385 ///
10386 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
10387 /// As a consequence it isn't appropriate to use from within the DAG combiner or
10388 /// the legalizer which maintain worklists that would need to be updated when
10389 /// deleting things.
MorphNodeTo(SDNode * N,unsigned Opc,SDVTList VTs,ArrayRef<SDValue> Ops)10390 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
10391 SDVTList VTs, ArrayRef<SDValue> Ops) {
10392 // If an identical node already exists, use it.
10393 void *IP = nullptr;
10394 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
10395 FoldingSetNodeID ID;
10396 AddNodeIDNode(ID, Opc, VTs, Ops);
10397 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
10398 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
10399 }
10400
10401 if (!RemoveNodeFromCSEMaps(N))
10402 IP = nullptr;
10403
10404 // Start the morphing.
10405 N->NodeType = Opc;
10406 N->ValueList = VTs.VTs;
10407 N->NumValues = VTs.NumVTs;
10408
10409 // Clear the operands list, updating used nodes to remove this from their
10410 // use list. Keep track of any operands that become dead as a result.
10411 SmallPtrSet<SDNode*, 16> DeadNodeSet;
10412 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
10413 SDUse &Use = *I++;
10414 SDNode *Used = Use.getNode();
10415 Use.set(SDValue());
10416 if (Used->use_empty())
10417 DeadNodeSet.insert(Used);
10418 }
10419
10420 // For MachineNode, initialize the memory references information.
10421 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
10422 MN->clearMemRefs();
10423
10424 // Swap for an appropriately sized array from the recycler.
10425 removeOperands(N);
10426 createOperands(N, Ops);
10427
10428 // Delete any nodes that are still dead after adding the uses for the
10429 // new operands.
10430 if (!DeadNodeSet.empty()) {
10431 SmallVector<SDNode *, 16> DeadNodes;
10432 for (SDNode *N : DeadNodeSet)
10433 if (N->use_empty())
10434 DeadNodes.push_back(N);
10435 RemoveDeadNodes(DeadNodes);
10436 }
10437
10438 if (IP)
10439 CSEMap.InsertNode(N, IP); // Memoize the new node.
10440 return N;
10441 }
10442
mutateStrictFPToFP(SDNode * Node)10443 SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
10444 unsigned OrigOpc = Node->getOpcode();
10445 unsigned NewOpc;
10446 switch (OrigOpc) {
10447 default:
10448 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
10449 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10450 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
10451 #define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10452 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
10453 #include "llvm/IR/ConstrainedOps.def"
10454 }
10455
10456 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
10457
10458 // We're taking this node out of the chain, so we need to re-link things.
10459 SDValue InputChain = Node->getOperand(0);
10460 SDValue OutputChain = SDValue(Node, 1);
10461 ReplaceAllUsesOfValueWith(OutputChain, InputChain);
10462
10463 SmallVector<SDValue, 3> Ops;
10464 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
10465 Ops.push_back(Node->getOperand(i));
10466
10467 SDVTList VTs = getVTList(Node->getValueType(0));
10468 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
10469
10470 // MorphNodeTo can operate in two ways: if an existing node with the
10471 // specified operands exists, it can just return it. Otherwise, it
10472 // updates the node in place to have the requested operands.
10473 if (Res == Node) {
10474 // If we updated the node in place, reset the node ID. To the isel,
10475 // this should be just like a newly allocated machine node.
10476 Res->setNodeId(-1);
10477 } else {
10478 ReplaceAllUsesWith(Node, Res);
10479 RemoveDeadNode(Node);
10480 }
10481
10482 return Res;
10483 }
10484
10485 /// getMachineNode - These are used for target selectors to create a new node
10486 /// with specified return type(s), MachineInstr opcode, and operands.
10487 ///
10488 /// Note that getMachineNode returns the resultant node. If there is already a
10489 /// node of the specified opcode and operands, it returns that node instead of
10490 /// the current one.
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT)10491 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10492 EVT VT) {
10493 SDVTList VTs = getVTList(VT);
10494 return getMachineNode(Opcode, dl, VTs, std::nullopt);
10495 }
10496
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT,SDValue Op1)10497 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10498 EVT VT, SDValue Op1) {
10499 SDVTList VTs = getVTList(VT);
10500 SDValue Ops[] = { Op1 };
10501 return getMachineNode(Opcode, dl, VTs, Ops);
10502 }
10503
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT,SDValue Op1,SDValue Op2)10504 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10505 EVT VT, SDValue Op1, SDValue Op2) {
10506 SDVTList VTs = getVTList(VT);
10507 SDValue Ops[] = { Op1, Op2 };
10508 return getMachineNode(Opcode, dl, VTs, Ops);
10509 }
10510
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT,SDValue Op1,SDValue Op2,SDValue Op3)10511 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10512 EVT VT, SDValue Op1, SDValue Op2,
10513 SDValue Op3) {
10514 SDVTList VTs = getVTList(VT);
10515 SDValue Ops[] = { Op1, Op2, Op3 };
10516 return getMachineNode(Opcode, dl, VTs, Ops);
10517 }
10518
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT,ArrayRef<SDValue> Ops)10519 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10520 EVT VT, ArrayRef<SDValue> Ops) {
10521 SDVTList VTs = getVTList(VT);
10522 return getMachineNode(Opcode, dl, VTs, Ops);
10523 }
10524
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT1,EVT VT2,SDValue Op1,SDValue Op2)10525 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10526 EVT VT1, EVT VT2, SDValue Op1,
10527 SDValue Op2) {
10528 SDVTList VTs = getVTList(VT1, VT2);
10529 SDValue Ops[] = { Op1, Op2 };
10530 return getMachineNode(Opcode, dl, VTs, Ops);
10531 }
10532
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT1,EVT VT2,SDValue Op1,SDValue Op2,SDValue Op3)10533 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10534 EVT VT1, EVT VT2, SDValue Op1,
10535 SDValue Op2, SDValue Op3) {
10536 SDVTList VTs = getVTList(VT1, VT2);
10537 SDValue Ops[] = { Op1, Op2, Op3 };
10538 return getMachineNode(Opcode, dl, VTs, Ops);
10539 }
10540
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT1,EVT VT2,ArrayRef<SDValue> Ops)10541 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10542 EVT VT1, EVT VT2,
10543 ArrayRef<SDValue> Ops) {
10544 SDVTList VTs = getVTList(VT1, VT2);
10545 return getMachineNode(Opcode, dl, VTs, Ops);
10546 }
10547
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT1,EVT VT2,EVT VT3,SDValue Op1,SDValue Op2)10548 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10549 EVT VT1, EVT VT2, EVT VT3,
10550 SDValue Op1, SDValue Op2) {
10551 SDVTList VTs = getVTList(VT1, VT2, VT3);
10552 SDValue Ops[] = { Op1, Op2 };
10553 return getMachineNode(Opcode, dl, VTs, Ops);
10554 }
10555
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT1,EVT VT2,EVT VT3,SDValue Op1,SDValue Op2,SDValue Op3)10556 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10557 EVT VT1, EVT VT2, EVT VT3,
10558 SDValue Op1, SDValue Op2,
10559 SDValue Op3) {
10560 SDVTList VTs = getVTList(VT1, VT2, VT3);
10561 SDValue Ops[] = { Op1, Op2, Op3 };
10562 return getMachineNode(Opcode, dl, VTs, Ops);
10563 }
10564
getMachineNode(unsigned Opcode,const SDLoc & dl,EVT VT1,EVT VT2,EVT VT3,ArrayRef<SDValue> Ops)10565 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10566 EVT VT1, EVT VT2, EVT VT3,
10567 ArrayRef<SDValue> Ops) {
10568 SDVTList VTs = getVTList(VT1, VT2, VT3);
10569 return getMachineNode(Opcode, dl, VTs, Ops);
10570 }
10571
getMachineNode(unsigned Opcode,const SDLoc & dl,ArrayRef<EVT> ResultTys,ArrayRef<SDValue> Ops)10572 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10573 ArrayRef<EVT> ResultTys,
10574 ArrayRef<SDValue> Ops) {
10575 SDVTList VTs = getVTList(ResultTys);
10576 return getMachineNode(Opcode, dl, VTs, Ops);
10577 }
10578
getMachineNode(unsigned Opcode,const SDLoc & DL,SDVTList VTs,ArrayRef<SDValue> Ops)10579 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
10580 SDVTList VTs,
10581 ArrayRef<SDValue> Ops) {
10582 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
10583 MachineSDNode *N;
10584 void *IP = nullptr;
10585
10586 if (DoCSE) {
10587 FoldingSetNodeID ID;
10588 AddNodeIDNode(ID, ~Opcode, VTs, Ops);
10589 IP = nullptr;
10590 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
10591 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
10592 }
10593 }
10594
10595 // Allocate a new MachineSDNode.
10596 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
10597 createOperands(N, Ops);
10598
10599 if (DoCSE)
10600 CSEMap.InsertNode(N, IP);
10601
10602 InsertNode(N);
10603 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
10604 return N;
10605 }
10606
10607 /// getTargetExtractSubreg - A convenience function for creating
10608 /// TargetOpcode::EXTRACT_SUBREG nodes.
getTargetExtractSubreg(int SRIdx,const SDLoc & DL,EVT VT,SDValue Operand)10609 SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
10610 SDValue Operand) {
10611 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10612 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
10613 VT, Operand, SRIdxVal);
10614 return SDValue(Subreg, 0);
10615 }
10616
10617 /// getTargetInsertSubreg - A convenience function for creating
10618 /// TargetOpcode::INSERT_SUBREG nodes.
getTargetInsertSubreg(int SRIdx,const SDLoc & DL,EVT VT,SDValue Operand,SDValue Subreg)10619 SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
10620 SDValue Operand, SDValue Subreg) {
10621 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10622 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
10623 VT, Operand, Subreg, SRIdxVal);
10624 return SDValue(Result, 0);
10625 }
10626
10627 /// getNodeIfExists - Get the specified node if it's already available, or
10628 /// else return NULL.
getNodeIfExists(unsigned Opcode,SDVTList VTList,ArrayRef<SDValue> Ops)10629 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
10630 ArrayRef<SDValue> Ops) {
10631 SDNodeFlags Flags;
10632 if (Inserter)
10633 Flags = Inserter->getFlags();
10634 return getNodeIfExists(Opcode, VTList, Ops, Flags);
10635 }
10636
getNodeIfExists(unsigned Opcode,SDVTList VTList,ArrayRef<SDValue> Ops,const SDNodeFlags Flags)10637 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
10638 ArrayRef<SDValue> Ops,
10639 const SDNodeFlags Flags) {
10640 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10641 FoldingSetNodeID ID;
10642 AddNodeIDNode(ID, Opcode, VTList, Ops);
10643 void *IP = nullptr;
10644 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
10645 E->intersectFlagsWith(Flags);
10646 return E;
10647 }
10648 }
10649 return nullptr;
10650 }
10651
10652 /// doesNodeExist - Check if a node exists without modifying its flags.
doesNodeExist(unsigned Opcode,SDVTList VTList,ArrayRef<SDValue> Ops)10653 bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
10654 ArrayRef<SDValue> Ops) {
10655 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10656 FoldingSetNodeID ID;
10657 AddNodeIDNode(ID, Opcode, VTList, Ops);
10658 void *IP = nullptr;
10659 if (FindNodeOrInsertPos(ID, SDLoc(), IP))
10660 return true;
10661 }
10662 return false;
10663 }
10664
10665 /// getDbgValue - Creates a SDDbgValue node.
10666 ///
10667 /// SDNode
getDbgValue(DIVariable * Var,DIExpression * Expr,SDNode * N,unsigned R,bool IsIndirect,const DebugLoc & DL,unsigned O)10668 SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
10669 SDNode *N, unsigned R, bool IsIndirect,
10670 const DebugLoc &DL, unsigned O) {
10671 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10672 "Expected inlined-at fields to agree");
10673 return new (DbgInfo->getAlloc())
10674 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
10675 {}, IsIndirect, DL, O,
10676 /*IsVariadic=*/false);
10677 }
10678
10679 /// Constant
getConstantDbgValue(DIVariable * Var,DIExpression * Expr,const Value * C,const DebugLoc & DL,unsigned O)10680 SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
10681 DIExpression *Expr,
10682 const Value *C,
10683 const DebugLoc &DL, unsigned O) {
10684 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10685 "Expected inlined-at fields to agree");
10686 return new (DbgInfo->getAlloc())
10687 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
10688 /*IsIndirect=*/false, DL, O,
10689 /*IsVariadic=*/false);
10690 }
10691
10692 /// FrameIndex
getFrameIndexDbgValue(DIVariable * Var,DIExpression * Expr,unsigned FI,bool IsIndirect,const DebugLoc & DL,unsigned O)10693 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
10694 DIExpression *Expr, unsigned FI,
10695 bool IsIndirect,
10696 const DebugLoc &DL,
10697 unsigned O) {
10698 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10699 "Expected inlined-at fields to agree");
10700 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
10701 }
10702
10703 /// FrameIndex with dependencies
getFrameIndexDbgValue(DIVariable * Var,DIExpression * Expr,unsigned FI,ArrayRef<SDNode * > Dependencies,bool IsIndirect,const DebugLoc & DL,unsigned O)10704 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
10705 DIExpression *Expr, unsigned FI,
10706 ArrayRef<SDNode *> Dependencies,
10707 bool IsIndirect,
10708 const DebugLoc &DL,
10709 unsigned O) {
10710 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10711 "Expected inlined-at fields to agree");
10712 return new (DbgInfo->getAlloc())
10713 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
10714 Dependencies, IsIndirect, DL, O,
10715 /*IsVariadic=*/false);
10716 }
10717
10718 /// VReg
getVRegDbgValue(DIVariable * Var,DIExpression * Expr,unsigned VReg,bool IsIndirect,const DebugLoc & DL,unsigned O)10719 SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
10720 unsigned VReg, bool IsIndirect,
10721 const DebugLoc &DL, unsigned O) {
10722 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10723 "Expected inlined-at fields to agree");
10724 return new (DbgInfo->getAlloc())
10725 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
10726 {}, IsIndirect, DL, O,
10727 /*IsVariadic=*/false);
10728 }
10729
getDbgValueList(DIVariable * Var,DIExpression * Expr,ArrayRef<SDDbgOperand> Locs,ArrayRef<SDNode * > Dependencies,bool IsIndirect,const DebugLoc & DL,unsigned O,bool IsVariadic)10730 SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
10731 ArrayRef<SDDbgOperand> Locs,
10732 ArrayRef<SDNode *> Dependencies,
10733 bool IsIndirect, const DebugLoc &DL,
10734 unsigned O, bool IsVariadic) {
10735 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10736 "Expected inlined-at fields to agree");
10737 return new (DbgInfo->getAlloc())
10738 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
10739 DL, O, IsVariadic);
10740 }
10741
transferDbgValues(SDValue From,SDValue To,unsigned OffsetInBits,unsigned SizeInBits,bool InvalidateDbg)10742 void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
10743 unsigned OffsetInBits, unsigned SizeInBits,
10744 bool InvalidateDbg) {
10745 SDNode *FromNode = From.getNode();
10746 SDNode *ToNode = To.getNode();
10747 assert(FromNode && ToNode && "Can't modify dbg values");
10748
10749 // PR35338
10750 // TODO: assert(From != To && "Redundant dbg value transfer");
10751 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
10752 if (From == To || FromNode == ToNode)
10753 return;
10754
10755 if (!FromNode->getHasDebugValue())
10756 return;
10757
10758 SDDbgOperand FromLocOp =
10759 SDDbgOperand::fromNode(From.getNode(), From.getResNo());
10760 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo());
10761
10762 SmallVector<SDDbgValue *, 2> ClonedDVs;
10763 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
10764 if (Dbg->isInvalidated())
10765 continue;
10766
10767 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
10768
10769 // Create a new location ops vector that is equal to the old vector, but
10770 // with each instance of FromLocOp replaced with ToLocOp.
10771 bool Changed = false;
10772 auto NewLocOps = Dbg->copyLocationOps();
10773 std::replace_if(
10774 NewLocOps.begin(), NewLocOps.end(),
10775 [&Changed, FromLocOp](const SDDbgOperand &Op) {
10776 bool Match = Op == FromLocOp;
10777 Changed |= Match;
10778 return Match;
10779 },
10780 ToLocOp);
10781 // Ignore this SDDbgValue if we didn't find a matching location.
10782 if (!Changed)
10783 continue;
10784
10785 DIVariable *Var = Dbg->getVariable();
10786 auto *Expr = Dbg->getExpression();
10787 // If a fragment is requested, update the expression.
10788 if (SizeInBits) {
10789 // When splitting a larger (e.g., sign-extended) value whose
10790 // lower bits are described with an SDDbgValue, do not attempt
10791 // to transfer the SDDbgValue to the upper bits.
10792 if (auto FI = Expr->getFragmentInfo())
10793 if (OffsetInBits + SizeInBits > FI->SizeInBits)
10794 continue;
10795 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
10796 SizeInBits);
10797 if (!Fragment)
10798 continue;
10799 Expr = *Fragment;
10800 }
10801
10802 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
10803 // Clone the SDDbgValue and move it to To.
10804 SDDbgValue *Clone = getDbgValueList(
10805 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
10806 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
10807 Dbg->isVariadic());
10808 ClonedDVs.push_back(Clone);
10809
10810 if (InvalidateDbg) {
10811 // Invalidate value and indicate the SDDbgValue should not be emitted.
10812 Dbg->setIsInvalidated();
10813 Dbg->setIsEmitted();
10814 }
10815 }
10816
10817 for (SDDbgValue *Dbg : ClonedDVs) {
10818 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
10819 "Transferred DbgValues should depend on the new SDNode");
10820 AddDbgValue(Dbg, false);
10821 }
10822 }
10823
salvageDebugInfo(SDNode & N)10824 void SelectionDAG::salvageDebugInfo(SDNode &N) {
10825 if (!N.getHasDebugValue())
10826 return;
10827
10828 SmallVector<SDDbgValue *, 2> ClonedDVs;
10829 for (auto *DV : GetDbgValues(&N)) {
10830 if (DV->isInvalidated())
10831 continue;
10832 switch (N.getOpcode()) {
10833 default:
10834 break;
10835 case ISD::ADD: {
10836 SDValue N0 = N.getOperand(0);
10837 SDValue N1 = N.getOperand(1);
10838 if (!isa<ConstantSDNode>(N0)) {
10839 bool RHSConstant = isa<ConstantSDNode>(N1);
10840 uint64_t Offset;
10841 if (RHSConstant)
10842 Offset = N.getConstantOperandVal(1);
10843 // We are not allowed to turn indirect debug values variadic, so
10844 // don't salvage those.
10845 if (!RHSConstant && DV->isIndirect())
10846 continue;
10847
10848 // Rewrite an ADD constant node into a DIExpression. Since we are
10849 // performing arithmetic to compute the variable's *value* in the
10850 // DIExpression, we need to mark the expression with a
10851 // DW_OP_stack_value.
10852 auto *DIExpr = DV->getExpression();
10853 auto NewLocOps = DV->copyLocationOps();
10854 bool Changed = false;
10855 size_t OrigLocOpsSize = NewLocOps.size();
10856 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
10857 // We're not given a ResNo to compare against because the whole
10858 // node is going away. We know that any ISD::ADD only has one
10859 // result, so we can assume any node match is using the result.
10860 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10861 NewLocOps[i].getSDNode() != &N)
10862 continue;
10863 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
10864 if (RHSConstant) {
10865 SmallVector<uint64_t, 3> ExprOps;
10866 DIExpression::appendOffset(ExprOps, Offset);
10867 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
10868 } else {
10869 // Convert to a variadic expression (if not already).
10870 // convertToVariadicExpression() returns a const pointer, so we use
10871 // a temporary const variable here.
10872 const auto *TmpDIExpr =
10873 DIExpression::convertToVariadicExpression(DIExpr);
10874 SmallVector<uint64_t, 3> ExprOps;
10875 ExprOps.push_back(dwarf::DW_OP_LLVM_arg);
10876 ExprOps.push_back(NewLocOps.size());
10877 ExprOps.push_back(dwarf::DW_OP_plus);
10878 SDDbgOperand RHS =
10879 SDDbgOperand::fromNode(N1.getNode(), N1.getResNo());
10880 NewLocOps.push_back(RHS);
10881 DIExpr = DIExpression::appendOpsToArg(TmpDIExpr, ExprOps, i, true);
10882 }
10883 Changed = true;
10884 }
10885 (void)Changed;
10886 assert(Changed && "Salvage target doesn't use N");
10887
10888 bool IsVariadic =
10889 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
10890
10891 auto AdditionalDependencies = DV->getAdditionalDependencies();
10892 SDDbgValue *Clone = getDbgValueList(
10893 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
10894 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
10895 ClonedDVs.push_back(Clone);
10896 DV->setIsInvalidated();
10897 DV->setIsEmitted();
10898 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
10899 N0.getNode()->dumprFull(this);
10900 dbgs() << " into " << *DIExpr << '\n');
10901 }
10902 break;
10903 }
10904 case ISD::TRUNCATE: {
10905 SDValue N0 = N.getOperand(0);
10906 TypeSize FromSize = N0.getValueSizeInBits();
10907 TypeSize ToSize = N.getValueSizeInBits(0);
10908
10909 DIExpression *DbgExpression = DV->getExpression();
10910 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, false);
10911 auto NewLocOps = DV->copyLocationOps();
10912 bool Changed = false;
10913 for (size_t i = 0; i < NewLocOps.size(); ++i) {
10914 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10915 NewLocOps[i].getSDNode() != &N)
10916 continue;
10917
10918 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
10919 DbgExpression = DIExpression::appendOpsToArg(DbgExpression, ExtOps, i);
10920 Changed = true;
10921 }
10922 assert(Changed && "Salvage target doesn't use N");
10923 (void)Changed;
10924
10925 SDDbgValue *Clone =
10926 getDbgValueList(DV->getVariable(), DbgExpression, NewLocOps,
10927 DV->getAdditionalDependencies(), DV->isIndirect(),
10928 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
10929
10930 ClonedDVs.push_back(Clone);
10931 DV->setIsInvalidated();
10932 DV->setIsEmitted();
10933 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
10934 dbgs() << " into " << *DbgExpression << '\n');
10935 break;
10936 }
10937 }
10938 }
10939
10940 for (SDDbgValue *Dbg : ClonedDVs) {
10941 assert(!Dbg->getSDNodes().empty() &&
10942 "Salvaged DbgValue should depend on a new SDNode");
10943 AddDbgValue(Dbg, false);
10944 }
10945 }
10946
10947 /// Creates a SDDbgLabel node.
getDbgLabel(DILabel * Label,const DebugLoc & DL,unsigned O)10948 SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
10949 const DebugLoc &DL, unsigned O) {
10950 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
10951 "Expected inlined-at fields to agree");
10952 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
10953 }
10954
10955 namespace {
10956
10957 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
10958 /// pointed to by a use iterator is deleted, increment the use iterator
10959 /// so that it doesn't dangle.
10960 ///
10961 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
10962 SDNode::use_iterator &UI;
10963 SDNode::use_iterator &UE;
10964
NodeDeleted(SDNode * N,SDNode * E)10965 void NodeDeleted(SDNode *N, SDNode *E) override {
10966 // Increment the iterator as needed.
10967 while (UI != UE && N == *UI)
10968 ++UI;
10969 }
10970
10971 public:
RAUWUpdateListener(SelectionDAG & d,SDNode::use_iterator & ui,SDNode::use_iterator & ue)10972 RAUWUpdateListener(SelectionDAG &d,
10973 SDNode::use_iterator &ui,
10974 SDNode::use_iterator &ue)
10975 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
10976 };
10977
10978 } // end anonymous namespace
10979
10980 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
10981 /// This can cause recursive merging of nodes in the DAG.
10982 ///
10983 /// This version assumes From has a single result value.
10984 ///
ReplaceAllUsesWith(SDValue FromN,SDValue To)10985 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
10986 SDNode *From = FromN.getNode();
10987 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
10988 "Cannot replace with this method!");
10989 assert(From != To.getNode() && "Cannot replace uses of with self");
10990
10991 // Preserve Debug Values
10992 transferDbgValues(FromN, To);
10993 // Preserve extra info.
10994 copyExtraInfo(From, To.getNode());
10995
10996 // Iterate over all the existing uses of From. New uses will be added
10997 // to the beginning of the use list, which we avoid visiting.
10998 // This specifically avoids visiting uses of From that arise while the
10999 // replacement is happening, because any such uses would be the result
11000 // of CSE: If an existing node looks like From after one of its operands
11001 // is replaced by To, we don't want to replace of all its users with To
11002 // too. See PR3018 for more info.
11003 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11004 RAUWUpdateListener Listener(*this, UI, UE);
11005 while (UI != UE) {
11006 SDNode *User = *UI;
11007
11008 // This node is about to morph, remove its old self from the CSE maps.
11009 RemoveNodeFromCSEMaps(User);
11010
11011 // A user can appear in a use list multiple times, and when this
11012 // happens the uses are usually next to each other in the list.
11013 // To help reduce the number of CSE recomputations, process all
11014 // the uses of this user that we can find this way.
11015 do {
11016 SDUse &Use = UI.getUse();
11017 ++UI;
11018 Use.set(To);
11019 if (To->isDivergent() != From->isDivergent())
11020 updateDivergence(User);
11021 } while (UI != UE && *UI == User);
11022 // Now that we have modified User, add it back to the CSE maps. If it
11023 // already exists there, recursively merge the results together.
11024 AddModifiedNodeToCSEMaps(User);
11025 }
11026
11027 // If we just RAUW'd the root, take note.
11028 if (FromN == getRoot())
11029 setRoot(To);
11030 }
11031
11032 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11033 /// This can cause recursive merging of nodes in the DAG.
11034 ///
11035 /// This version assumes that for each value of From, there is a
11036 /// corresponding value in To in the same position with the same type.
11037 ///
ReplaceAllUsesWith(SDNode * From,SDNode * To)11038 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
11039 #ifndef NDEBUG
11040 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11041 assert((!From->hasAnyUseOfValue(i) ||
11042 From->getValueType(i) == To->getValueType(i)) &&
11043 "Cannot use this version of ReplaceAllUsesWith!");
11044 #endif
11045
11046 // Handle the trivial case.
11047 if (From == To)
11048 return;
11049
11050 // Preserve Debug Info. Only do this if there's a use.
11051 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11052 if (From->hasAnyUseOfValue(i)) {
11053 assert((i < To->getNumValues()) && "Invalid To location");
11054 transferDbgValues(SDValue(From, i), SDValue(To, i));
11055 }
11056 // Preserve extra info.
11057 copyExtraInfo(From, To);
11058
11059 // Iterate over just the existing users of From. See the comments in
11060 // the ReplaceAllUsesWith above.
11061 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11062 RAUWUpdateListener Listener(*this, UI, UE);
11063 while (UI != UE) {
11064 SDNode *User = *UI;
11065
11066 // This node is about to morph, remove its old self from the CSE maps.
11067 RemoveNodeFromCSEMaps(User);
11068
11069 // A user can appear in a use list multiple times, and when this
11070 // happens the uses are usually next to each other in the list.
11071 // To help reduce the number of CSE recomputations, process all
11072 // the uses of this user that we can find this way.
11073 do {
11074 SDUse &Use = UI.getUse();
11075 ++UI;
11076 Use.setNode(To);
11077 if (To->isDivergent() != From->isDivergent())
11078 updateDivergence(User);
11079 } while (UI != UE && *UI == User);
11080
11081 // Now that we have modified User, add it back to the CSE maps. If it
11082 // already exists there, recursively merge the results together.
11083 AddModifiedNodeToCSEMaps(User);
11084 }
11085
11086 // If we just RAUW'd the root, take note.
11087 if (From == getRoot().getNode())
11088 setRoot(SDValue(To, getRoot().getResNo()));
11089 }
11090
11091 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11092 /// This can cause recursive merging of nodes in the DAG.
11093 ///
11094 /// This version can replace From with any result values. To must match the
11095 /// number and types of values returned by From.
ReplaceAllUsesWith(SDNode * From,const SDValue * To)11096 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
11097 if (From->getNumValues() == 1) // Handle the simple case efficiently.
11098 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
11099
11100 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
11101 // Preserve Debug Info.
11102 transferDbgValues(SDValue(From, i), To[i]);
11103 // Preserve extra info.
11104 copyExtraInfo(From, To[i].getNode());
11105 }
11106
11107 // Iterate over just the existing users of From. See the comments in
11108 // the ReplaceAllUsesWith above.
11109 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11110 RAUWUpdateListener Listener(*this, UI, UE);
11111 while (UI != UE) {
11112 SDNode *User = *UI;
11113
11114 // This node is about to morph, remove its old self from the CSE maps.
11115 RemoveNodeFromCSEMaps(User);
11116
11117 // A user can appear in a use list multiple times, and when this happens the
11118 // uses are usually next to each other in the list. To help reduce the
11119 // number of CSE and divergence recomputations, process all the uses of this
11120 // user that we can find this way.
11121 bool To_IsDivergent = false;
11122 do {
11123 SDUse &Use = UI.getUse();
11124 const SDValue &ToOp = To[Use.getResNo()];
11125 ++UI;
11126 Use.set(ToOp);
11127 To_IsDivergent |= ToOp->isDivergent();
11128 } while (UI != UE && *UI == User);
11129
11130 if (To_IsDivergent != From->isDivergent())
11131 updateDivergence(User);
11132
11133 // Now that we have modified User, add it back to the CSE maps. If it
11134 // already exists there, recursively merge the results together.
11135 AddModifiedNodeToCSEMaps(User);
11136 }
11137
11138 // If we just RAUW'd the root, take note.
11139 if (From == getRoot().getNode())
11140 setRoot(SDValue(To[getRoot().getResNo()]));
11141 }
11142
11143 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
11144 /// uses of other values produced by From.getNode() alone. The Deleted
11145 /// vector is handled the same way as for ReplaceAllUsesWith.
ReplaceAllUsesOfValueWith(SDValue From,SDValue To)11146 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
11147 // Handle the really simple, really trivial case efficiently.
11148 if (From == To) return;
11149
11150 // Handle the simple, trivial, case efficiently.
11151 if (From.getNode()->getNumValues() == 1) {
11152 ReplaceAllUsesWith(From, To);
11153 return;
11154 }
11155
11156 // Preserve Debug Info.
11157 transferDbgValues(From, To);
11158 copyExtraInfo(From.getNode(), To.getNode());
11159
11160 // Iterate over just the existing users of From. See the comments in
11161 // the ReplaceAllUsesWith above.
11162 SDNode::use_iterator UI = From.getNode()->use_begin(),
11163 UE = From.getNode()->use_end();
11164 RAUWUpdateListener Listener(*this, UI, UE);
11165 while (UI != UE) {
11166 SDNode *User = *UI;
11167 bool UserRemovedFromCSEMaps = false;
11168
11169 // A user can appear in a use list multiple times, and when this
11170 // happens the uses are usually next to each other in the list.
11171 // To help reduce the number of CSE recomputations, process all
11172 // the uses of this user that we can find this way.
11173 do {
11174 SDUse &Use = UI.getUse();
11175
11176 // Skip uses of different values from the same node.
11177 if (Use.getResNo() != From.getResNo()) {
11178 ++UI;
11179 continue;
11180 }
11181
11182 // If this node hasn't been modified yet, it's still in the CSE maps,
11183 // so remove its old self from the CSE maps.
11184 if (!UserRemovedFromCSEMaps) {
11185 RemoveNodeFromCSEMaps(User);
11186 UserRemovedFromCSEMaps = true;
11187 }
11188
11189 ++UI;
11190 Use.set(To);
11191 if (To->isDivergent() != From->isDivergent())
11192 updateDivergence(User);
11193 } while (UI != UE && *UI == User);
11194 // We are iterating over all uses of the From node, so if a use
11195 // doesn't use the specific value, no changes are made.
11196 if (!UserRemovedFromCSEMaps)
11197 continue;
11198
11199 // Now that we have modified User, add it back to the CSE maps. If it
11200 // already exists there, recursively merge the results together.
11201 AddModifiedNodeToCSEMaps(User);
11202 }
11203
11204 // If we just RAUW'd the root, take note.
11205 if (From == getRoot())
11206 setRoot(To);
11207 }
11208
11209 namespace {
11210
11211 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
11212 /// to record information about a use.
11213 struct UseMemo {
11214 SDNode *User;
11215 unsigned Index;
11216 SDUse *Use;
11217 };
11218
11219 /// operator< - Sort Memos by User.
operator <(const UseMemo & L,const UseMemo & R)11220 bool operator<(const UseMemo &L, const UseMemo &R) {
11221 return (intptr_t)L.User < (intptr_t)R.User;
11222 }
11223
11224 /// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
11225 /// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
11226 /// the node already has been taken care of recursively.
11227 class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
11228 SmallVector<UseMemo, 4> &Uses;
11229
NodeDeleted(SDNode * N,SDNode * E)11230 void NodeDeleted(SDNode *N, SDNode *E) override {
11231 for (UseMemo &Memo : Uses)
11232 if (Memo.User == N)
11233 Memo.User = nullptr;
11234 }
11235
11236 public:
RAUOVWUpdateListener(SelectionDAG & d,SmallVector<UseMemo,4> & uses)11237 RAUOVWUpdateListener(SelectionDAG &d, SmallVector<UseMemo, 4> &uses)
11238 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
11239 };
11240
11241 } // end anonymous namespace
11242
calculateDivergence(SDNode * N)11243 bool SelectionDAG::calculateDivergence(SDNode *N) {
11244 if (TLI->isSDNodeAlwaysUniform(N)) {
11245 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
11246 "Conflicting divergence information!");
11247 return false;
11248 }
11249 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
11250 return true;
11251 for (const auto &Op : N->ops()) {
11252 if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
11253 return true;
11254 }
11255 return false;
11256 }
11257
updateDivergence(SDNode * N)11258 void SelectionDAG::updateDivergence(SDNode *N) {
11259 SmallVector<SDNode *, 16> Worklist(1, N);
11260 do {
11261 N = Worklist.pop_back_val();
11262 bool IsDivergent = calculateDivergence(N);
11263 if (N->SDNodeBits.IsDivergent != IsDivergent) {
11264 N->SDNodeBits.IsDivergent = IsDivergent;
11265 llvm::append_range(Worklist, N->uses());
11266 }
11267 } while (!Worklist.empty());
11268 }
11269
CreateTopologicalOrder(std::vector<SDNode * > & Order)11270 void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11271 DenseMap<SDNode *, unsigned> Degree;
11272 Order.reserve(AllNodes.size());
11273 for (auto &N : allnodes()) {
11274 unsigned NOps = N.getNumOperands();
11275 Degree[&N] = NOps;
11276 if (0 == NOps)
11277 Order.push_back(&N);
11278 }
11279 for (size_t I = 0; I != Order.size(); ++I) {
11280 SDNode *N = Order[I];
11281 for (auto *U : N->uses()) {
11282 unsigned &UnsortedOps = Degree[U];
11283 if (0 == --UnsortedOps)
11284 Order.push_back(U);
11285 }
11286 }
11287 }
11288
11289 #ifndef NDEBUG
VerifyDAGDivergence()11290 void SelectionDAG::VerifyDAGDivergence() {
11291 std::vector<SDNode *> TopoOrder;
11292 CreateTopologicalOrder(TopoOrder);
11293 for (auto *N : TopoOrder) {
11294 assert(calculateDivergence(N) == N->isDivergent() &&
11295 "Divergence bit inconsistency detected");
11296 }
11297 }
11298 #endif
11299
11300 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
11301 /// uses of other values produced by From.getNode() alone. The same value
11302 /// may appear in both the From and To list. The Deleted vector is
11303 /// handled the same way as for ReplaceAllUsesWith.
ReplaceAllUsesOfValuesWith(const SDValue * From,const SDValue * To,unsigned Num)11304 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
11305 const SDValue *To,
11306 unsigned Num){
11307 // Handle the simple, trivial case efficiently.
11308 if (Num == 1)
11309 return ReplaceAllUsesOfValueWith(*From, *To);
11310
11311 transferDbgValues(*From, *To);
11312 copyExtraInfo(From->getNode(), To->getNode());
11313
11314 // Read up all the uses and make records of them. This helps
11315 // processing new uses that are introduced during the
11316 // replacement process.
11317 SmallVector<UseMemo, 4> Uses;
11318 for (unsigned i = 0; i != Num; ++i) {
11319 unsigned FromResNo = From[i].getResNo();
11320 SDNode *FromNode = From[i].getNode();
11321 for (SDNode::use_iterator UI = FromNode->use_begin(),
11322 E = FromNode->use_end(); UI != E; ++UI) {
11323 SDUse &Use = UI.getUse();
11324 if (Use.getResNo() == FromResNo) {
11325 UseMemo Memo = { *UI, i, &Use };
11326 Uses.push_back(Memo);
11327 }
11328 }
11329 }
11330
11331 // Sort the uses, so that all the uses from a given User are together.
11332 llvm::sort(Uses);
11333 RAUOVWUpdateListener Listener(*this, Uses);
11334
11335 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
11336 UseIndex != UseIndexEnd; ) {
11337 // We know that this user uses some value of From. If it is the right
11338 // value, update it.
11339 SDNode *User = Uses[UseIndex].User;
11340 // If the node has been deleted by recursive CSE updates when updating
11341 // another node, then just skip this entry.
11342 if (User == nullptr) {
11343 ++UseIndex;
11344 continue;
11345 }
11346
11347 // This node is about to morph, remove its old self from the CSE maps.
11348 RemoveNodeFromCSEMaps(User);
11349
11350 // The Uses array is sorted, so all the uses for a given User
11351 // are next to each other in the list.
11352 // To help reduce the number of CSE recomputations, process all
11353 // the uses of this user that we can find this way.
11354 do {
11355 unsigned i = Uses[UseIndex].Index;
11356 SDUse &Use = *Uses[UseIndex].Use;
11357 ++UseIndex;
11358
11359 Use.set(To[i]);
11360 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
11361
11362 // Now that we have modified User, add it back to the CSE maps. If it
11363 // already exists there, recursively merge the results together.
11364 AddModifiedNodeToCSEMaps(User);
11365 }
11366 }
11367
11368 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
11369 /// based on their topological order. It returns the maximum id and a vector
11370 /// of the SDNodes* in assigned order by reference.
AssignTopologicalOrder()11371 unsigned SelectionDAG::AssignTopologicalOrder() {
11372 unsigned DAGSize = 0;
11373
11374 // SortedPos tracks the progress of the algorithm. Nodes before it are
11375 // sorted, nodes after it are unsorted. When the algorithm completes
11376 // it is at the end of the list.
11377 allnodes_iterator SortedPos = allnodes_begin();
11378
11379 // Visit all the nodes. Move nodes with no operands to the front of
11380 // the list immediately. Annotate nodes that do have operands with their
11381 // operand count. Before we do this, the Node Id fields of the nodes
11382 // may contain arbitrary values. After, the Node Id fields for nodes
11383 // before SortedPos will contain the topological sort index, and the
11384 // Node Id fields for nodes At SortedPos and after will contain the
11385 // count of outstanding operands.
11386 for (SDNode &N : llvm::make_early_inc_range(allnodes())) {
11387 checkForCycles(&N, this);
11388 unsigned Degree = N.getNumOperands();
11389 if (Degree == 0) {
11390 // A node with no uses, add it to the result array immediately.
11391 N.setNodeId(DAGSize++);
11392 allnodes_iterator Q(&N);
11393 if (Q != SortedPos)
11394 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
11395 assert(SortedPos != AllNodes.end() && "Overran node list");
11396 ++SortedPos;
11397 } else {
11398 // Temporarily use the Node Id as scratch space for the degree count.
11399 N.setNodeId(Degree);
11400 }
11401 }
11402
11403 // Visit all the nodes. As we iterate, move nodes into sorted order,
11404 // such that by the time the end is reached all nodes will be sorted.
11405 for (SDNode &Node : allnodes()) {
11406 SDNode *N = &Node;
11407 checkForCycles(N, this);
11408 // N is in sorted position, so all its uses have one less operand
11409 // that needs to be sorted.
11410 for (SDNode *P : N->uses()) {
11411 unsigned Degree = P->getNodeId();
11412 assert(Degree != 0 && "Invalid node degree");
11413 --Degree;
11414 if (Degree == 0) {
11415 // All of P's operands are sorted, so P may sorted now.
11416 P->setNodeId(DAGSize++);
11417 if (P->getIterator() != SortedPos)
11418 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
11419 assert(SortedPos != AllNodes.end() && "Overran node list");
11420 ++SortedPos;
11421 } else {
11422 // Update P's outstanding operand count.
11423 P->setNodeId(Degree);
11424 }
11425 }
11426 if (Node.getIterator() == SortedPos) {
11427 #ifndef NDEBUG
11428 allnodes_iterator I(N);
11429 SDNode *S = &*++I;
11430 dbgs() << "Overran sorted position:\n";
11431 S->dumprFull(this); dbgs() << "\n";
11432 dbgs() << "Checking if this is due to cycles\n";
11433 checkForCycles(this, true);
11434 #endif
11435 llvm_unreachable(nullptr);
11436 }
11437 }
11438
11439 assert(SortedPos == AllNodes.end() &&
11440 "Topological sort incomplete!");
11441 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
11442 "First node in topological sort is not the entry token!");
11443 assert(AllNodes.front().getNodeId() == 0 &&
11444 "First node in topological sort has non-zero id!");
11445 assert(AllNodes.front().getNumOperands() == 0 &&
11446 "First node in topological sort has operands!");
11447 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
11448 "Last node in topologic sort has unexpected id!");
11449 assert(AllNodes.back().use_empty() &&
11450 "Last node in topologic sort has users!");
11451 assert(DAGSize == allnodes_size() && "Node count mismatch!");
11452 return DAGSize;
11453 }
11454
11455 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
11456 /// value is produced by SD.
AddDbgValue(SDDbgValue * DB,bool isParameter)11457 void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
11458 for (SDNode *SD : DB->getSDNodes()) {
11459 if (!SD)
11460 continue;
11461 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
11462 SD->setHasDebugValue(true);
11463 }
11464 DbgInfo->add(DB, isParameter);
11465 }
11466
AddDbgLabel(SDDbgLabel * DB)11467 void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
11468
makeEquivalentMemoryOrdering(SDValue OldChain,SDValue NewMemOpChain)11469 SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
11470 SDValue NewMemOpChain) {
11471 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
11472 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
11473 // The new memory operation must have the same position as the old load in
11474 // terms of memory dependency. Create a TokenFactor for the old load and new
11475 // memory operation and update uses of the old load's output chain to use that
11476 // TokenFactor.
11477 if (OldChain == NewMemOpChain || OldChain.use_empty())
11478 return NewMemOpChain;
11479
11480 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
11481 OldChain, NewMemOpChain);
11482 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
11483 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);
11484 return TokenFactor;
11485 }
11486
makeEquivalentMemoryOrdering(LoadSDNode * OldLoad,SDValue NewMemOp)11487 SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
11488 SDValue NewMemOp) {
11489 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
11490 SDValue OldChain = SDValue(OldLoad, 1);
11491 SDValue NewMemOpChain = NewMemOp.getValue(1);
11492 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
11493 }
11494
getSymbolFunctionGlobalAddress(SDValue Op,Function ** OutFunction)11495 SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
11496 Function **OutFunction) {
11497 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
11498
11499 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
11500 auto *Module = MF->getFunction().getParent();
11501 auto *Function = Module->getFunction(Symbol);
11502
11503 if (OutFunction != nullptr)
11504 *OutFunction = Function;
11505
11506 if (Function != nullptr) {
11507 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
11508 return getGlobalAddress(Function, SDLoc(Op), PtrTy);
11509 }
11510
11511 std::string ErrorStr;
11512 raw_string_ostream ErrorFormatter(ErrorStr);
11513 ErrorFormatter << "Undefined external symbol ";
11514 ErrorFormatter << '"' << Symbol << '"';
11515 report_fatal_error(Twine(ErrorFormatter.str()));
11516 }
11517
11518 //===----------------------------------------------------------------------===//
11519 // SDNode Class
11520 //===----------------------------------------------------------------------===//
11521
isNullConstant(SDValue V)11522 bool llvm::isNullConstant(SDValue V) {
11523 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11524 return Const != nullptr && Const->isZero();
11525 }
11526
isNullFPConstant(SDValue V)11527 bool llvm::isNullFPConstant(SDValue V) {
11528 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
11529 return Const != nullptr && Const->isZero() && !Const->isNegative();
11530 }
11531
isAllOnesConstant(SDValue V)11532 bool llvm::isAllOnesConstant(SDValue V) {
11533 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11534 return Const != nullptr && Const->isAllOnes();
11535 }
11536
isOneConstant(SDValue V)11537 bool llvm::isOneConstant(SDValue V) {
11538 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11539 return Const != nullptr && Const->isOne();
11540 }
11541
isMinSignedConstant(SDValue V)11542 bool llvm::isMinSignedConstant(SDValue V) {
11543 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11544 return Const != nullptr && Const->isMinSignedValue();
11545 }
11546
isNeutralConstant(unsigned Opcode,SDNodeFlags Flags,SDValue V,unsigned OperandNo)11547 bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
11548 unsigned OperandNo) {
11549 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
11550 // TODO: Target-specific opcodes could be added.
11551 if (auto *Const = isConstOrConstSplat(V)) {
11552 switch (Opcode) {
11553 case ISD::ADD:
11554 case ISD::OR:
11555 case ISD::XOR:
11556 case ISD::UMAX:
11557 return Const->isZero();
11558 case ISD::MUL:
11559 return Const->isOne();
11560 case ISD::AND:
11561 case ISD::UMIN:
11562 return Const->isAllOnes();
11563 case ISD::SMAX:
11564 return Const->isMinSignedValue();
11565 case ISD::SMIN:
11566 return Const->isMaxSignedValue();
11567 case ISD::SUB:
11568 case ISD::SHL:
11569 case ISD::SRA:
11570 case ISD::SRL:
11571 return OperandNo == 1 && Const->isZero();
11572 case ISD::UDIV:
11573 case ISD::SDIV:
11574 return OperandNo == 1 && Const->isOne();
11575 }
11576 } else if (auto *ConstFP = isConstOrConstSplatFP(V)) {
11577 switch (Opcode) {
11578 case ISD::FADD:
11579 return ConstFP->isZero() &&
11580 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
11581 case ISD::FSUB:
11582 return OperandNo == 1 && ConstFP->isZero() &&
11583 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
11584 case ISD::FMUL:
11585 return ConstFP->isExactlyValue(1.0);
11586 case ISD::FDIV:
11587 return OperandNo == 1 && ConstFP->isExactlyValue(1.0);
11588 case ISD::FMINNUM:
11589 case ISD::FMAXNUM: {
11590 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
11591 EVT VT = V.getValueType();
11592 const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT);
11593 APFloat NeutralAF = !Flags.hasNoNaNs()
11594 ? APFloat::getQNaN(Semantics)
11595 : !Flags.hasNoInfs()
11596 ? APFloat::getInf(Semantics)
11597 : APFloat::getLargest(Semantics);
11598 if (Opcode == ISD::FMAXNUM)
11599 NeutralAF.changeSign();
11600
11601 return ConstFP->isExactlyValue(NeutralAF);
11602 }
11603 }
11604 }
11605 return false;
11606 }
11607
peekThroughBitcasts(SDValue V)11608 SDValue llvm::peekThroughBitcasts(SDValue V) {
11609 while (V.getOpcode() == ISD::BITCAST)
11610 V = V.getOperand(0);
11611 return V;
11612 }
11613
peekThroughOneUseBitcasts(SDValue V)11614 SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
11615 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
11616 V = V.getOperand(0);
11617 return V;
11618 }
11619
peekThroughExtractSubvectors(SDValue V)11620 SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
11621 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
11622 V = V.getOperand(0);
11623 return V;
11624 }
11625
peekThroughTruncates(SDValue V)11626 SDValue llvm::peekThroughTruncates(SDValue V) {
11627 while (V.getOpcode() == ISD::TRUNCATE)
11628 V = V.getOperand(0);
11629 return V;
11630 }
11631
isBitwiseNot(SDValue V,bool AllowUndefs)11632 bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
11633 if (V.getOpcode() != ISD::XOR)
11634 return false;
11635 V = peekThroughBitcasts(V.getOperand(1));
11636 unsigned NumBits = V.getScalarValueSizeInBits();
11637 ConstantSDNode *C =
11638 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
11639 return C && (C->getAPIntValue().countr_one() >= NumBits);
11640 }
11641
isConstOrConstSplat(SDValue N,bool AllowUndefs,bool AllowTruncation)11642 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
11643 bool AllowTruncation) {
11644 EVT VT = N.getValueType();
11645 APInt DemandedElts = VT.isFixedLengthVector()
11646 ? APInt::getAllOnes(VT.getVectorMinNumElements())
11647 : APInt(1, 1);
11648 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
11649 }
11650
isConstOrConstSplat(SDValue N,const APInt & DemandedElts,bool AllowUndefs,bool AllowTruncation)11651 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
11652 bool AllowUndefs,
11653 bool AllowTruncation) {
11654 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
11655 return CN;
11656
11657 // SplatVectors can truncate their operands. Ignore that case here unless
11658 // AllowTruncation is set.
11659 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
11660 EVT VecEltVT = N->getValueType(0).getVectorElementType();
11661 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
11662 EVT CVT = CN->getValueType(0);
11663 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
11664 if (AllowTruncation || CVT == VecEltVT)
11665 return CN;
11666 }
11667 }
11668
11669 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
11670 BitVector UndefElements;
11671 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
11672
11673 // BuildVectors can truncate their operands. Ignore that case here unless
11674 // AllowTruncation is set.
11675 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11676 if (CN && (UndefElements.none() || AllowUndefs)) {
11677 EVT CVT = CN->getValueType(0);
11678 EVT NSVT = N.getValueType().getScalarType();
11679 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
11680 if (AllowTruncation || (CVT == NSVT))
11681 return CN;
11682 }
11683 }
11684
11685 return nullptr;
11686 }
11687
isConstOrConstSplatFP(SDValue N,bool AllowUndefs)11688 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
11689 EVT VT = N.getValueType();
11690 APInt DemandedElts = VT.isFixedLengthVector()
11691 ? APInt::getAllOnes(VT.getVectorMinNumElements())
11692 : APInt(1, 1);
11693 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
11694 }
11695
isConstOrConstSplatFP(SDValue N,const APInt & DemandedElts,bool AllowUndefs)11696 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
11697 const APInt &DemandedElts,
11698 bool AllowUndefs) {
11699 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
11700 return CN;
11701
11702 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
11703 BitVector UndefElements;
11704 ConstantFPSDNode *CN =
11705 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
11706 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11707 if (CN && (UndefElements.none() || AllowUndefs))
11708 return CN;
11709 }
11710
11711 if (N.getOpcode() == ISD::SPLAT_VECTOR)
11712 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
11713 return CN;
11714
11715 return nullptr;
11716 }
11717
isNullOrNullSplat(SDValue N,bool AllowUndefs)11718 bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
11719 // TODO: may want to use peekThroughBitcast() here.
11720 ConstantSDNode *C =
11721 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
11722 return C && C->isZero();
11723 }
11724
isOneOrOneSplat(SDValue N,bool AllowUndefs)11725 bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
11726 ConstantSDNode *C =
11727 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
11728 return C && C->isOne();
11729 }
11730
isAllOnesOrAllOnesSplat(SDValue N,bool AllowUndefs)11731 bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
11732 N = peekThroughBitcasts(N);
11733 unsigned BitWidth = N.getScalarValueSizeInBits();
11734 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
11735 return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth;
11736 }
11737
~HandleSDNode()11738 HandleSDNode::~HandleSDNode() {
11739 DropOperands();
11740 }
11741
GlobalAddressSDNode(unsigned Opc,unsigned Order,const DebugLoc & DL,const GlobalValue * GA,EVT VT,int64_t o,unsigned TF)11742 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
11743 const DebugLoc &DL,
11744 const GlobalValue *GA, EVT VT,
11745 int64_t o, unsigned TF)
11746 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
11747 TheGlobal = GA;
11748 }
11749
AddrSpaceCastSDNode(unsigned Order,const DebugLoc & dl,EVT VT,unsigned SrcAS,unsigned DestAS)11750 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
11751 EVT VT, unsigned SrcAS,
11752 unsigned DestAS)
11753 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
11754 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
11755
MemSDNode(unsigned Opc,unsigned Order,const DebugLoc & dl,SDVTList VTs,EVT memvt,MachineMemOperand * mmo)11756 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
11757 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
11758 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
11759 MemSDNodeBits.IsVolatile = MMO->isVolatile();
11760 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
11761 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
11762 MemSDNodeBits.IsInvariant = MMO->isInvariant();
11763
11764 // We check here that the size of the memory operand fits within the size of
11765 // the MMO. This is because the MMO might indicate only a possible address
11766 // range instead of specifying the affected memory addresses precisely.
11767 // TODO: Make MachineMemOperands aware of scalable vectors.
11768 assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() &&
11769 "Size mismatch!");
11770 }
11771
11772 /// Profile - Gather unique data for the node.
11773 ///
Profile(FoldingSetNodeID & ID) const11774 void SDNode::Profile(FoldingSetNodeID &ID) const {
11775 AddNodeIDNode(ID, this);
11776 }
11777
11778 namespace {
11779
11780 struct EVTArray {
11781 std::vector<EVT> VTs;
11782
EVTArray__anon21bbf9af1811::EVTArray11783 EVTArray() {
11784 VTs.reserve(MVT::VALUETYPE_SIZE);
11785 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
11786 VTs.push_back(MVT((MVT::SimpleValueType)i));
11787 }
11788 };
11789
11790 } // end anonymous namespace
11791
11792 /// getValueTypeList - Return a pointer to the specified value type.
11793 ///
getValueTypeList(EVT VT)11794 const EVT *SDNode::getValueTypeList(EVT VT) {
11795 static std::set<EVT, EVT::compareRawBits> EVTs;
11796 static EVTArray SimpleVTArray;
11797 static sys::SmartMutex<true> VTMutex;
11798
11799 if (VT.isExtended()) {
11800 sys::SmartScopedLock<true> Lock(VTMutex);
11801 return &(*EVTs.insert(VT).first);
11802 }
11803 assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!");
11804 return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy];
11805 }
11806
11807 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
11808 /// indicated value. This method ignores uses of other values defined by this
11809 /// operation.
hasNUsesOfValue(unsigned NUses,unsigned Value) const11810 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
11811 assert(Value < getNumValues() && "Bad value!");
11812
11813 // TODO: Only iterate over uses of a given value of the node
11814 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
11815 if (UI.getUse().getResNo() == Value) {
11816 if (NUses == 0)
11817 return false;
11818 --NUses;
11819 }
11820 }
11821
11822 // Found exactly the right number of uses?
11823 return NUses == 0;
11824 }
11825
11826 /// hasAnyUseOfValue - Return true if there are any use of the indicated
11827 /// value. This method ignores uses of other values defined by this operation.
hasAnyUseOfValue(unsigned Value) const11828 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
11829 assert(Value < getNumValues() && "Bad value!");
11830
11831 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
11832 if (UI.getUse().getResNo() == Value)
11833 return true;
11834
11835 return false;
11836 }
11837
11838 /// isOnlyUserOf - Return true if this node is the only use of N.
isOnlyUserOf(const SDNode * N) const11839 bool SDNode::isOnlyUserOf(const SDNode *N) const {
11840 bool Seen = false;
11841 for (const SDNode *User : N->uses()) {
11842 if (User == this)
11843 Seen = true;
11844 else
11845 return false;
11846 }
11847
11848 return Seen;
11849 }
11850
11851 /// Return true if the only users of N are contained in Nodes.
areOnlyUsersOf(ArrayRef<const SDNode * > Nodes,const SDNode * N)11852 bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
11853 bool Seen = false;
11854 for (const SDNode *User : N->uses()) {
11855 if (llvm::is_contained(Nodes, User))
11856 Seen = true;
11857 else
11858 return false;
11859 }
11860
11861 return Seen;
11862 }
11863
11864 /// isOperand - Return true if this node is an operand of N.
isOperandOf(const SDNode * N) const11865 bool SDValue::isOperandOf(const SDNode *N) const {
11866 return is_contained(N->op_values(), *this);
11867 }
11868
isOperandOf(const SDNode * N) const11869 bool SDNode::isOperandOf(const SDNode *N) const {
11870 return any_of(N->op_values(),
11871 [this](SDValue Op) { return this == Op.getNode(); });
11872 }
11873
11874 /// reachesChainWithoutSideEffects - Return true if this operand (which must
11875 /// be a chain) reaches the specified operand without crossing any
11876 /// side-effecting instructions on any chain path. In practice, this looks
11877 /// through token factors and non-volatile loads. In order to remain efficient,
11878 /// this only looks a couple of nodes in, it does not do an exhaustive search.
11879 ///
11880 /// Note that we only need to examine chains when we're searching for
11881 /// side-effects; SelectionDAG requires that all side-effects are represented
11882 /// by chains, even if another operand would force a specific ordering. This
11883 /// constraint is necessary to allow transformations like splitting loads.
reachesChainWithoutSideEffects(SDValue Dest,unsigned Depth) const11884 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
11885 unsigned Depth) const {
11886 if (*this == Dest) return true;
11887
11888 // Don't search too deeply, we just want to be able to see through
11889 // TokenFactor's etc.
11890 if (Depth == 0) return false;
11891
11892 // If this is a token factor, all inputs to the TF happen in parallel.
11893 if (getOpcode() == ISD::TokenFactor) {
11894 // First, try a shallow search.
11895 if (is_contained((*this)->ops(), Dest)) {
11896 // We found the chain we want as an operand of this TokenFactor.
11897 // Essentially, we reach the chain without side-effects if we could
11898 // serialize the TokenFactor into a simple chain of operations with
11899 // Dest as the last operation. This is automatically true if the
11900 // chain has one use: there are no other ordering constraints.
11901 // If the chain has more than one use, we give up: some other
11902 // use of Dest might force a side-effect between Dest and the current
11903 // node.
11904 if (Dest.hasOneUse())
11905 return true;
11906 }
11907 // Next, try a deep search: check whether every operand of the TokenFactor
11908 // reaches Dest.
11909 return llvm::all_of((*this)->ops(), [=](SDValue Op) {
11910 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
11911 });
11912 }
11913
11914 // Loads don't have side effects, look through them.
11915 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
11916 if (Ld->isUnordered())
11917 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
11918 }
11919 return false;
11920 }
11921
hasPredecessor(const SDNode * N) const11922 bool SDNode::hasPredecessor(const SDNode *N) const {
11923 SmallPtrSet<const SDNode *, 32> Visited;
11924 SmallVector<const SDNode *, 16> Worklist;
11925 Worklist.push_back(this);
11926 return hasPredecessorHelper(N, Visited, Worklist);
11927 }
11928
intersectFlagsWith(const SDNodeFlags Flags)11929 void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
11930 this->Flags.intersectWith(Flags);
11931 }
11932
11933 SDValue
matchBinOpReduction(SDNode * Extract,ISD::NodeType & BinOp,ArrayRef<ISD::NodeType> CandidateBinOps,bool AllowPartials)11934 SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
11935 ArrayRef<ISD::NodeType> CandidateBinOps,
11936 bool AllowPartials) {
11937 // The pattern must end in an extract from index 0.
11938 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
11939 !isNullConstant(Extract->getOperand(1)))
11940 return SDValue();
11941
11942 // Match against one of the candidate binary ops.
11943 SDValue Op = Extract->getOperand(0);
11944 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
11945 return Op.getOpcode() == unsigned(BinOp);
11946 }))
11947 return SDValue();
11948
11949 // Floating-point reductions may require relaxed constraints on the final step
11950 // of the reduction because they may reorder intermediate operations.
11951 unsigned CandidateBinOp = Op.getOpcode();
11952 if (Op.getValueType().isFloatingPoint()) {
11953 SDNodeFlags Flags = Op->getFlags();
11954 switch (CandidateBinOp) {
11955 case ISD::FADD:
11956 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
11957 return SDValue();
11958 break;
11959 default:
11960 llvm_unreachable("Unhandled FP opcode for binop reduction");
11961 }
11962 }
11963
11964 // Matching failed - attempt to see if we did enough stages that a partial
11965 // reduction from a subvector is possible.
11966 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
11967 if (!AllowPartials || !Op)
11968 return SDValue();
11969 EVT OpVT = Op.getValueType();
11970 EVT OpSVT = OpVT.getScalarType();
11971 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
11972 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))
11973 return SDValue();
11974 BinOp = (ISD::NodeType)CandidateBinOp;
11975 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op,
11976 getVectorIdxConstant(0, SDLoc(Op)));
11977 };
11978
11979 // At each stage, we're looking for something that looks like:
11980 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
11981 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
11982 // i32 undef, i32 undef, i32 undef, i32 undef>
11983 // %a = binop <8 x i32> %op, %s
11984 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
11985 // we expect something like:
11986 // <4,5,6,7,u,u,u,u>
11987 // <2,3,u,u,u,u,u,u>
11988 // <1,u,u,u,u,u,u,u>
11989 // While a partial reduction match would be:
11990 // <2,3,u,u,u,u,u,u>
11991 // <1,u,u,u,u,u,u,u>
11992 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
11993 SDValue PrevOp;
11994 for (unsigned i = 0; i < Stages; ++i) {
11995 unsigned MaskEnd = (1 << i);
11996
11997 if (Op.getOpcode() != CandidateBinOp)
11998 return PartialReduction(PrevOp, MaskEnd);
11999
12000 SDValue Op0 = Op.getOperand(0);
12001 SDValue Op1 = Op.getOperand(1);
12002
12003 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
12004 if (Shuffle) {
12005 Op = Op1;
12006 } else {
12007 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
12008 Op = Op0;
12009 }
12010
12011 // The first operand of the shuffle should be the same as the other operand
12012 // of the binop.
12013 if (!Shuffle || Shuffle->getOperand(0) != Op)
12014 return PartialReduction(PrevOp, MaskEnd);
12015
12016 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
12017 for (int Index = 0; Index < (int)MaskEnd; ++Index)
12018 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
12019 return PartialReduction(PrevOp, MaskEnd);
12020
12021 PrevOp = Op;
12022 }
12023
12024 // Handle subvector reductions, which tend to appear after the shuffle
12025 // reduction stages.
12026 while (Op.getOpcode() == CandidateBinOp) {
12027 unsigned NumElts = Op.getValueType().getVectorNumElements();
12028 SDValue Op0 = Op.getOperand(0);
12029 SDValue Op1 = Op.getOperand(1);
12030 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12031 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12032 Op0.getOperand(0) != Op1.getOperand(0))
12033 break;
12034 SDValue Src = Op0.getOperand(0);
12035 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12036 if (NumSrcElts != (2 * NumElts))
12037 break;
12038 if (!(Op0.getConstantOperandAPInt(1) == 0 &&
12039 Op1.getConstantOperandAPInt(1) == NumElts) &&
12040 !(Op1.getConstantOperandAPInt(1) == 0 &&
12041 Op0.getConstantOperandAPInt(1) == NumElts))
12042 break;
12043 Op = Src;
12044 }
12045
12046 BinOp = (ISD::NodeType)CandidateBinOp;
12047 return Op;
12048 }
12049
UnrollVectorOp(SDNode * N,unsigned ResNE)12050 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
12051 EVT VT = N->getValueType(0);
12052 EVT EltVT = VT.getVectorElementType();
12053 unsigned NE = VT.getVectorNumElements();
12054
12055 SDLoc dl(N);
12056
12057 // If ResNE is 0, fully unroll the vector op.
12058 if (ResNE == 0)
12059 ResNE = NE;
12060 else if (NE > ResNE)
12061 NE = ResNE;
12062
12063 if (N->getNumValues() == 2) {
12064 SmallVector<SDValue, 8> Scalars0, Scalars1;
12065 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12066 EVT VT1 = N->getValueType(1);
12067 EVT EltVT1 = VT1.getVectorElementType();
12068
12069 unsigned i;
12070 for (i = 0; i != NE; ++i) {
12071 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12072 SDValue Operand = N->getOperand(j);
12073 EVT OperandVT = Operand.getValueType();
12074
12075 // A vector operand; extract a single element.
12076 EVT OperandEltVT = OperandVT.getVectorElementType();
12077 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12078 Operand, getVectorIdxConstant(i, dl));
12079 }
12080
12081 SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands);
12082 Scalars0.push_back(EltOp);
12083 Scalars1.push_back(EltOp.getValue(1));
12084 }
12085
12086 SDValue Vec0 = getBuildVector(VT, dl, Scalars0);
12087 SDValue Vec1 = getBuildVector(VT1, dl, Scalars1);
12088 return getMergeValues({Vec0, Vec1}, dl);
12089 }
12090
12091 assert(N->getNumValues() == 1 &&
12092 "Can't unroll a vector with multiple results!");
12093
12094 SmallVector<SDValue, 8> Scalars;
12095 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12096
12097 unsigned i;
12098 for (i= 0; i != NE; ++i) {
12099 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12100 SDValue Operand = N->getOperand(j);
12101 EVT OperandVT = Operand.getValueType();
12102 if (OperandVT.isVector()) {
12103 // A vector operand; extract a single element.
12104 EVT OperandEltVT = OperandVT.getVectorElementType();
12105 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12106 Operand, getVectorIdxConstant(i, dl));
12107 } else {
12108 // A scalar operand; just use it as is.
12109 Operands[j] = Operand;
12110 }
12111 }
12112
12113 switch (N->getOpcode()) {
12114 default: {
12115 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
12116 N->getFlags()));
12117 break;
12118 }
12119 case ISD::VSELECT:
12120 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
12121 break;
12122 case ISD::SHL:
12123 case ISD::SRA:
12124 case ISD::SRL:
12125 case ISD::ROTL:
12126 case ISD::ROTR:
12127 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
12128 getShiftAmountOperand(Operands[0].getValueType(),
12129 Operands[1])));
12130 break;
12131 case ISD::SIGN_EXTEND_INREG: {
12132 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
12133 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
12134 Operands[0],
12135 getValueType(ExtVT)));
12136 }
12137 }
12138 }
12139
12140 for (; i < ResNE; ++i)
12141 Scalars.push_back(getUNDEF(EltVT));
12142
12143 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
12144 return getBuildVector(VecVT, dl, Scalars);
12145 }
12146
UnrollVectorOverflowOp(SDNode * N,unsigned ResNE)12147 std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
12148 SDNode *N, unsigned ResNE) {
12149 unsigned Opcode = N->getOpcode();
12150 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
12151 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
12152 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
12153 "Expected an overflow opcode");
12154
12155 EVT ResVT = N->getValueType(0);
12156 EVT OvVT = N->getValueType(1);
12157 EVT ResEltVT = ResVT.getVectorElementType();
12158 EVT OvEltVT = OvVT.getVectorElementType();
12159 SDLoc dl(N);
12160
12161 // If ResNE is 0, fully unroll the vector op.
12162 unsigned NE = ResVT.getVectorNumElements();
12163 if (ResNE == 0)
12164 ResNE = NE;
12165 else if (NE > ResNE)
12166 NE = ResNE;
12167
12168 SmallVector<SDValue, 8> LHSScalars;
12169 SmallVector<SDValue, 8> RHSScalars;
12170 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
12171 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
12172
12173 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
12174 SDVTList VTs = getVTList(ResEltVT, SVT);
12175 SmallVector<SDValue, 8> ResScalars;
12176 SmallVector<SDValue, 8> OvScalars;
12177 for (unsigned i = 0; i < NE; ++i) {
12178 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
12179 SDValue Ov =
12180 getSelect(dl, OvEltVT, Res.getValue(1),
12181 getBoolConstant(true, dl, OvEltVT, ResVT),
12182 getConstant(0, dl, OvEltVT));
12183
12184 ResScalars.push_back(Res);
12185 OvScalars.push_back(Ov);
12186 }
12187
12188 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
12189 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
12190
12191 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
12192 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
12193 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
12194 getBuildVector(NewOvVT, dl, OvScalars));
12195 }
12196
areNonVolatileConsecutiveLoads(LoadSDNode * LD,LoadSDNode * Base,unsigned Bytes,int Dist) const12197 bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
12198 LoadSDNode *Base,
12199 unsigned Bytes,
12200 int Dist) const {
12201 if (LD->isVolatile() || Base->isVolatile())
12202 return false;
12203 // TODO: probably too restrictive for atomics, revisit
12204 if (!LD->isSimple())
12205 return false;
12206 if (LD->isIndexed() || Base->isIndexed())
12207 return false;
12208 if (LD->getChain() != Base->getChain())
12209 return false;
12210 EVT VT = LD->getMemoryVT();
12211 if (VT.getSizeInBits() / 8 != Bytes)
12212 return false;
12213
12214 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
12215 auto LocDecomp = BaseIndexOffset::match(LD, *this);
12216
12217 int64_t Offset = 0;
12218 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
12219 return (Dist * (int64_t)Bytes == Offset);
12220 return false;
12221 }
12222
12223 /// InferPtrAlignment - Infer alignment of a load / store address. Return
12224 /// std::nullopt if it cannot be inferred.
InferPtrAlign(SDValue Ptr) const12225 MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
12226 // If this is a GlobalAddress + cst, return the alignment.
12227 const GlobalValue *GV = nullptr;
12228 int64_t GVOffset = 0;
12229 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
12230 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
12231 KnownBits Known(PtrWidth);
12232 llvm::computeKnownBits(GV, Known, getDataLayout());
12233 unsigned AlignBits = Known.countMinTrailingZeros();
12234 if (AlignBits)
12235 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
12236 }
12237
12238 // If this is a direct reference to a stack slot, use information about the
12239 // stack slot's alignment.
12240 int FrameIdx = INT_MIN;
12241 int64_t FrameOffset = 0;
12242 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
12243 FrameIdx = FI->getIndex();
12244 } else if (isBaseWithConstantOffset(Ptr) &&
12245 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
12246 // Handle FI+Cst
12247 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
12248 FrameOffset = Ptr.getConstantOperandVal(1);
12249 }
12250
12251 if (FrameIdx != INT_MIN) {
12252 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
12253 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
12254 }
12255
12256 return std::nullopt;
12257 }
12258
12259 /// Split the scalar node with EXTRACT_ELEMENT using the provided
12260 /// VTs and return the low/high part.
SplitScalar(const SDValue & N,const SDLoc & DL,const EVT & LoVT,const EVT & HiVT)12261 std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
12262 const SDLoc &DL,
12263 const EVT &LoVT,
12264 const EVT &HiVT) {
12265 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
12266 "Split node must be a scalar type");
12267 SDValue Lo =
12268 getNode(ISD::EXTRACT_ELEMENT, DL, LoVT, N, getIntPtrConstant(0, DL));
12269 SDValue Hi =
12270 getNode(ISD::EXTRACT_ELEMENT, DL, HiVT, N, getIntPtrConstant(1, DL));
12271 return std::make_pair(Lo, Hi);
12272 }
12273
12274 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
12275 /// which is split (or expanded) into two not necessarily identical pieces.
GetSplitDestVTs(const EVT & VT) const12276 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
12277 // Currently all types are split in half.
12278 EVT LoVT, HiVT;
12279 if (!VT.isVector())
12280 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
12281 else
12282 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
12283
12284 return std::make_pair(LoVT, HiVT);
12285 }
12286
12287 /// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
12288 /// type, dependent on an enveloping VT that has been split into two identical
12289 /// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
12290 std::pair<EVT, EVT>
GetDependentSplitDestVTs(const EVT & VT,const EVT & EnvVT,bool * HiIsEmpty) const12291 SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
12292 bool *HiIsEmpty) const {
12293 EVT EltTp = VT.getVectorElementType();
12294 // Examples:
12295 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
12296 // custom VL=9 with enveloping VL=8/8 yields 8/1
12297 // custom VL=10 with enveloping VL=8/8 yields 8/2
12298 // etc.
12299 ElementCount VTNumElts = VT.getVectorElementCount();
12300 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
12301 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
12302 "Mixing fixed width and scalable vectors when enveloping a type");
12303 EVT LoVT, HiVT;
12304 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
12305 LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12306 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
12307 *HiIsEmpty = false;
12308 } else {
12309 // Flag that hi type has zero storage size, but return split envelop type
12310 // (this would be easier if vector types with zero elements were allowed).
12311 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
12312 HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12313 *HiIsEmpty = true;
12314 }
12315 return std::make_pair(LoVT, HiVT);
12316 }
12317
12318 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
12319 /// low/high part.
12320 std::pair<SDValue, SDValue>
SplitVector(const SDValue & N,const SDLoc & DL,const EVT & LoVT,const EVT & HiVT)12321 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
12322 const EVT &HiVT) {
12323 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
12324 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
12325 "Splitting vector with an invalid mixture of fixed and scalable "
12326 "vector types");
12327 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
12328 N.getValueType().getVectorMinNumElements() &&
12329 "More vector elements requested than available!");
12330 SDValue Lo, Hi;
12331 Lo =
12332 getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL));
12333 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
12334 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
12335 // IDX with the runtime scaling factor of the result vector type. For
12336 // fixed-width result vectors, that runtime scaling factor is 1.
12337 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
12338 getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
12339 return std::make_pair(Lo, Hi);
12340 }
12341
SplitEVL(SDValue N,EVT VecVT,const SDLoc & DL)12342 std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
12343 const SDLoc &DL) {
12344 // Split the vector length parameter.
12345 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
12346 EVT VT = N.getValueType();
12347 assert(VecVT.getVectorElementCount().isKnownEven() &&
12348 "Expecting the mask to be an evenly-sized vector");
12349 unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
12350 SDValue HalfNumElts =
12351 VecVT.isFixedLengthVector()
12352 ? getConstant(HalfMinNumElts, DL, VT)
12353 : getVScale(DL, VT, APInt(VT.getScalarSizeInBits(), HalfMinNumElts));
12354 SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts);
12355 SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts);
12356 return std::make_pair(Lo, Hi);
12357 }
12358
12359 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
WidenVector(const SDValue & N,const SDLoc & DL)12360 SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
12361 EVT VT = N.getValueType();
12362 EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
12363 NextPowerOf2(VT.getVectorNumElements()));
12364 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
12365 getVectorIdxConstant(0, DL));
12366 }
12367
ExtractVectorElements(SDValue Op,SmallVectorImpl<SDValue> & Args,unsigned Start,unsigned Count,EVT EltVT)12368 void SelectionDAG::ExtractVectorElements(SDValue Op,
12369 SmallVectorImpl<SDValue> &Args,
12370 unsigned Start, unsigned Count,
12371 EVT EltVT) {
12372 EVT VT = Op.getValueType();
12373 if (Count == 0)
12374 Count = VT.getVectorNumElements();
12375 if (EltVT == EVT())
12376 EltVT = VT.getVectorElementType();
12377 SDLoc SL(Op);
12378 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
12379 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op,
12380 getVectorIdxConstant(i, SL)));
12381 }
12382 }
12383
12384 // getAddressSpace - Return the address space this GlobalAddress belongs to.
getAddressSpace() const12385 unsigned GlobalAddressSDNode::getAddressSpace() const {
12386 return getGlobal()->getType()->getAddressSpace();
12387 }
12388
getType() const12389 Type *ConstantPoolSDNode::getType() const {
12390 if (isMachineConstantPoolEntry())
12391 return Val.MachineCPVal->getType();
12392 return Val.ConstVal->getType();
12393 }
12394
isConstantSplat(APInt & SplatValue,APInt & SplatUndef,unsigned & SplatBitSize,bool & HasAnyUndefs,unsigned MinSplatBits,bool IsBigEndian) const12395 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
12396 unsigned &SplatBitSize,
12397 bool &HasAnyUndefs,
12398 unsigned MinSplatBits,
12399 bool IsBigEndian) const {
12400 EVT VT = getValueType(0);
12401 assert(VT.isVector() && "Expected a vector type");
12402 unsigned VecWidth = VT.getSizeInBits();
12403 if (MinSplatBits > VecWidth)
12404 return false;
12405
12406 // FIXME: The widths are based on this node's type, but build vectors can
12407 // truncate their operands.
12408 SplatValue = APInt(VecWidth, 0);
12409 SplatUndef = APInt(VecWidth, 0);
12410
12411 // Get the bits. Bits with undefined values (when the corresponding element
12412 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
12413 // in SplatValue. If any of the values are not constant, give up and return
12414 // false.
12415 unsigned int NumOps = getNumOperands();
12416 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
12417 unsigned EltWidth = VT.getScalarSizeInBits();
12418
12419 for (unsigned j = 0; j < NumOps; ++j) {
12420 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
12421 SDValue OpVal = getOperand(i);
12422 unsigned BitPos = j * EltWidth;
12423
12424 if (OpVal.isUndef())
12425 SplatUndef.setBits(BitPos, BitPos + EltWidth);
12426 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
12427 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
12428 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
12429 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
12430 else
12431 return false;
12432 }
12433
12434 // The build_vector is all constants or undefs. Find the smallest element
12435 // size that splats the vector.
12436 HasAnyUndefs = (SplatUndef != 0);
12437
12438 // FIXME: This does not work for vectors with elements less than 8 bits.
12439 while (VecWidth > 8) {
12440 // If we can't split in half, stop here.
12441 if (VecWidth & 1)
12442 break;
12443
12444 unsigned HalfSize = VecWidth / 2;
12445 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
12446 APInt LowValue = SplatValue.extractBits(HalfSize, 0);
12447 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
12448 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
12449
12450 // If the two halves do not match (ignoring undef bits), stop here.
12451 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
12452 MinSplatBits > HalfSize)
12453 break;
12454
12455 SplatValue = HighValue | LowValue;
12456 SplatUndef = HighUndef & LowUndef;
12457
12458 VecWidth = HalfSize;
12459 }
12460
12461 // FIXME: The loop above only tries to split in halves. But if the input
12462 // vector for example is <3 x i16> it wouldn't be able to detect a
12463 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
12464 // optimizations. I guess that back in the days when this helper was created
12465 // vectors normally was power-of-2 sized.
12466
12467 SplatBitSize = VecWidth;
12468 return true;
12469 }
12470
getSplatValue(const APInt & DemandedElts,BitVector * UndefElements) const12471 SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
12472 BitVector *UndefElements) const {
12473 unsigned NumOps = getNumOperands();
12474 if (UndefElements) {
12475 UndefElements->clear();
12476 UndefElements->resize(NumOps);
12477 }
12478 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12479 if (!DemandedElts)
12480 return SDValue();
12481 SDValue Splatted;
12482 for (unsigned i = 0; i != NumOps; ++i) {
12483 if (!DemandedElts[i])
12484 continue;
12485 SDValue Op = getOperand(i);
12486 if (Op.isUndef()) {
12487 if (UndefElements)
12488 (*UndefElements)[i] = true;
12489 } else if (!Splatted) {
12490 Splatted = Op;
12491 } else if (Splatted != Op) {
12492 return SDValue();
12493 }
12494 }
12495
12496 if (!Splatted) {
12497 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
12498 assert(getOperand(FirstDemandedIdx).isUndef() &&
12499 "Can only have a splat without a constant for all undefs.");
12500 return getOperand(FirstDemandedIdx);
12501 }
12502
12503 return Splatted;
12504 }
12505
getSplatValue(BitVector * UndefElements) const12506 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
12507 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
12508 return getSplatValue(DemandedElts, UndefElements);
12509 }
12510
getRepeatedSequence(const APInt & DemandedElts,SmallVectorImpl<SDValue> & Sequence,BitVector * UndefElements) const12511 bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
12512 SmallVectorImpl<SDValue> &Sequence,
12513 BitVector *UndefElements) const {
12514 unsigned NumOps = getNumOperands();
12515 Sequence.clear();
12516 if (UndefElements) {
12517 UndefElements->clear();
12518 UndefElements->resize(NumOps);
12519 }
12520 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12521 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
12522 return false;
12523
12524 // Set the undefs even if we don't find a sequence (like getSplatValue).
12525 if (UndefElements)
12526 for (unsigned I = 0; I != NumOps; ++I)
12527 if (DemandedElts[I] && getOperand(I).isUndef())
12528 (*UndefElements)[I] = true;
12529
12530 // Iteratively widen the sequence length looking for repetitions.
12531 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
12532 Sequence.append(SeqLen, SDValue());
12533 for (unsigned I = 0; I != NumOps; ++I) {
12534 if (!DemandedElts[I])
12535 continue;
12536 SDValue &SeqOp = Sequence[I % SeqLen];
12537 SDValue Op = getOperand(I);
12538 if (Op.isUndef()) {
12539 if (!SeqOp)
12540 SeqOp = Op;
12541 continue;
12542 }
12543 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
12544 Sequence.clear();
12545 break;
12546 }
12547 SeqOp = Op;
12548 }
12549 if (!Sequence.empty())
12550 return true;
12551 }
12552
12553 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
12554 return false;
12555 }
12556
getRepeatedSequence(SmallVectorImpl<SDValue> & Sequence,BitVector * UndefElements) const12557 bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
12558 BitVector *UndefElements) const {
12559 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
12560 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
12561 }
12562
12563 ConstantSDNode *
getConstantSplatNode(const APInt & DemandedElts,BitVector * UndefElements) const12564 BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
12565 BitVector *UndefElements) const {
12566 return dyn_cast_or_null<ConstantSDNode>(
12567 getSplatValue(DemandedElts, UndefElements));
12568 }
12569
12570 ConstantSDNode *
getConstantSplatNode(BitVector * UndefElements) const12571 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
12572 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
12573 }
12574
12575 ConstantFPSDNode *
getConstantFPSplatNode(const APInt & DemandedElts,BitVector * UndefElements) const12576 BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
12577 BitVector *UndefElements) const {
12578 return dyn_cast_or_null<ConstantFPSDNode>(
12579 getSplatValue(DemandedElts, UndefElements));
12580 }
12581
12582 ConstantFPSDNode *
getConstantFPSplatNode(BitVector * UndefElements) const12583 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
12584 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
12585 }
12586
12587 int32_t
getConstantFPSplatPow2ToLog2Int(BitVector * UndefElements,uint32_t BitWidth) const12588 BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
12589 uint32_t BitWidth) const {
12590 if (ConstantFPSDNode *CN =
12591 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
12592 bool IsExact;
12593 APSInt IntVal(BitWidth);
12594 const APFloat &APF = CN->getValueAPF();
12595 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
12596 APFloat::opOK ||
12597 !IsExact)
12598 return -1;
12599
12600 return IntVal.exactLogBase2();
12601 }
12602 return -1;
12603 }
12604
getConstantRawBits(bool IsLittleEndian,unsigned DstEltSizeInBits,SmallVectorImpl<APInt> & RawBitElements,BitVector & UndefElements) const12605 bool BuildVectorSDNode::getConstantRawBits(
12606 bool IsLittleEndian, unsigned DstEltSizeInBits,
12607 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
12608 // Early-out if this contains anything but Undef/Constant/ConstantFP.
12609 if (!isConstant())
12610 return false;
12611
12612 unsigned NumSrcOps = getNumOperands();
12613 unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits();
12614 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12615 "Invalid bitcast scale");
12616
12617 // Extract raw src bits.
12618 SmallVector<APInt> SrcBitElements(NumSrcOps,
12619 APInt::getZero(SrcEltSizeInBits));
12620 BitVector SrcUndeElements(NumSrcOps, false);
12621
12622 for (unsigned I = 0; I != NumSrcOps; ++I) {
12623 SDValue Op = getOperand(I);
12624 if (Op.isUndef()) {
12625 SrcUndeElements.set(I);
12626 continue;
12627 }
12628 auto *CInt = dyn_cast<ConstantSDNode>(Op);
12629 auto *CFP = dyn_cast<ConstantFPSDNode>(Op);
12630 assert((CInt || CFP) && "Unknown constant");
12631 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
12632 : CFP->getValueAPF().bitcastToAPInt();
12633 }
12634
12635 // Recast to dst width.
12636 recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
12637 SrcBitElements, UndefElements, SrcUndeElements);
12638 return true;
12639 }
12640
recastRawBits(bool IsLittleEndian,unsigned DstEltSizeInBits,SmallVectorImpl<APInt> & DstBitElements,ArrayRef<APInt> SrcBitElements,BitVector & DstUndefElements,const BitVector & SrcUndefElements)12641 void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
12642 unsigned DstEltSizeInBits,
12643 SmallVectorImpl<APInt> &DstBitElements,
12644 ArrayRef<APInt> SrcBitElements,
12645 BitVector &DstUndefElements,
12646 const BitVector &SrcUndefElements) {
12647 unsigned NumSrcOps = SrcBitElements.size();
12648 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
12649 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12650 "Invalid bitcast scale");
12651 assert(NumSrcOps == SrcUndefElements.size() &&
12652 "Vector size mismatch");
12653
12654 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
12655 DstUndefElements.clear();
12656 DstUndefElements.resize(NumDstOps, false);
12657 DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits));
12658
12659 // Concatenate src elements constant bits together into dst element.
12660 if (SrcEltSizeInBits <= DstEltSizeInBits) {
12661 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
12662 for (unsigned I = 0; I != NumDstOps; ++I) {
12663 DstUndefElements.set(I);
12664 APInt &DstBits = DstBitElements[I];
12665 for (unsigned J = 0; J != Scale; ++J) {
12666 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12667 if (SrcUndefElements[Idx])
12668 continue;
12669 DstUndefElements.reset(I);
12670 const APInt &SrcBits = SrcBitElements[Idx];
12671 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
12672 "Illegal constant bitwidths");
12673 DstBits.insertBits(SrcBits, J * SrcEltSizeInBits);
12674 }
12675 }
12676 return;
12677 }
12678
12679 // Split src element constant bits into dst elements.
12680 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
12681 for (unsigned I = 0; I != NumSrcOps; ++I) {
12682 if (SrcUndefElements[I]) {
12683 DstUndefElements.set(I * Scale, (I + 1) * Scale);
12684 continue;
12685 }
12686 const APInt &SrcBits = SrcBitElements[I];
12687 for (unsigned J = 0; J != Scale; ++J) {
12688 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12689 APInt &DstBits = DstBitElements[Idx];
12690 DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
12691 }
12692 }
12693 }
12694
isConstant() const12695 bool BuildVectorSDNode::isConstant() const {
12696 for (const SDValue &Op : op_values()) {
12697 unsigned Opc = Op.getOpcode();
12698 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
12699 return false;
12700 }
12701 return true;
12702 }
12703
12704 std::optional<std::pair<APInt, APInt>>
isConstantSequence() const12705 BuildVectorSDNode::isConstantSequence() const {
12706 unsigned NumOps = getNumOperands();
12707 if (NumOps < 2)
12708 return std::nullopt;
12709
12710 if (!isa<ConstantSDNode>(getOperand(0)) ||
12711 !isa<ConstantSDNode>(getOperand(1)))
12712 return std::nullopt;
12713
12714 unsigned EltSize = getValueType(0).getScalarSizeInBits();
12715 APInt Start = getConstantOperandAPInt(0).trunc(EltSize);
12716 APInt Stride = getConstantOperandAPInt(1).trunc(EltSize) - Start;
12717
12718 if (Stride.isZero())
12719 return std::nullopt;
12720
12721 for (unsigned i = 2; i < NumOps; ++i) {
12722 if (!isa<ConstantSDNode>(getOperand(i)))
12723 return std::nullopt;
12724
12725 APInt Val = getConstantOperandAPInt(i).trunc(EltSize);
12726 if (Val != (Start + (Stride * i)))
12727 return std::nullopt;
12728 }
12729
12730 return std::make_pair(Start, Stride);
12731 }
12732
isSplatMask(const int * Mask,EVT VT)12733 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
12734 // Find the first non-undef value in the shuffle mask.
12735 unsigned i, e;
12736 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
12737 /* search */;
12738
12739 // If all elements are undefined, this shuffle can be considered a splat
12740 // (although it should eventually get simplified away completely).
12741 if (i == e)
12742 return true;
12743
12744 // Make sure all remaining elements are either undef or the same as the first
12745 // non-undef value.
12746 for (int Idx = Mask[i]; i != e; ++i)
12747 if (Mask[i] >= 0 && Mask[i] != Idx)
12748 return false;
12749 return true;
12750 }
12751
12752 // Returns the SDNode if it is a constant integer BuildVector
12753 // or constant integer.
isConstantIntBuildVectorOrConstantInt(SDValue N) const12754 SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const {
12755 if (isa<ConstantSDNode>(N))
12756 return N.getNode();
12757 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
12758 return N.getNode();
12759 // Treat a GlobalAddress supporting constant offset folding as a
12760 // constant integer.
12761 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
12762 if (GA->getOpcode() == ISD::GlobalAddress &&
12763 TLI->isOffsetFoldingLegal(GA))
12764 return GA;
12765 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12766 isa<ConstantSDNode>(N.getOperand(0)))
12767 return N.getNode();
12768 return nullptr;
12769 }
12770
12771 // Returns the SDNode if it is a constant float BuildVector
12772 // or constant float.
isConstantFPBuildVectorOrConstantFP(SDValue N) const12773 SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
12774 if (isa<ConstantFPSDNode>(N))
12775 return N.getNode();
12776
12777 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
12778 return N.getNode();
12779
12780 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12781 isa<ConstantFPSDNode>(N.getOperand(0)))
12782 return N.getNode();
12783
12784 return nullptr;
12785 }
12786
createOperands(SDNode * Node,ArrayRef<SDValue> Vals)12787 void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
12788 assert(!Node->OperandList && "Node already has operands");
12789 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
12790 "too many operands to fit into SDNode");
12791 SDUse *Ops = OperandRecycler.allocate(
12792 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
12793
12794 bool IsDivergent = false;
12795 for (unsigned I = 0; I != Vals.size(); ++I) {
12796 Ops[I].setUser(Node);
12797 Ops[I].setInitial(Vals[I]);
12798 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
12799 IsDivergent |= Ops[I].getNode()->isDivergent();
12800 }
12801 Node->NumOperands = Vals.size();
12802 Node->OperandList = Ops;
12803 if (!TLI->isSDNodeAlwaysUniform(Node)) {
12804 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
12805 Node->SDNodeBits.IsDivergent = IsDivergent;
12806 }
12807 checkForCycles(Node);
12808 }
12809
getTokenFactor(const SDLoc & DL,SmallVectorImpl<SDValue> & Vals)12810 SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
12811 SmallVectorImpl<SDValue> &Vals) {
12812 size_t Limit = SDNode::getMaxNumOperands();
12813 while (Vals.size() > Limit) {
12814 unsigned SliceIdx = Vals.size() - Limit;
12815 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
12816 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
12817 Vals.erase(Vals.begin() + SliceIdx, Vals.end());
12818 Vals.emplace_back(NewTF);
12819 }
12820 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
12821 }
12822
getNeutralElement(unsigned Opcode,const SDLoc & DL,EVT VT,SDNodeFlags Flags)12823 SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
12824 EVT VT, SDNodeFlags Flags) {
12825 switch (Opcode) {
12826 default:
12827 return SDValue();
12828 case ISD::ADD:
12829 case ISD::OR:
12830 case ISD::XOR:
12831 case ISD::UMAX:
12832 return getConstant(0, DL, VT);
12833 case ISD::MUL:
12834 return getConstant(1, DL, VT);
12835 case ISD::AND:
12836 case ISD::UMIN:
12837 return getAllOnesConstant(DL, VT);
12838 case ISD::SMAX:
12839 return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT);
12840 case ISD::SMIN:
12841 return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT);
12842 case ISD::FADD:
12843 return getConstantFP(-0.0, DL, VT);
12844 case ISD::FMUL:
12845 return getConstantFP(1.0, DL, VT);
12846 case ISD::FMINNUM:
12847 case ISD::FMAXNUM: {
12848 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
12849 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12850 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
12851 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
12852 APFloat::getLargest(Semantics);
12853 if (Opcode == ISD::FMAXNUM)
12854 NeutralAF.changeSign();
12855
12856 return getConstantFP(NeutralAF, DL, VT);
12857 }
12858 case ISD::FMINIMUM:
12859 case ISD::FMAXIMUM: {
12860 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
12861 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12862 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics)
12863 : APFloat::getLargest(Semantics);
12864 if (Opcode == ISD::FMAXIMUM)
12865 NeutralAF.changeSign();
12866
12867 return getConstantFP(NeutralAF, DL, VT);
12868 }
12869
12870 }
12871 }
12872
12873 /// Helper used to make a call to a library function that has one argument of
12874 /// pointer type.
12875 ///
12876 /// Such functions include 'fegetmode', 'fesetenv' and some others, which are
12877 /// used to get or set floating-point state. They have one argument of pointer
12878 /// type, which points to the memory region containing bits of the
12879 /// floating-point state. The value returned by such function is ignored in the
12880 /// created call.
12881 ///
12882 /// \param LibFunc Reference to library function (value of RTLIB::Libcall).
12883 /// \param Ptr Pointer used to save/load state.
12884 /// \param InChain Ingoing token chain.
12885 /// \returns Outgoing chain token.
makeStateFunctionCall(unsigned LibFunc,SDValue Ptr,SDValue InChain,const SDLoc & DLoc)12886 SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
12887 SDValue InChain,
12888 const SDLoc &DLoc) {
12889 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
12890 TargetLowering::ArgListTy Args;
12891 TargetLowering::ArgListEntry Entry;
12892 Entry.Node = Ptr;
12893 Entry.Ty = Ptr.getValueType().getTypeForEVT(*getContext());
12894 Args.push_back(Entry);
12895 RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
12896 SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC),
12897 TLI->getPointerTy(getDataLayout()));
12898 TargetLowering::CallLoweringInfo CLI(*this);
12899 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
12900 TLI->getLibcallCallingConv(LC), Type::getVoidTy(*getContext()), Callee,
12901 std::move(Args));
12902 return TLI->LowerCallTo(CLI).second;
12903 }
12904
copyExtraInfo(SDNode * From,SDNode * To)12905 void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
12906 assert(From && To && "Invalid SDNode; empty source SDValue?");
12907 auto I = SDEI.find(From);
12908 if (I == SDEI.end())
12909 return;
12910
12911 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
12912 // the iterator, hence the need to make a copy to prevent a use-after-free.
12913 NodeExtraInfo NEI = I->second;
12914 if (LLVM_LIKELY(!NEI.PCSections)) {
12915 // No deep copy required for the types of extra info set.
12916 //
12917 // FIXME: Investigate if other types of extra info also need deep copy. This
12918 // depends on the types of nodes they can be attached to: if some extra info
12919 // is only ever attached to nodes where a replacement To node is always the
12920 // node where later use and propagation of the extra info has the intended
12921 // semantics, no deep copy is required.
12922 SDEI[To] = std::move(NEI);
12923 return;
12924 }
12925
12926 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
12927 // through the replacement of From with To. Otherwise, replacements of a node
12928 // (From) with more complex nodes (To and its operands) may result in lost
12929 // extra info where the root node (To) is insignificant in further propagating
12930 // and using extra info when further lowering to MIR.
12931 //
12932 // In the first step pre-populate the visited set with the nodes reachable
12933 // from the old From node. This avoids copying NodeExtraInfo to parts of the
12934 // DAG that is not new and should be left untouched.
12935 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
12936 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
12937 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
12938 if (MaxDepth == 0) {
12939 // Remember this node in case we need to increase MaxDepth and continue
12940 // populating FromReach from this node.
12941 Leafs.emplace_back(N);
12942 return;
12943 }
12944 if (!FromReach.insert(N).second)
12945 return;
12946 for (const SDValue &Op : N->op_values())
12947 Self(Self, Op.getNode(), MaxDepth - 1);
12948 };
12949
12950 // Copy extra info to To and all its transitive operands (that are new).
12951 SmallPtrSet<const SDNode *, 8> Visited;
12952 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
12953 if (FromReach.contains(N))
12954 return true;
12955 if (!Visited.insert(N).second)
12956 return true;
12957 if (getEntryNode().getNode() == N)
12958 return false;
12959 for (const SDValue &Op : N->op_values()) {
12960 if (!Self(Self, Op.getNode()))
12961 return false;
12962 }
12963 // Copy only if entry node was not reached.
12964 SDEI[N] = NEI;
12965 return true;
12966 };
12967
12968 // We first try with a lower MaxDepth, assuming that the path to common
12969 // operands between From and To is relatively short. This significantly
12970 // improves performance in the common case. The initial MaxDepth is big
12971 // enough to avoid retry in the common case; the last MaxDepth is large
12972 // enough to avoid having to use the fallback below (and protects from
12973 // potential stack exhaustion from recursion).
12974 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
12975 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
12976 // StartFrom is the previous (or initial) set of leafs reachable at the
12977 // previous maximum depth.
12978 SmallVector<const SDNode *> StartFrom;
12979 std::swap(StartFrom, Leafs);
12980 for (const SDNode *N : StartFrom)
12981 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
12982 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
12983 return;
12984 // This should happen very rarely (reached the entry node).
12985 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
12986 assert(!Leafs.empty());
12987 }
12988
12989 // This should not happen - but if it did, that means the subgraph reachable
12990 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
12991 // could not visit all reachable common operands. Consequently, we were able
12992 // to reach the entry node.
12993 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
12994 assert(false && "From subgraph too complex - increase max. MaxDepth?");
12995 // Best-effort fallback if assertions disabled.
12996 SDEI[To] = std::move(NEI);
12997 }
12998
12999 #ifndef NDEBUG
checkForCyclesHelper(const SDNode * N,SmallPtrSetImpl<const SDNode * > & Visited,SmallPtrSetImpl<const SDNode * > & Checked,const llvm::SelectionDAG * DAG)13000 static void checkForCyclesHelper(const SDNode *N,
13001 SmallPtrSetImpl<const SDNode*> &Visited,
13002 SmallPtrSetImpl<const SDNode*> &Checked,
13003 const llvm::SelectionDAG *DAG) {
13004 // If this node has already been checked, don't check it again.
13005 if (Checked.count(N))
13006 return;
13007
13008 // If a node has already been visited on this depth-first walk, reject it as
13009 // a cycle.
13010 if (!Visited.insert(N).second) {
13011 errs() << "Detected cycle in SelectionDAG\n";
13012 dbgs() << "Offending node:\n";
13013 N->dumprFull(DAG); dbgs() << "\n";
13014 abort();
13015 }
13016
13017 for (const SDValue &Op : N->op_values())
13018 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
13019
13020 Checked.insert(N);
13021 Visited.erase(N);
13022 }
13023 #endif
13024
checkForCycles(const llvm::SDNode * N,const llvm::SelectionDAG * DAG,bool force)13025 void llvm::checkForCycles(const llvm::SDNode *N,
13026 const llvm::SelectionDAG *DAG,
13027 bool force) {
13028 #ifndef NDEBUG
13029 bool check = force;
13030 #ifdef EXPENSIVE_CHECKS
13031 check = true;
13032 #endif // EXPENSIVE_CHECKS
13033 if (check) {
13034 assert(N && "Checking nonexistent SDNode");
13035 SmallPtrSet<const SDNode*, 32> visited;
13036 SmallPtrSet<const SDNode*, 32> checked;
13037 checkForCyclesHelper(N, visited, checked, DAG);
13038 }
13039 #endif // !NDEBUG
13040 }
13041
checkForCycles(const llvm::SelectionDAG * DAG,bool force)13042 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
13043 checkForCycles(DAG->getRoot().getNode(), DAG, force);
13044 }
13045