1 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the SelectionDAG class.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/CodeGen/SelectionDAG.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/SetVector.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/Analysis/ValueTracking.h"
22 #include "llvm/CodeGen/MachineBasicBlock.h"
23 #include "llvm/CodeGen/MachineConstantPool.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineModuleInfo.h"
26 #include "llvm/IR/CallingConv.h"
27 #include "llvm/IR/Constants.h"
28 #include "llvm/IR/DataLayout.h"
29 #include "llvm/IR/DebugInfo.h"
30 #include "llvm/IR/DerivedTypes.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalAlias.h"
33 #include "llvm/IR/GlobalVariable.h"
34 #include "llvm/IR/Intrinsics.h"
35 #include "llvm/Support/CommandLine.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/ManagedStatic.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/Mutex.h"
41 #include "llvm/Support/raw_ostream.h"
42 #include "llvm/Target/TargetInstrInfo.h"
43 #include "llvm/Target/TargetIntrinsicInfo.h"
44 #include "llvm/Target/TargetLowering.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
47 #include "llvm/Target/TargetRegisterInfo.h"
48 #include "llvm/Target/TargetSelectionDAGInfo.h"
49 #include "llvm/Target/TargetSubtargetInfo.h"
50 #include <algorithm>
51 #include <cmath>
52
53 using namespace llvm;
54
55 /// makeVTList - Return an instance of the SDVTList struct initialized with the
56 /// specified members.
makeVTList(const EVT * VTs,unsigned NumVTs)57 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
58 SDVTList Res = {VTs, NumVTs};
59 return Res;
60 }
61
62 // Default null implementations of the callbacks.
NodeDeleted(SDNode *,SDNode *)63 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
NodeUpdated(SDNode *)64 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
65
66 //===----------------------------------------------------------------------===//
67 // ConstantFPSDNode Class
68 //===----------------------------------------------------------------------===//
69
70 /// isExactlyValue - We don't rely on operator== working on double values, as
71 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
72 /// As such, this method can be used to do an exact bit-for-bit comparison of
73 /// two floating point values.
isExactlyValue(const APFloat & V) const74 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
75 return getValueAPF().bitwiseIsEqual(V);
76 }
77
isValueValidForType(EVT VT,const APFloat & Val)78 bool ConstantFPSDNode::isValueValidForType(EVT VT,
79 const APFloat& Val) {
80 assert(VT.isFloatingPoint() && "Can only convert between FP types");
81
82 // convert modifies in place, so make a copy.
83 APFloat Val2 = APFloat(Val);
84 bool losesInfo;
85 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
86 APFloat::rmNearestTiesToEven,
87 &losesInfo);
88 return !losesInfo;
89 }
90
91 //===----------------------------------------------------------------------===//
92 // ISD Namespace
93 //===----------------------------------------------------------------------===//
94
95 /// isBuildVectorAllOnes - Return true if the specified node is a
96 /// BUILD_VECTOR where all of the elements are ~0 or undef.
isBuildVectorAllOnes(const SDNode * N)97 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
98 // Look through a bit convert.
99 while (N->getOpcode() == ISD::BITCAST)
100 N = N->getOperand(0).getNode();
101
102 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
103
104 unsigned i = 0, e = N->getNumOperands();
105
106 // Skip over all of the undef values.
107 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
108 ++i;
109
110 // Do not accept an all-undef vector.
111 if (i == e) return false;
112
113 // Do not accept build_vectors that aren't all constants or which have non-~0
114 // elements. We have to be a bit careful here, as the type of the constant
115 // may not be the same as the type of the vector elements due to type
116 // legalization (the elements are promoted to a legal type for the target and
117 // a vector of a type may be legal when the base element type is not).
118 // We only want to check enough bits to cover the vector elements, because
119 // we care if the resultant vector is all ones, not whether the individual
120 // constants are.
121 SDValue NotZero = N->getOperand(i);
122 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
123 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
124 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
125 return false;
126 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
127 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
128 return false;
129 } else
130 return false;
131
132 // Okay, we have at least one ~0 value, check to see if the rest match or are
133 // undefs. Even with the above element type twiddling, this should be OK, as
134 // the same type legalization should have applied to all the elements.
135 for (++i; i != e; ++i)
136 if (N->getOperand(i) != NotZero &&
137 N->getOperand(i).getOpcode() != ISD::UNDEF)
138 return false;
139 return true;
140 }
141
142
143 /// isBuildVectorAllZeros - Return true if the specified node is a
144 /// BUILD_VECTOR where all of the elements are 0 or undef.
isBuildVectorAllZeros(const SDNode * N)145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
146 // Look through a bit convert.
147 while (N->getOpcode() == ISD::BITCAST)
148 N = N->getOperand(0).getNode();
149
150 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
151
152 bool IsAllUndef = true;
153 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
154 if (N->getOperand(i).getOpcode() == ISD::UNDEF)
155 continue;
156 IsAllUndef = false;
157 // Do not accept build_vectors that aren't all constants or which have non-0
158 // elements. We have to be a bit careful here, as the type of the constant
159 // may not be the same as the type of the vector elements due to type
160 // legalization (the elements are promoted to a legal type for the target
161 // and a vector of a type may be legal when the base element type is not).
162 // We only want to check enough bits to cover the vector elements, because
163 // we care if the resultant vector is all zeros, not whether the individual
164 // constants are.
165 SDValue Zero = N->getOperand(i);
166 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
167 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
168 if (CN->getAPIntValue().countTrailingZeros() < EltSize)
169 return false;
170 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
171 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
172 return false;
173 } else
174 return false;
175 }
176
177 // Do not accept an all-undef vector.
178 if (IsAllUndef)
179 return false;
180 return true;
181 }
182
183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
184 /// all ConstantSDNode or undef.
isBuildVectorOfConstantSDNodes(const SDNode * N)185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
186 if (N->getOpcode() != ISD::BUILD_VECTOR)
187 return false;
188
189 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
190 SDValue Op = N->getOperand(i);
191 if (Op.getOpcode() == ISD::UNDEF)
192 continue;
193 if (!isa<ConstantSDNode>(Op))
194 return false;
195 }
196 return true;
197 }
198
199 /// isScalarToVector - Return true if the specified node is a
200 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
201 /// element is not an undef.
isScalarToVector(const SDNode * N)202 bool ISD::isScalarToVector(const SDNode *N) {
203 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
204 return true;
205
206 if (N->getOpcode() != ISD::BUILD_VECTOR)
207 return false;
208 if (N->getOperand(0).getOpcode() == ISD::UNDEF)
209 return false;
210 unsigned NumElems = N->getNumOperands();
211 if (NumElems == 1)
212 return false;
213 for (unsigned i = 1; i < NumElems; ++i) {
214 SDValue V = N->getOperand(i);
215 if (V.getOpcode() != ISD::UNDEF)
216 return false;
217 }
218 return true;
219 }
220
221 /// allOperandsUndef - Return true if the node has at least one operand
222 /// and all operands of the specified node are ISD::UNDEF.
allOperandsUndef(const SDNode * N)223 bool ISD::allOperandsUndef(const SDNode *N) {
224 // Return false if the node has no operands.
225 // This is "logically inconsistent" with the definition of "all" but
226 // is probably the desired behavior.
227 if (N->getNumOperands() == 0)
228 return false;
229
230 for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
231 if (N->getOperand(i).getOpcode() != ISD::UNDEF)
232 return false;
233
234 return true;
235 }
236
getExtForLoadExtType(bool IsFP,ISD::LoadExtType ExtType)237 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
238 switch (ExtType) {
239 case ISD::EXTLOAD:
240 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
241 case ISD::SEXTLOAD:
242 return ISD::SIGN_EXTEND;
243 case ISD::ZEXTLOAD:
244 return ISD::ZERO_EXTEND;
245 default:
246 break;
247 }
248
249 llvm_unreachable("Invalid LoadExtType");
250 }
251
252 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
253 /// when given the operation for (X op Y).
getSetCCSwappedOperands(ISD::CondCode Operation)254 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
255 // To perform this operation, we just need to swap the L and G bits of the
256 // operation.
257 unsigned OldL = (Operation >> 2) & 1;
258 unsigned OldG = (Operation >> 1) & 1;
259 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
260 (OldL << 1) | // New G bit
261 (OldG << 2)); // New L bit.
262 }
263
264 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
265 /// 'op' is a valid SetCC operation.
getSetCCInverse(ISD::CondCode Op,bool isInteger)266 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
267 unsigned Operation = Op;
268 if (isInteger)
269 Operation ^= 7; // Flip L, G, E bits, but not U.
270 else
271 Operation ^= 15; // Flip all of the condition bits.
272
273 if (Operation > ISD::SETTRUE2)
274 Operation &= ~8; // Don't let N and U bits get set.
275
276 return ISD::CondCode(Operation);
277 }
278
279
280 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
281 /// signed operation and 2 if the result is an unsigned comparison. Return zero
282 /// if the operation does not depend on the sign of the input (setne and seteq).
isSignedOp(ISD::CondCode Opcode)283 static int isSignedOp(ISD::CondCode Opcode) {
284 switch (Opcode) {
285 default: llvm_unreachable("Illegal integer setcc operation!");
286 case ISD::SETEQ:
287 case ISD::SETNE: return 0;
288 case ISD::SETLT:
289 case ISD::SETLE:
290 case ISD::SETGT:
291 case ISD::SETGE: return 1;
292 case ISD::SETULT:
293 case ISD::SETULE:
294 case ISD::SETUGT:
295 case ISD::SETUGE: return 2;
296 }
297 }
298
299 /// getSetCCOrOperation - Return the result of a logical OR between different
300 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function
301 /// returns SETCC_INVALID if it is not possible to represent the resultant
302 /// comparison.
getSetCCOrOperation(ISD::CondCode Op1,ISD::CondCode Op2,bool isInteger)303 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
304 bool isInteger) {
305 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
306 // Cannot fold a signed integer setcc with an unsigned integer setcc.
307 return ISD::SETCC_INVALID;
308
309 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
310
311 // If the N and U bits get set then the resultant comparison DOES suddenly
312 // care about orderedness, and is true when ordered.
313 if (Op > ISD::SETTRUE2)
314 Op &= ~16; // Clear the U bit if the N bit is set.
315
316 // Canonicalize illegal integer setcc's.
317 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
318 Op = ISD::SETNE;
319
320 return ISD::CondCode(Op);
321 }
322
323 /// getSetCCAndOperation - Return the result of a logical AND between different
324 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This
325 /// function returns zero if it is not possible to represent the resultant
326 /// comparison.
getSetCCAndOperation(ISD::CondCode Op1,ISD::CondCode Op2,bool isInteger)327 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
328 bool isInteger) {
329 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
330 // Cannot fold a signed setcc with an unsigned setcc.
331 return ISD::SETCC_INVALID;
332
333 // Combine all of the condition bits.
334 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
335
336 // Canonicalize illegal integer setcc's.
337 if (isInteger) {
338 switch (Result) {
339 default: break;
340 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
341 case ISD::SETOEQ: // SETEQ & SETU[LG]E
342 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
343 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
344 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
345 }
346 }
347
348 return Result;
349 }
350
351 //===----------------------------------------------------------------------===//
352 // SDNode Profile Support
353 //===----------------------------------------------------------------------===//
354
355 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
356 ///
AddNodeIDOpcode(FoldingSetNodeID & ID,unsigned OpC)357 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
358 ID.AddInteger(OpC);
359 }
360
361 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
362 /// solely with their pointer.
AddNodeIDValueTypes(FoldingSetNodeID & ID,SDVTList VTList)363 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
364 ID.AddPointer(VTList.VTs);
365 }
366
367 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
368 ///
AddNodeIDOperands(FoldingSetNodeID & ID,ArrayRef<SDValue> Ops)369 static void AddNodeIDOperands(FoldingSetNodeID &ID,
370 ArrayRef<SDValue> Ops) {
371 for (auto& Op : Ops) {
372 ID.AddPointer(Op.getNode());
373 ID.AddInteger(Op.getResNo());
374 }
375 }
376
377 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
378 ///
AddNodeIDOperands(FoldingSetNodeID & ID,ArrayRef<SDUse> Ops)379 static void AddNodeIDOperands(FoldingSetNodeID &ID,
380 ArrayRef<SDUse> Ops) {
381 for (auto& Op : Ops) {
382 ID.AddPointer(Op.getNode());
383 ID.AddInteger(Op.getResNo());
384 }
385 }
386
AddBinaryNodeIDCustom(FoldingSetNodeID & ID,bool nuw,bool nsw,bool exact)387 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
388 bool exact) {
389 ID.AddBoolean(nuw);
390 ID.AddBoolean(nsw);
391 ID.AddBoolean(exact);
392 }
393
394 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
AddBinaryNodeIDCustom(FoldingSetNodeID & ID,unsigned Opcode,bool nuw,bool nsw,bool exact)395 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
396 bool nuw, bool nsw, bool exact) {
397 if (isBinOpWithFlags(Opcode))
398 AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
399 }
400
AddNodeIDNode(FoldingSetNodeID & ID,unsigned short OpC,SDVTList VTList,ArrayRef<SDValue> OpList)401 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
402 SDVTList VTList, ArrayRef<SDValue> OpList) {
403 AddNodeIDOpcode(ID, OpC);
404 AddNodeIDValueTypes(ID, VTList);
405 AddNodeIDOperands(ID, OpList);
406 }
407
408 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
409 /// the NodeID data.
AddNodeIDCustom(FoldingSetNodeID & ID,const SDNode * N)410 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
411 switch (N->getOpcode()) {
412 case ISD::TargetExternalSymbol:
413 case ISD::ExternalSymbol:
414 llvm_unreachable("Should only be used on nodes with operands");
415 default: break; // Normal nodes don't need extra info.
416 case ISD::TargetConstant:
417 case ISD::Constant: {
418 const ConstantSDNode *C = cast<ConstantSDNode>(N);
419 ID.AddPointer(C->getConstantIntValue());
420 ID.AddBoolean(C->isOpaque());
421 break;
422 }
423 case ISD::TargetConstantFP:
424 case ISD::ConstantFP: {
425 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
426 break;
427 }
428 case ISD::TargetGlobalAddress:
429 case ISD::GlobalAddress:
430 case ISD::TargetGlobalTLSAddress:
431 case ISD::GlobalTLSAddress: {
432 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
433 ID.AddPointer(GA->getGlobal());
434 ID.AddInteger(GA->getOffset());
435 ID.AddInteger(GA->getTargetFlags());
436 ID.AddInteger(GA->getAddressSpace());
437 break;
438 }
439 case ISD::BasicBlock:
440 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
441 break;
442 case ISD::Register:
443 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
444 break;
445 case ISD::RegisterMask:
446 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
447 break;
448 case ISD::SRCVALUE:
449 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
450 break;
451 case ISD::FrameIndex:
452 case ISD::TargetFrameIndex:
453 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
454 break;
455 case ISD::JumpTable:
456 case ISD::TargetJumpTable:
457 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
458 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
459 break;
460 case ISD::ConstantPool:
461 case ISD::TargetConstantPool: {
462 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
463 ID.AddInteger(CP->getAlignment());
464 ID.AddInteger(CP->getOffset());
465 if (CP->isMachineConstantPoolEntry())
466 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
467 else
468 ID.AddPointer(CP->getConstVal());
469 ID.AddInteger(CP->getTargetFlags());
470 break;
471 }
472 case ISD::TargetIndex: {
473 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
474 ID.AddInteger(TI->getIndex());
475 ID.AddInteger(TI->getOffset());
476 ID.AddInteger(TI->getTargetFlags());
477 break;
478 }
479 case ISD::LOAD: {
480 const LoadSDNode *LD = cast<LoadSDNode>(N);
481 ID.AddInteger(LD->getMemoryVT().getRawBits());
482 ID.AddInteger(LD->getRawSubclassData());
483 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
484 break;
485 }
486 case ISD::STORE: {
487 const StoreSDNode *ST = cast<StoreSDNode>(N);
488 ID.AddInteger(ST->getMemoryVT().getRawBits());
489 ID.AddInteger(ST->getRawSubclassData());
490 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
491 break;
492 }
493 case ISD::SDIV:
494 case ISD::UDIV:
495 case ISD::SRA:
496 case ISD::SRL:
497 case ISD::MUL:
498 case ISD::ADD:
499 case ISD::SUB:
500 case ISD::SHL: {
501 const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
502 AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
503 BinNode->hasNoSignedWrap(), BinNode->isExact());
504 break;
505 }
506 case ISD::ATOMIC_CMP_SWAP:
507 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
508 case ISD::ATOMIC_SWAP:
509 case ISD::ATOMIC_LOAD_ADD:
510 case ISD::ATOMIC_LOAD_SUB:
511 case ISD::ATOMIC_LOAD_AND:
512 case ISD::ATOMIC_LOAD_OR:
513 case ISD::ATOMIC_LOAD_XOR:
514 case ISD::ATOMIC_LOAD_NAND:
515 case ISD::ATOMIC_LOAD_MIN:
516 case ISD::ATOMIC_LOAD_MAX:
517 case ISD::ATOMIC_LOAD_UMIN:
518 case ISD::ATOMIC_LOAD_UMAX:
519 case ISD::ATOMIC_LOAD:
520 case ISD::ATOMIC_STORE: {
521 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
522 ID.AddInteger(AT->getMemoryVT().getRawBits());
523 ID.AddInteger(AT->getRawSubclassData());
524 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
525 break;
526 }
527 case ISD::PREFETCH: {
528 const MemSDNode *PF = cast<MemSDNode>(N);
529 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
530 break;
531 }
532 case ISD::VECTOR_SHUFFLE: {
533 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
534 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
535 i != e; ++i)
536 ID.AddInteger(SVN->getMaskElt(i));
537 break;
538 }
539 case ISD::TargetBlockAddress:
540 case ISD::BlockAddress: {
541 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
542 ID.AddPointer(BA->getBlockAddress());
543 ID.AddInteger(BA->getOffset());
544 ID.AddInteger(BA->getTargetFlags());
545 break;
546 }
547 } // end switch (N->getOpcode())
548
549 // Target specific memory nodes could also have address spaces to check.
550 if (N->isTargetMemoryOpcode())
551 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
552 }
553
554 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
555 /// data.
AddNodeIDNode(FoldingSetNodeID & ID,const SDNode * N)556 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
557 AddNodeIDOpcode(ID, N->getOpcode());
558 // Add the return value info.
559 AddNodeIDValueTypes(ID, N->getVTList());
560 // Add the operand info.
561 AddNodeIDOperands(ID, N->ops());
562
563 // Handle SDNode leafs with special info.
564 AddNodeIDCustom(ID, N);
565 }
566
567 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
568 /// the CSE map that carries volatility, temporalness, indexing mode, and
569 /// extension/truncation information.
570 ///
571 static inline unsigned
encodeMemSDNodeFlags(int ConvType,ISD::MemIndexedMode AM,bool isVolatile,bool isNonTemporal,bool isInvariant)572 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
573 bool isNonTemporal, bool isInvariant) {
574 assert((ConvType & 3) == ConvType &&
575 "ConvType may not require more than 2 bits!");
576 assert((AM & 7) == AM &&
577 "AM may not require more than 3 bits!");
578 return ConvType |
579 (AM << 2) |
580 (isVolatile << 5) |
581 (isNonTemporal << 6) |
582 (isInvariant << 7);
583 }
584
585 //===----------------------------------------------------------------------===//
586 // SelectionDAG Class
587 //===----------------------------------------------------------------------===//
588
589 /// doNotCSE - Return true if CSE should not be performed for this node.
doNotCSE(SDNode * N)590 static bool doNotCSE(SDNode *N) {
591 if (N->getValueType(0) == MVT::Glue)
592 return true; // Never CSE anything that produces a flag.
593
594 switch (N->getOpcode()) {
595 default: break;
596 case ISD::HANDLENODE:
597 case ISD::EH_LABEL:
598 return true; // Never CSE these nodes.
599 }
600
601 // Check that remaining values produced are not flags.
602 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
603 if (N->getValueType(i) == MVT::Glue)
604 return true; // Never CSE anything that produces a flag.
605
606 return false;
607 }
608
609 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
610 /// SelectionDAG.
RemoveDeadNodes()611 void SelectionDAG::RemoveDeadNodes() {
612 // Create a dummy node (which is not added to allnodes), that adds a reference
613 // to the root node, preventing it from being deleted.
614 HandleSDNode Dummy(getRoot());
615
616 SmallVector<SDNode*, 128> DeadNodes;
617
618 // Add all obviously-dead nodes to the DeadNodes worklist.
619 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
620 if (I->use_empty())
621 DeadNodes.push_back(I);
622
623 RemoveDeadNodes(DeadNodes);
624
625 // If the root changed (e.g. it was a dead load, update the root).
626 setRoot(Dummy.getValue());
627 }
628
629 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
630 /// given list, and any nodes that become unreachable as a result.
RemoveDeadNodes(SmallVectorImpl<SDNode * > & DeadNodes)631 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
632
633 // Process the worklist, deleting the nodes and adding their uses to the
634 // worklist.
635 while (!DeadNodes.empty()) {
636 SDNode *N = DeadNodes.pop_back_val();
637
638 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
639 DUL->NodeDeleted(N, nullptr);
640
641 // Take the node out of the appropriate CSE map.
642 RemoveNodeFromCSEMaps(N);
643
644 // Next, brutally remove the operand list. This is safe to do, as there are
645 // no cycles in the graph.
646 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
647 SDUse &Use = *I++;
648 SDNode *Operand = Use.getNode();
649 Use.set(SDValue());
650
651 // Now that we removed this operand, see if there are no uses of it left.
652 if (Operand->use_empty())
653 DeadNodes.push_back(Operand);
654 }
655
656 DeallocateNode(N);
657 }
658 }
659
RemoveDeadNode(SDNode * N)660 void SelectionDAG::RemoveDeadNode(SDNode *N){
661 SmallVector<SDNode*, 16> DeadNodes(1, N);
662
663 // Create a dummy node that adds a reference to the root node, preventing
664 // it from being deleted. (This matters if the root is an operand of the
665 // dead node.)
666 HandleSDNode Dummy(getRoot());
667
668 RemoveDeadNodes(DeadNodes);
669 }
670
DeleteNode(SDNode * N)671 void SelectionDAG::DeleteNode(SDNode *N) {
672 // First take this out of the appropriate CSE map.
673 RemoveNodeFromCSEMaps(N);
674
675 // Finally, remove uses due to operands of this node, remove from the
676 // AllNodes list, and delete the node.
677 DeleteNodeNotInCSEMaps(N);
678 }
679
DeleteNodeNotInCSEMaps(SDNode * N)680 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
681 assert(N != AllNodes.begin() && "Cannot delete the entry node!");
682 assert(N->use_empty() && "Cannot delete a node that is not dead!");
683
684 // Drop all of the operands and decrement used node's use counts.
685 N->DropOperands();
686
687 DeallocateNode(N);
688 }
689
erase(const SDNode * Node)690 void SDDbgInfo::erase(const SDNode *Node) {
691 DbgValMapType::iterator I = DbgValMap.find(Node);
692 if (I == DbgValMap.end())
693 return;
694 for (auto &Val: I->second)
695 Val->setIsInvalidated();
696 DbgValMap.erase(I);
697 }
698
DeallocateNode(SDNode * N)699 void SelectionDAG::DeallocateNode(SDNode *N) {
700 if (N->OperandsNeedDelete)
701 delete[] N->OperandList;
702
703 // Set the opcode to DELETED_NODE to help catch bugs when node
704 // memory is reallocated.
705 N->NodeType = ISD::DELETED_NODE;
706
707 NodeAllocator.Deallocate(AllNodes.remove(N));
708
709 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
710 // them and forget about that node.
711 DbgInfo->erase(N);
712 }
713
714 #ifndef NDEBUG
715 /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
VerifySDNode(SDNode * N)716 static void VerifySDNode(SDNode *N) {
717 switch (N->getOpcode()) {
718 default:
719 break;
720 case ISD::BUILD_PAIR: {
721 EVT VT = N->getValueType(0);
722 assert(N->getNumValues() == 1 && "Too many results!");
723 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
724 "Wrong return type!");
725 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
726 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
727 "Mismatched operand types!");
728 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
729 "Wrong operand type!");
730 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
731 "Wrong return type size");
732 break;
733 }
734 case ISD::BUILD_VECTOR: {
735 assert(N->getNumValues() == 1 && "Too many results!");
736 assert(N->getValueType(0).isVector() && "Wrong return type!");
737 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
738 "Wrong number of operands!");
739 EVT EltVT = N->getValueType(0).getVectorElementType();
740 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
741 assert((I->getValueType() == EltVT ||
742 (EltVT.isInteger() && I->getValueType().isInteger() &&
743 EltVT.bitsLE(I->getValueType()))) &&
744 "Wrong operand type!");
745 assert(I->getValueType() == N->getOperand(0).getValueType() &&
746 "Operands must all have the same type");
747 }
748 break;
749 }
750 }
751 }
752 #endif // NDEBUG
753
754 /// \brief Insert a newly allocated node into the DAG.
755 ///
756 /// Handles insertion into the all nodes list and CSE map, as well as
757 /// verification and other common operations when a new node is allocated.
InsertNode(SDNode * N)758 void SelectionDAG::InsertNode(SDNode *N) {
759 AllNodes.push_back(N);
760 #ifndef NDEBUG
761 VerifySDNode(N);
762 #endif
763 }
764
765 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
766 /// correspond to it. This is useful when we're about to delete or repurpose
767 /// the node. We don't want future request for structurally identical nodes
768 /// to return N anymore.
RemoveNodeFromCSEMaps(SDNode * N)769 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
770 bool Erased = false;
771 switch (N->getOpcode()) {
772 case ISD::HANDLENODE: return false; // noop.
773 case ISD::CONDCODE:
774 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
775 "Cond code doesn't exist!");
776 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
777 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
778 break;
779 case ISD::ExternalSymbol:
780 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
781 break;
782 case ISD::TargetExternalSymbol: {
783 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
784 Erased = TargetExternalSymbols.erase(
785 std::pair<std::string,unsigned char>(ESN->getSymbol(),
786 ESN->getTargetFlags()));
787 break;
788 }
789 case ISD::VALUETYPE: {
790 EVT VT = cast<VTSDNode>(N)->getVT();
791 if (VT.isExtended()) {
792 Erased = ExtendedValueTypeNodes.erase(VT);
793 } else {
794 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
795 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
796 }
797 break;
798 }
799 default:
800 // Remove it from the CSE Map.
801 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
802 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
803 Erased = CSEMap.RemoveNode(N);
804 break;
805 }
806 #ifndef NDEBUG
807 // Verify that the node was actually in one of the CSE maps, unless it has a
808 // flag result (which cannot be CSE'd) or is one of the special cases that are
809 // not subject to CSE.
810 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
811 !N->isMachineOpcode() && !doNotCSE(N)) {
812 N->dump(this);
813 dbgs() << "\n";
814 llvm_unreachable("Node is not in map!");
815 }
816 #endif
817 return Erased;
818 }
819
820 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
821 /// maps and modified in place. Add it back to the CSE maps, unless an identical
822 /// node already exists, in which case transfer all its users to the existing
823 /// node. This transfer can potentially trigger recursive merging.
824 ///
825 void
AddModifiedNodeToCSEMaps(SDNode * N)826 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
827 // For node types that aren't CSE'd, just act as if no identical node
828 // already exists.
829 if (!doNotCSE(N)) {
830 SDNode *Existing = CSEMap.GetOrInsertNode(N);
831 if (Existing != N) {
832 // If there was already an existing matching node, use ReplaceAllUsesWith
833 // to replace the dead one with the existing one. This can cause
834 // recursive merging of other unrelated nodes down the line.
835 ReplaceAllUsesWith(N, Existing);
836
837 // N is now dead. Inform the listeners and delete it.
838 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
839 DUL->NodeDeleted(N, Existing);
840 DeleteNodeNotInCSEMaps(N);
841 return;
842 }
843 }
844
845 // If the node doesn't already exist, we updated it. Inform listeners.
846 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
847 DUL->NodeUpdated(N);
848 }
849
850 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
851 /// were replaced with those specified. If this node is never memoized,
852 /// return null, otherwise return a pointer to the slot it would take. If a
853 /// node already exists with these operands, the slot will be non-null.
FindModifiedNodeSlot(SDNode * N,SDValue Op,void * & InsertPos)854 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
855 void *&InsertPos) {
856 if (doNotCSE(N))
857 return nullptr;
858
859 SDValue Ops[] = { Op };
860 FoldingSetNodeID ID;
861 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
862 AddNodeIDCustom(ID, N);
863 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
864 return Node;
865 }
866
867 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
868 /// were replaced with those specified. If this node is never memoized,
869 /// return null, otherwise return a pointer to the slot it would take. If a
870 /// node already exists with these operands, the slot will be non-null.
FindModifiedNodeSlot(SDNode * N,SDValue Op1,SDValue Op2,void * & InsertPos)871 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
872 SDValue Op1, SDValue Op2,
873 void *&InsertPos) {
874 if (doNotCSE(N))
875 return nullptr;
876
877 SDValue Ops[] = { Op1, Op2 };
878 FoldingSetNodeID ID;
879 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
880 AddNodeIDCustom(ID, N);
881 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
882 return Node;
883 }
884
885
886 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
887 /// were replaced with those specified. If this node is never memoized,
888 /// return null, otherwise return a pointer to the slot it would take. If a
889 /// node already exists with these operands, the slot will be non-null.
FindModifiedNodeSlot(SDNode * N,ArrayRef<SDValue> Ops,void * & InsertPos)890 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
891 void *&InsertPos) {
892 if (doNotCSE(N))
893 return nullptr;
894
895 FoldingSetNodeID ID;
896 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
897 AddNodeIDCustom(ID, N);
898 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
899 return Node;
900 }
901
902 /// getEVTAlignment - Compute the default alignment value for the
903 /// given type.
904 ///
getEVTAlignment(EVT VT) const905 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
906 Type *Ty = VT == MVT::iPTR ?
907 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
908 VT.getTypeForEVT(*getContext());
909
910 return TLI->getDataLayout()->getABITypeAlignment(Ty);
911 }
912
913 // EntryNode could meaningfully have debug info if we can find it...
SelectionDAG(const TargetMachine & tm,CodeGenOpt::Level OL)914 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
915 : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
916 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
917 Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
918 UpdateListeners(nullptr) {
919 AllNodes.push_back(&EntryNode);
920 DbgInfo = new SDDbgInfo();
921 }
922
init(MachineFunction & mf)923 void SelectionDAG::init(MachineFunction &mf) {
924 MF = &mf;
925 TLI = getSubtarget().getTargetLowering();
926 TSI = getSubtarget().getSelectionDAGInfo();
927 Context = &mf.getFunction()->getContext();
928 }
929
~SelectionDAG()930 SelectionDAG::~SelectionDAG() {
931 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
932 allnodes_clear();
933 delete DbgInfo;
934 }
935
allnodes_clear()936 void SelectionDAG::allnodes_clear() {
937 assert(&*AllNodes.begin() == &EntryNode);
938 AllNodes.remove(AllNodes.begin());
939 while (!AllNodes.empty())
940 DeallocateNode(AllNodes.begin());
941 }
942
GetBinarySDNode(unsigned Opcode,SDLoc DL,SDVTList VTs,SDValue N1,SDValue N2,bool nuw,bool nsw,bool exact)943 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
944 SDVTList VTs, SDValue N1,
945 SDValue N2, bool nuw, bool nsw,
946 bool exact) {
947 if (isBinOpWithFlags(Opcode)) {
948 BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
949 Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
950 FN->setHasNoUnsignedWrap(nuw);
951 FN->setHasNoSignedWrap(nsw);
952 FN->setIsExact(exact);
953
954 return FN;
955 }
956
957 BinarySDNode *N = new (NodeAllocator)
958 BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
959 return N;
960 }
961
clear()962 void SelectionDAG::clear() {
963 allnodes_clear();
964 OperandAllocator.Reset();
965 CSEMap.clear();
966
967 ExtendedValueTypeNodes.clear();
968 ExternalSymbols.clear();
969 TargetExternalSymbols.clear();
970 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
971 static_cast<CondCodeSDNode*>(nullptr));
972 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
973 static_cast<SDNode*>(nullptr));
974
975 EntryNode.UseList = nullptr;
976 AllNodes.push_back(&EntryNode);
977 Root = getEntryNode();
978 DbgInfo->clear();
979 }
980
getAnyExtOrTrunc(SDValue Op,SDLoc DL,EVT VT)981 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
982 return VT.bitsGT(Op.getValueType()) ?
983 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
984 getNode(ISD::TRUNCATE, DL, VT, Op);
985 }
986
getSExtOrTrunc(SDValue Op,SDLoc DL,EVT VT)987 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
988 return VT.bitsGT(Op.getValueType()) ?
989 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
990 getNode(ISD::TRUNCATE, DL, VT, Op);
991 }
992
getZExtOrTrunc(SDValue Op,SDLoc DL,EVT VT)993 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
994 return VT.bitsGT(Op.getValueType()) ?
995 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
996 getNode(ISD::TRUNCATE, DL, VT, Op);
997 }
998
getBoolExtOrTrunc(SDValue Op,SDLoc SL,EVT VT,EVT OpVT)999 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
1000 EVT OpVT) {
1001 if (VT.bitsLE(Op.getValueType()))
1002 return getNode(ISD::TRUNCATE, SL, VT, Op);
1003
1004 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1005 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1006 }
1007
getZeroExtendInReg(SDValue Op,SDLoc DL,EVT VT)1008 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
1009 assert(!VT.isVector() &&
1010 "getZeroExtendInReg should use the vector element type instead of "
1011 "the vector type!");
1012 if (Op.getValueType() == VT) return Op;
1013 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
1014 APInt Imm = APInt::getLowBitsSet(BitWidth,
1015 VT.getSizeInBits());
1016 return getNode(ISD::AND, DL, Op.getValueType(), Op,
1017 getConstant(Imm, Op.getValueType()));
1018 }
1019
getAnyExtendVectorInReg(SDValue Op,SDLoc DL,EVT VT)1020 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
1021 assert(VT.isVector() && "This DAG node is restricted to vector types.");
1022 assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1023 "The sizes of the input and result must match in order to perform the "
1024 "extend in-register.");
1025 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1026 "The destination vector type must have fewer lanes than the input.");
1027 return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
1028 }
1029
getSignExtendVectorInReg(SDValue Op,SDLoc DL,EVT VT)1030 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
1031 assert(VT.isVector() && "This DAG node is restricted to vector types.");
1032 assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1033 "The sizes of the input and result must match in order to perform the "
1034 "extend in-register.");
1035 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1036 "The destination vector type must have fewer lanes than the input.");
1037 return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
1038 }
1039
getZeroExtendVectorInReg(SDValue Op,SDLoc DL,EVT VT)1040 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
1041 assert(VT.isVector() && "This DAG node is restricted to vector types.");
1042 assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1043 "The sizes of the input and result must match in order to perform the "
1044 "extend in-register.");
1045 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1046 "The destination vector type must have fewer lanes than the input.");
1047 return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
1048 }
1049
1050 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1051 ///
getNOT(SDLoc DL,SDValue Val,EVT VT)1052 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
1053 EVT EltVT = VT.getScalarType();
1054 SDValue NegOne =
1055 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
1056 return getNode(ISD::XOR, DL, VT, Val, NegOne);
1057 }
1058
getLogicalNOT(SDLoc DL,SDValue Val,EVT VT)1059 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
1060 EVT EltVT = VT.getScalarType();
1061 SDValue TrueValue;
1062 switch (TLI->getBooleanContents(VT)) {
1063 case TargetLowering::ZeroOrOneBooleanContent:
1064 case TargetLowering::UndefinedBooleanContent:
1065 TrueValue = getConstant(1, VT);
1066 break;
1067 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1068 TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
1069 VT);
1070 break;
1071 }
1072 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1073 }
1074
getConstant(uint64_t Val,EVT VT,bool isT,bool isO)1075 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
1076 EVT EltVT = VT.getScalarType();
1077 assert((EltVT.getSizeInBits() >= 64 ||
1078 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1079 "getConstant with a uint64_t value that doesn't fit in the type!");
1080 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
1081 }
1082
getConstant(const APInt & Val,EVT VT,bool isT,bool isO)1083 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
1084 {
1085 return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
1086 }
1087
getConstant(const ConstantInt & Val,EVT VT,bool isT,bool isO)1088 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
1089 bool isO) {
1090 assert(VT.isInteger() && "Cannot create FP integer constant!");
1091
1092 EVT EltVT = VT.getScalarType();
1093 const ConstantInt *Elt = &Val;
1094
1095 // In some cases the vector type is legal but the element type is illegal and
1096 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1097 // inserted value (the type does not need to match the vector element type).
1098 // Any extra bits introduced will be truncated away.
1099 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1100 TargetLowering::TypePromoteInteger) {
1101 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1102 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
1103 Elt = ConstantInt::get(*getContext(), NewVal);
1104 }
1105 // In other cases the element type is illegal and needs to be expanded, for
1106 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1107 // the value into n parts and use a vector type with n-times the elements.
1108 // Then bitcast to the type requested.
1109 // Legalizing constants too early makes the DAGCombiner's job harder so we
1110 // only legalize if the DAG tells us we must produce legal types.
1111 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1112 TLI->getTypeAction(*getContext(), EltVT) ==
1113 TargetLowering::TypeExpandInteger) {
1114 APInt NewVal = Elt->getValue();
1115 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1116 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1117 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1118 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1119
1120 // Check the temporary vector is the correct size. If this fails then
1121 // getTypeToTransformTo() probably returned a type whose size (in bits)
1122 // isn't a power-of-2 factor of the requested type size.
1123 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1124
1125 SmallVector<SDValue, 2> EltParts;
1126 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
1127 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1128 .trunc(ViaEltSizeInBits),
1129 ViaEltVT, isT, isO));
1130 }
1131
1132 // EltParts is currently in little endian order. If we actually want
1133 // big-endian order then reverse it now.
1134 if (TLI->isBigEndian())
1135 std::reverse(EltParts.begin(), EltParts.end());
1136
1137 // The elements must be reversed when the element order is different
1138 // to the endianness of the elements (because the BITCAST is itself a
1139 // vector shuffle in this situation). However, we do not need any code to
1140 // perform this reversal because getConstant() is producing a vector
1141 // splat.
1142 // This situation occurs in MIPS MSA.
1143
1144 SmallVector<SDValue, 8> Ops;
1145 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
1146 Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1147
1148 SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
1149 getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
1150 Ops));
1151 return Result;
1152 }
1153
1154 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1155 "APInt size does not match type size!");
1156 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1157 FoldingSetNodeID ID;
1158 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1159 ID.AddPointer(Elt);
1160 ID.AddBoolean(isO);
1161 void *IP = nullptr;
1162 SDNode *N = nullptr;
1163 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
1164 if (!VT.isVector())
1165 return SDValue(N, 0);
1166
1167 if (!N) {
1168 N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
1169 CSEMap.InsertNode(N, IP);
1170 InsertNode(N);
1171 }
1172
1173 SDValue Result(N, 0);
1174 if (VT.isVector()) {
1175 SmallVector<SDValue, 8> Ops;
1176 Ops.assign(VT.getVectorNumElements(), Result);
1177 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
1178 }
1179 return Result;
1180 }
1181
getIntPtrConstant(uint64_t Val,bool isTarget)1182 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
1183 return getConstant(Val, TLI->getPointerTy(), isTarget);
1184 }
1185
1186
getConstantFP(const APFloat & V,EVT VT,bool isTarget)1187 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
1188 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
1189 }
1190
getConstantFP(const ConstantFP & V,EVT VT,bool isTarget)1191 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
1192 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1193
1194 EVT EltVT = VT.getScalarType();
1195
1196 // Do the map lookup using the actual bit pattern for the floating point
1197 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1198 // we don't have issues with SNANs.
1199 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1200 FoldingSetNodeID ID;
1201 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1202 ID.AddPointer(&V);
1203 void *IP = nullptr;
1204 SDNode *N = nullptr;
1205 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
1206 if (!VT.isVector())
1207 return SDValue(N, 0);
1208
1209 if (!N) {
1210 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
1211 CSEMap.InsertNode(N, IP);
1212 InsertNode(N);
1213 }
1214
1215 SDValue Result(N, 0);
1216 if (VT.isVector()) {
1217 SmallVector<SDValue, 8> Ops;
1218 Ops.assign(VT.getVectorNumElements(), Result);
1219 // FIXME SDLoc info might be appropriate here
1220 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
1221 }
1222 return Result;
1223 }
1224
getConstantFP(double Val,EVT VT,bool isTarget)1225 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
1226 EVT EltVT = VT.getScalarType();
1227 if (EltVT==MVT::f32)
1228 return getConstantFP(APFloat((float)Val), VT, isTarget);
1229 else if (EltVT==MVT::f64)
1230 return getConstantFP(APFloat(Val), VT, isTarget);
1231 else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
1232 EltVT==MVT::f16) {
1233 bool ignored;
1234 APFloat apf = APFloat(Val);
1235 apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1236 &ignored);
1237 return getConstantFP(apf, VT, isTarget);
1238 } else
1239 llvm_unreachable("Unsupported type in getConstantFP");
1240 }
1241
getGlobalAddress(const GlobalValue * GV,SDLoc DL,EVT VT,int64_t Offset,bool isTargetGA,unsigned char TargetFlags)1242 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
1243 EVT VT, int64_t Offset,
1244 bool isTargetGA,
1245 unsigned char TargetFlags) {
1246 assert((TargetFlags == 0 || isTargetGA) &&
1247 "Cannot set target flags on target-independent globals");
1248
1249 // Truncate (with sign-extension) the offset value to the pointer size.
1250 unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
1251 if (BitWidth < 64)
1252 Offset = SignExtend64(Offset, BitWidth);
1253
1254 unsigned Opc;
1255 if (GV->isThreadLocal())
1256 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1257 else
1258 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1259
1260 FoldingSetNodeID ID;
1261 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1262 ID.AddPointer(GV);
1263 ID.AddInteger(Offset);
1264 ID.AddInteger(TargetFlags);
1265 ID.AddInteger(GV->getType()->getAddressSpace());
1266 void *IP = nullptr;
1267 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1268 return SDValue(E, 0);
1269
1270 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
1271 DL.getDebugLoc(), GV, VT,
1272 Offset, TargetFlags);
1273 CSEMap.InsertNode(N, IP);
1274 InsertNode(N);
1275 return SDValue(N, 0);
1276 }
1277
getFrameIndex(int FI,EVT VT,bool isTarget)1278 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1279 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1280 FoldingSetNodeID ID;
1281 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1282 ID.AddInteger(FI);
1283 void *IP = nullptr;
1284 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1285 return SDValue(E, 0);
1286
1287 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
1288 CSEMap.InsertNode(N, IP);
1289 InsertNode(N);
1290 return SDValue(N, 0);
1291 }
1292
getJumpTable(int JTI,EVT VT,bool isTarget,unsigned char TargetFlags)1293 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1294 unsigned char TargetFlags) {
1295 assert((TargetFlags == 0 || isTarget) &&
1296 "Cannot set target flags on target-independent jump tables");
1297 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1298 FoldingSetNodeID ID;
1299 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1300 ID.AddInteger(JTI);
1301 ID.AddInteger(TargetFlags);
1302 void *IP = nullptr;
1303 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1304 return SDValue(E, 0);
1305
1306 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
1307 TargetFlags);
1308 CSEMap.InsertNode(N, IP);
1309 InsertNode(N);
1310 return SDValue(N, 0);
1311 }
1312
getConstantPool(const Constant * C,EVT VT,unsigned Alignment,int Offset,bool isTarget,unsigned char TargetFlags)1313 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1314 unsigned Alignment, int Offset,
1315 bool isTarget,
1316 unsigned char TargetFlags) {
1317 assert((TargetFlags == 0 || isTarget) &&
1318 "Cannot set target flags on target-independent globals");
1319 if (Alignment == 0)
1320 Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
1321 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1322 FoldingSetNodeID ID;
1323 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1324 ID.AddInteger(Alignment);
1325 ID.AddInteger(Offset);
1326 ID.AddPointer(C);
1327 ID.AddInteger(TargetFlags);
1328 void *IP = nullptr;
1329 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1330 return SDValue(E, 0);
1331
1332 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1333 Alignment, TargetFlags);
1334 CSEMap.InsertNode(N, IP);
1335 InsertNode(N);
1336 return SDValue(N, 0);
1337 }
1338
1339
getConstantPool(MachineConstantPoolValue * C,EVT VT,unsigned Alignment,int Offset,bool isTarget,unsigned char TargetFlags)1340 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1341 unsigned Alignment, int Offset,
1342 bool isTarget,
1343 unsigned char TargetFlags) {
1344 assert((TargetFlags == 0 || isTarget) &&
1345 "Cannot set target flags on target-independent globals");
1346 if (Alignment == 0)
1347 Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
1348 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1349 FoldingSetNodeID ID;
1350 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1351 ID.AddInteger(Alignment);
1352 ID.AddInteger(Offset);
1353 C->addSelectionDAGCSEId(ID);
1354 ID.AddInteger(TargetFlags);
1355 void *IP = nullptr;
1356 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1357 return SDValue(E, 0);
1358
1359 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1360 Alignment, TargetFlags);
1361 CSEMap.InsertNode(N, IP);
1362 InsertNode(N);
1363 return SDValue(N, 0);
1364 }
1365
getTargetIndex(int Index,EVT VT,int64_t Offset,unsigned char TargetFlags)1366 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1367 unsigned char TargetFlags) {
1368 FoldingSetNodeID ID;
1369 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1370 ID.AddInteger(Index);
1371 ID.AddInteger(Offset);
1372 ID.AddInteger(TargetFlags);
1373 void *IP = nullptr;
1374 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1375 return SDValue(E, 0);
1376
1377 SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
1378 TargetFlags);
1379 CSEMap.InsertNode(N, IP);
1380 InsertNode(N);
1381 return SDValue(N, 0);
1382 }
1383
getBasicBlock(MachineBasicBlock * MBB)1384 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1385 FoldingSetNodeID ID;
1386 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1387 ID.AddPointer(MBB);
1388 void *IP = nullptr;
1389 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1390 return SDValue(E, 0);
1391
1392 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
1393 CSEMap.InsertNode(N, IP);
1394 InsertNode(N);
1395 return SDValue(N, 0);
1396 }
1397
getValueType(EVT VT)1398 SDValue SelectionDAG::getValueType(EVT VT) {
1399 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1400 ValueTypeNodes.size())
1401 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1402
1403 SDNode *&N = VT.isExtended() ?
1404 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1405
1406 if (N) return SDValue(N, 0);
1407 N = new (NodeAllocator) VTSDNode(VT);
1408 InsertNode(N);
1409 return SDValue(N, 0);
1410 }
1411
getExternalSymbol(const char * Sym,EVT VT)1412 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1413 SDNode *&N = ExternalSymbols[Sym];
1414 if (N) return SDValue(N, 0);
1415 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
1416 InsertNode(N);
1417 return SDValue(N, 0);
1418 }
1419
getTargetExternalSymbol(const char * Sym,EVT VT,unsigned char TargetFlags)1420 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1421 unsigned char TargetFlags) {
1422 SDNode *&N =
1423 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1424 TargetFlags)];
1425 if (N) return SDValue(N, 0);
1426 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
1427 InsertNode(N);
1428 return SDValue(N, 0);
1429 }
1430
getCondCode(ISD::CondCode Cond)1431 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1432 if ((unsigned)Cond >= CondCodeNodes.size())
1433 CondCodeNodes.resize(Cond+1);
1434
1435 if (!CondCodeNodes[Cond]) {
1436 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
1437 CondCodeNodes[Cond] = N;
1438 InsertNode(N);
1439 }
1440
1441 return SDValue(CondCodeNodes[Cond], 0);
1442 }
1443
1444 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
1445 // the shuffle mask M that point at N1 to point at N2, and indices that point
1446 // N2 to point at N1.
commuteShuffle(SDValue & N1,SDValue & N2,SmallVectorImpl<int> & M)1447 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
1448 std::swap(N1, N2);
1449 int NElts = M.size();
1450 for (int i = 0; i != NElts; ++i) {
1451 if (M[i] >= NElts)
1452 M[i] -= NElts;
1453 else if (M[i] >= 0)
1454 M[i] += NElts;
1455 }
1456 }
1457
getVectorShuffle(EVT VT,SDLoc dl,SDValue N1,SDValue N2,const int * Mask)1458 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
1459 SDValue N2, const int *Mask) {
1460 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1461 "Invalid VECTOR_SHUFFLE");
1462
1463 // Canonicalize shuffle undef, undef -> undef
1464 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
1465 return getUNDEF(VT);
1466
1467 // Validate that all indices in Mask are within the range of the elements
1468 // input to the shuffle.
1469 unsigned NElts = VT.getVectorNumElements();
1470 SmallVector<int, 8> MaskVec;
1471 for (unsigned i = 0; i != NElts; ++i) {
1472 assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
1473 MaskVec.push_back(Mask[i]);
1474 }
1475
1476 // Canonicalize shuffle v, v -> v, undef
1477 if (N1 == N2) {
1478 N2 = getUNDEF(VT);
1479 for (unsigned i = 0; i != NElts; ++i)
1480 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
1481 }
1482
1483 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1484 if (N1.getOpcode() == ISD::UNDEF)
1485 commuteShuffle(N1, N2, MaskVec);
1486
1487 // Canonicalize all index into lhs, -> shuffle lhs, undef
1488 // Canonicalize all index into rhs, -> shuffle rhs, undef
1489 bool AllLHS = true, AllRHS = true;
1490 bool N2Undef = N2.getOpcode() == ISD::UNDEF;
1491 for (unsigned i = 0; i != NElts; ++i) {
1492 if (MaskVec[i] >= (int)NElts) {
1493 if (N2Undef)
1494 MaskVec[i] = -1;
1495 else
1496 AllLHS = false;
1497 } else if (MaskVec[i] >= 0) {
1498 AllRHS = false;
1499 }
1500 }
1501 if (AllLHS && AllRHS)
1502 return getUNDEF(VT);
1503 if (AllLHS && !N2Undef)
1504 N2 = getUNDEF(VT);
1505 if (AllRHS) {
1506 N1 = getUNDEF(VT);
1507 commuteShuffle(N1, N2, MaskVec);
1508 }
1509 // Reset our undef status after accounting for the mask.
1510 N2Undef = N2.getOpcode() == ISD::UNDEF;
1511 // Re-check whether both sides ended up undef.
1512 if (N1.getOpcode() == ISD::UNDEF && N2Undef)
1513 return getUNDEF(VT);
1514
1515 // If Identity shuffle return that node.
1516 bool Identity = true;
1517 for (unsigned i = 0; i != NElts; ++i) {
1518 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
1519 }
1520 if (Identity && NElts)
1521 return N1;
1522
1523 // Shuffling a constant splat doesn't change the result.
1524 if (N2Undef) {
1525 SDValue V = N1;
1526
1527 // Look through any bitcasts. We check that these don't change the number
1528 // (and size) of elements and just changes their types.
1529 while (V.getOpcode() == ISD::BITCAST)
1530 V = V->getOperand(0);
1531
1532 // A splat should always show up as a build vector node.
1533 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1534 BitVector UndefElements;
1535 SDValue Splat = BV->getSplatValue(&UndefElements);
1536 // If this is a splat of an undef, shuffling it is also undef.
1537 if (Splat && Splat.getOpcode() == ISD::UNDEF)
1538 return getUNDEF(VT);
1539
1540 // We only have a splat which can skip shuffles if there is a splatted
1541 // value and no undef lanes rearranged by the shuffle.
1542 if (Splat && UndefElements.none()) {
1543 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1544 // number of elements match or the value splatted is a zero constant.
1545 if (V.getValueType().getVectorNumElements() ==
1546 VT.getVectorNumElements())
1547 return N1;
1548 if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1549 if (C->isNullValue())
1550 return N1;
1551 }
1552 }
1553 }
1554
1555 FoldingSetNodeID ID;
1556 SDValue Ops[2] = { N1, N2 };
1557 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1558 for (unsigned i = 0; i != NElts; ++i)
1559 ID.AddInteger(MaskVec[i]);
1560
1561 void* IP = nullptr;
1562 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1563 return SDValue(E, 0);
1564
1565 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1566 // SDNode doesn't have access to it. This memory will be "leaked" when
1567 // the node is deallocated, but recovered when the NodeAllocator is released.
1568 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1569 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
1570
1571 ShuffleVectorSDNode *N =
1572 new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
1573 dl.getDebugLoc(), N1, N2,
1574 MaskAlloc);
1575 CSEMap.InsertNode(N, IP);
1576 InsertNode(N);
1577 return SDValue(N, 0);
1578 }
1579
getCommutedVectorShuffle(const ShuffleVectorSDNode & SV)1580 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1581 MVT VT = SV.getSimpleValueType(0);
1582 unsigned NumElems = VT.getVectorNumElements();
1583 SmallVector<int, 8> MaskVec;
1584
1585 for (unsigned i = 0; i != NumElems; ++i) {
1586 int Idx = SV.getMaskElt(i);
1587 if (Idx >= 0) {
1588 if (Idx < (int)NumElems)
1589 Idx += NumElems;
1590 else
1591 Idx -= NumElems;
1592 }
1593 MaskVec.push_back(Idx);
1594 }
1595
1596 SDValue Op0 = SV.getOperand(0);
1597 SDValue Op1 = SV.getOperand(1);
1598 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
1599 }
1600
getConvertRndSat(EVT VT,SDLoc dl,SDValue Val,SDValue DTy,SDValue STy,SDValue Rnd,SDValue Sat,ISD::CvtCode Code)1601 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
1602 SDValue Val, SDValue DTy,
1603 SDValue STy, SDValue Rnd, SDValue Sat,
1604 ISD::CvtCode Code) {
1605 // If the src and dest types are the same and the conversion is between
1606 // integer types of the same sign or two floats, no conversion is necessary.
1607 if (DTy == STy &&
1608 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
1609 return Val;
1610
1611 FoldingSetNodeID ID;
1612 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
1613 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
1614 void* IP = nullptr;
1615 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1616 return SDValue(E, 0);
1617
1618 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
1619 dl.getDebugLoc(),
1620 Ops, Code);
1621 CSEMap.InsertNode(N, IP);
1622 InsertNode(N);
1623 return SDValue(N, 0);
1624 }
1625
getRegister(unsigned RegNo,EVT VT)1626 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1627 FoldingSetNodeID ID;
1628 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1629 ID.AddInteger(RegNo);
1630 void *IP = nullptr;
1631 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1632 return SDValue(E, 0);
1633
1634 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
1635 CSEMap.InsertNode(N, IP);
1636 InsertNode(N);
1637 return SDValue(N, 0);
1638 }
1639
getRegisterMask(const uint32_t * RegMask)1640 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1641 FoldingSetNodeID ID;
1642 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1643 ID.AddPointer(RegMask);
1644 void *IP = nullptr;
1645 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1646 return SDValue(E, 0);
1647
1648 SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
1649 CSEMap.InsertNode(N, IP);
1650 InsertNode(N);
1651 return SDValue(N, 0);
1652 }
1653
getEHLabel(SDLoc dl,SDValue Root,MCSymbol * Label)1654 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
1655 FoldingSetNodeID ID;
1656 SDValue Ops[] = { Root };
1657 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
1658 ID.AddPointer(Label);
1659 void *IP = nullptr;
1660 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1661 return SDValue(E, 0);
1662
1663 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
1664 dl.getDebugLoc(), Root, Label);
1665 CSEMap.InsertNode(N, IP);
1666 InsertNode(N);
1667 return SDValue(N, 0);
1668 }
1669
1670
getBlockAddress(const BlockAddress * BA,EVT VT,int64_t Offset,bool isTarget,unsigned char TargetFlags)1671 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1672 int64_t Offset,
1673 bool isTarget,
1674 unsigned char TargetFlags) {
1675 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1676
1677 FoldingSetNodeID ID;
1678 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1679 ID.AddPointer(BA);
1680 ID.AddInteger(Offset);
1681 ID.AddInteger(TargetFlags);
1682 void *IP = nullptr;
1683 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1684 return SDValue(E, 0);
1685
1686 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
1687 TargetFlags);
1688 CSEMap.InsertNode(N, IP);
1689 InsertNode(N);
1690 return SDValue(N, 0);
1691 }
1692
getSrcValue(const Value * V)1693 SDValue SelectionDAG::getSrcValue(const Value *V) {
1694 assert((!V || V->getType()->isPointerTy()) &&
1695 "SrcValue is not a pointer?");
1696
1697 FoldingSetNodeID ID;
1698 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
1699 ID.AddPointer(V);
1700
1701 void *IP = nullptr;
1702 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1703 return SDValue(E, 0);
1704
1705 SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
1706 CSEMap.InsertNode(N, IP);
1707 InsertNode(N);
1708 return SDValue(N, 0);
1709 }
1710
1711 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
getMDNode(const MDNode * MD)1712 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1713 FoldingSetNodeID ID;
1714 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
1715 ID.AddPointer(MD);
1716
1717 void *IP = nullptr;
1718 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1719 return SDValue(E, 0);
1720
1721 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
1722 CSEMap.InsertNode(N, IP);
1723 InsertNode(N);
1724 return SDValue(N, 0);
1725 }
1726
1727 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
getAddrSpaceCast(SDLoc dl,EVT VT,SDValue Ptr,unsigned SrcAS,unsigned DestAS)1728 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
1729 unsigned SrcAS, unsigned DestAS) {
1730 SDValue Ops[] = {Ptr};
1731 FoldingSetNodeID ID;
1732 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
1733 ID.AddInteger(SrcAS);
1734 ID.AddInteger(DestAS);
1735
1736 void *IP = nullptr;
1737 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1738 return SDValue(E, 0);
1739
1740 SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
1741 dl.getDebugLoc(),
1742 VT, Ptr, SrcAS, DestAS);
1743 CSEMap.InsertNode(N, IP);
1744 InsertNode(N);
1745 return SDValue(N, 0);
1746 }
1747
1748 /// getShiftAmountOperand - Return the specified value casted to
1749 /// the target's desired shift amount type.
getShiftAmountOperand(EVT LHSTy,SDValue Op)1750 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1751 EVT OpTy = Op.getValueType();
1752 EVT ShTy = TLI->getShiftAmountTy(LHSTy);
1753 if (OpTy == ShTy || OpTy.isVector()) return Op;
1754
1755 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
1756 return getNode(Opcode, SDLoc(Op), ShTy, Op);
1757 }
1758
1759 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
1760 /// specified value type.
CreateStackTemporary(EVT VT,unsigned minAlign)1761 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1762 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
1763 unsigned ByteSize = VT.getStoreSize();
1764 Type *Ty = VT.getTypeForEVT(*getContext());
1765 unsigned StackAlign =
1766 std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
1767
1768 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
1769 return getFrameIndex(FrameIdx, TLI->getPointerTy());
1770 }
1771
1772 /// CreateStackTemporary - Create a stack temporary suitable for holding
1773 /// either of the specified value types.
CreateStackTemporary(EVT VT1,EVT VT2)1774 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
1775 unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
1776 VT2.getStoreSizeInBits())/8;
1777 Type *Ty1 = VT1.getTypeForEVT(*getContext());
1778 Type *Ty2 = VT2.getTypeForEVT(*getContext());
1779 const DataLayout *TD = TLI->getDataLayout();
1780 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
1781 TD->getPrefTypeAlignment(Ty2));
1782
1783 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
1784 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
1785 return getFrameIndex(FrameIdx, TLI->getPointerTy());
1786 }
1787
FoldSetCC(EVT VT,SDValue N1,SDValue N2,ISD::CondCode Cond,SDLoc dl)1788 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
1789 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
1790 // These setcc operations always fold.
1791 switch (Cond) {
1792 default: break;
1793 case ISD::SETFALSE:
1794 case ISD::SETFALSE2: return getConstant(0, VT);
1795 case ISD::SETTRUE:
1796 case ISD::SETTRUE2: {
1797 TargetLowering::BooleanContent Cnt =
1798 TLI->getBooleanContents(N1->getValueType(0));
1799 return getConstant(
1800 Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
1801 }
1802
1803 case ISD::SETOEQ:
1804 case ISD::SETOGT:
1805 case ISD::SETOGE:
1806 case ISD::SETOLT:
1807 case ISD::SETOLE:
1808 case ISD::SETONE:
1809 case ISD::SETO:
1810 case ISD::SETUO:
1811 case ISD::SETUEQ:
1812 case ISD::SETUNE:
1813 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
1814 break;
1815 }
1816
1817 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
1818 const APInt &C2 = N2C->getAPIntValue();
1819 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
1820 const APInt &C1 = N1C->getAPIntValue();
1821
1822 switch (Cond) {
1823 default: llvm_unreachable("Unknown integer setcc!");
1824 case ISD::SETEQ: return getConstant(C1 == C2, VT);
1825 case ISD::SETNE: return getConstant(C1 != C2, VT);
1826 case ISD::SETULT: return getConstant(C1.ult(C2), VT);
1827 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
1828 case ISD::SETULE: return getConstant(C1.ule(C2), VT);
1829 case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
1830 case ISD::SETLT: return getConstant(C1.slt(C2), VT);
1831 case ISD::SETGT: return getConstant(C1.sgt(C2), VT);
1832 case ISD::SETLE: return getConstant(C1.sle(C2), VT);
1833 case ISD::SETGE: return getConstant(C1.sge(C2), VT);
1834 }
1835 }
1836 }
1837 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
1838 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
1839 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
1840 switch (Cond) {
1841 default: break;
1842 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
1843 return getUNDEF(VT);
1844 // fall through
1845 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
1846 case ISD::SETNE: if (R==APFloat::cmpUnordered)
1847 return getUNDEF(VT);
1848 // fall through
1849 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
1850 R==APFloat::cmpLessThan, VT);
1851 case ISD::SETLT: if (R==APFloat::cmpUnordered)
1852 return getUNDEF(VT);
1853 // fall through
1854 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
1855 case ISD::SETGT: if (R==APFloat::cmpUnordered)
1856 return getUNDEF(VT);
1857 // fall through
1858 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
1859 case ISD::SETLE: if (R==APFloat::cmpUnordered)
1860 return getUNDEF(VT);
1861 // fall through
1862 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
1863 R==APFloat::cmpEqual, VT);
1864 case ISD::SETGE: if (R==APFloat::cmpUnordered)
1865 return getUNDEF(VT);
1866 // fall through
1867 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
1868 R==APFloat::cmpEqual, VT);
1869 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT);
1870 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT);
1871 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
1872 R==APFloat::cmpEqual, VT);
1873 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
1874 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
1875 R==APFloat::cmpLessThan, VT);
1876 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
1877 R==APFloat::cmpUnordered, VT);
1878 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
1879 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
1880 }
1881 } else {
1882 // Ensure that the constant occurs on the RHS.
1883 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
1884 MVT CompVT = N1.getValueType().getSimpleVT();
1885 if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
1886 return SDValue();
1887
1888 return getSetCC(dl, VT, N2, N1, SwappedCond);
1889 }
1890 }
1891
1892 // Could not fold it.
1893 return SDValue();
1894 }
1895
1896 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
1897 /// use this predicate to simplify operations downstream.
SignBitIsZero(SDValue Op,unsigned Depth) const1898 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
1899 // This predicate is not safe for vector operations.
1900 if (Op.getValueType().isVector())
1901 return false;
1902
1903 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
1904 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
1905 }
1906
1907 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
1908 /// this predicate to simplify operations downstream. Mask is known to be zero
1909 /// for bits that V cannot have.
MaskedValueIsZero(SDValue Op,const APInt & Mask,unsigned Depth) const1910 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
1911 unsigned Depth) const {
1912 APInt KnownZero, KnownOne;
1913 computeKnownBits(Op, KnownZero, KnownOne, Depth);
1914 return (KnownZero & Mask) == Mask;
1915 }
1916
1917 /// Determine which bits of Op are known to be either zero or one and return
1918 /// them in the KnownZero/KnownOne bitsets.
computeKnownBits(SDValue Op,APInt & KnownZero,APInt & KnownOne,unsigned Depth) const1919 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
1920 APInt &KnownOne, unsigned Depth) const {
1921 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
1922
1923 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
1924 if (Depth == 6)
1925 return; // Limit search depth.
1926
1927 APInt KnownZero2, KnownOne2;
1928
1929 switch (Op.getOpcode()) {
1930 case ISD::Constant:
1931 // We know all of the bits for a constant!
1932 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
1933 KnownZero = ~KnownOne;
1934 break;
1935 case ISD::AND:
1936 // If either the LHS or the RHS are Zero, the result is zero.
1937 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1938 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1939
1940 // Output known-1 bits are only known if set in both the LHS & RHS.
1941 KnownOne &= KnownOne2;
1942 // Output known-0 are known to be clear if zero in either the LHS | RHS.
1943 KnownZero |= KnownZero2;
1944 break;
1945 case ISD::OR:
1946 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1947 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1948
1949 // Output known-0 bits are only known if clear in both the LHS & RHS.
1950 KnownZero &= KnownZero2;
1951 // Output known-1 are known to be set if set in either the LHS | RHS.
1952 KnownOne |= KnownOne2;
1953 break;
1954 case ISD::XOR: {
1955 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1956 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1957
1958 // Output known-0 bits are known if clear or set in both the LHS & RHS.
1959 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
1960 // Output known-1 are known to be set if set in only one of the LHS, RHS.
1961 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
1962 KnownZero = KnownZeroOut;
1963 break;
1964 }
1965 case ISD::MUL: {
1966 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1967 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1968
1969 // If low bits are zero in either operand, output low known-0 bits.
1970 // Also compute a conserative estimate for high known-0 bits.
1971 // More trickiness is possible, but this is sufficient for the
1972 // interesting case of alignment computation.
1973 KnownOne.clearAllBits();
1974 unsigned TrailZ = KnownZero.countTrailingOnes() +
1975 KnownZero2.countTrailingOnes();
1976 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() +
1977 KnownZero2.countLeadingOnes(),
1978 BitWidth) - BitWidth;
1979
1980 TrailZ = std::min(TrailZ, BitWidth);
1981 LeadZ = std::min(LeadZ, BitWidth);
1982 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
1983 APInt::getHighBitsSet(BitWidth, LeadZ);
1984 break;
1985 }
1986 case ISD::UDIV: {
1987 // For the purposes of computing leading zeros we can conservatively
1988 // treat a udiv as a logical right shift by the power of 2 known to
1989 // be less than the denominator.
1990 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1991 unsigned LeadZ = KnownZero2.countLeadingOnes();
1992
1993 KnownOne2.clearAllBits();
1994 KnownZero2.clearAllBits();
1995 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
1996 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
1997 if (RHSUnknownLeadingOnes != BitWidth)
1998 LeadZ = std::min(BitWidth,
1999 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
2000
2001 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
2002 break;
2003 }
2004 case ISD::SELECT:
2005 computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
2006 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2007
2008 // Only known if known in both the LHS and RHS.
2009 KnownOne &= KnownOne2;
2010 KnownZero &= KnownZero2;
2011 break;
2012 case ISD::SELECT_CC:
2013 computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
2014 computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
2015
2016 // Only known if known in both the LHS and RHS.
2017 KnownOne &= KnownOne2;
2018 KnownZero &= KnownZero2;
2019 break;
2020 case ISD::SADDO:
2021 case ISD::UADDO:
2022 case ISD::SSUBO:
2023 case ISD::USUBO:
2024 case ISD::SMULO:
2025 case ISD::UMULO:
2026 if (Op.getResNo() != 1)
2027 break;
2028 // The boolean result conforms to getBooleanContents.
2029 // If we know the result of a setcc has the top bits zero, use this info.
2030 // We know that we have an integer-based boolean since these operations
2031 // are only available for integer.
2032 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2033 TargetLowering::ZeroOrOneBooleanContent &&
2034 BitWidth > 1)
2035 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2036 break;
2037 case ISD::SETCC:
2038 // If we know the result of a setcc has the top bits zero, use this info.
2039 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2040 TargetLowering::ZeroOrOneBooleanContent &&
2041 BitWidth > 1)
2042 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2043 break;
2044 case ISD::SHL:
2045 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0
2046 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2047 unsigned ShAmt = SA->getZExtValue();
2048
2049 // If the shift count is an invalid immediate, don't do anything.
2050 if (ShAmt >= BitWidth)
2051 break;
2052
2053 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2054 KnownZero <<= ShAmt;
2055 KnownOne <<= ShAmt;
2056 // low bits known zero.
2057 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
2058 }
2059 break;
2060 case ISD::SRL:
2061 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
2062 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2063 unsigned ShAmt = SA->getZExtValue();
2064
2065 // If the shift count is an invalid immediate, don't do anything.
2066 if (ShAmt >= BitWidth)
2067 break;
2068
2069 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2070 KnownZero = KnownZero.lshr(ShAmt);
2071 KnownOne = KnownOne.lshr(ShAmt);
2072
2073 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
2074 KnownZero |= HighBits; // High bits known zero.
2075 }
2076 break;
2077 case ISD::SRA:
2078 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2079 unsigned ShAmt = SA->getZExtValue();
2080
2081 // If the shift count is an invalid immediate, don't do anything.
2082 if (ShAmt >= BitWidth)
2083 break;
2084
2085 // If any of the demanded bits are produced by the sign extension, we also
2086 // demand the input sign bit.
2087 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
2088
2089 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2090 KnownZero = KnownZero.lshr(ShAmt);
2091 KnownOne = KnownOne.lshr(ShAmt);
2092
2093 // Handle the sign bits.
2094 APInt SignBit = APInt::getSignBit(BitWidth);
2095 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask.
2096
2097 if (KnownZero.intersects(SignBit)) {
2098 KnownZero |= HighBits; // New bits are known zero.
2099 } else if (KnownOne.intersects(SignBit)) {
2100 KnownOne |= HighBits; // New bits are known one.
2101 }
2102 }
2103 break;
2104 case ISD::SIGN_EXTEND_INREG: {
2105 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2106 unsigned EBits = EVT.getScalarType().getSizeInBits();
2107
2108 // Sign extension. Compute the demanded bits in the result that are not
2109 // present in the input.
2110 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2111
2112 APInt InSignBit = APInt::getSignBit(EBits);
2113 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2114
2115 // If the sign extended bits are demanded, we know that the sign
2116 // bit is demanded.
2117 InSignBit = InSignBit.zext(BitWidth);
2118 if (NewBits.getBoolValue())
2119 InputDemandedBits |= InSignBit;
2120
2121 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2122 KnownOne &= InputDemandedBits;
2123 KnownZero &= InputDemandedBits;
2124
2125 // If the sign bit of the input is known set or clear, then we know the
2126 // top bits of the result.
2127 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear
2128 KnownZero |= NewBits;
2129 KnownOne &= ~NewBits;
2130 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set
2131 KnownOne |= NewBits;
2132 KnownZero &= ~NewBits;
2133 } else { // Input sign bit unknown
2134 KnownZero &= ~NewBits;
2135 KnownOne &= ~NewBits;
2136 }
2137 break;
2138 }
2139 case ISD::CTTZ:
2140 case ISD::CTTZ_ZERO_UNDEF:
2141 case ISD::CTLZ:
2142 case ISD::CTLZ_ZERO_UNDEF:
2143 case ISD::CTPOP: {
2144 unsigned LowBits = Log2_32(BitWidth)+1;
2145 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
2146 KnownOne.clearAllBits();
2147 break;
2148 }
2149 case ISD::LOAD: {
2150 LoadSDNode *LD = cast<LoadSDNode>(Op);
2151 // If this is a ZEXTLoad and we are looking at the loaded value.
2152 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
2153 EVT VT = LD->getMemoryVT();
2154 unsigned MemBits = VT.getScalarType().getSizeInBits();
2155 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
2156 } else if (const MDNode *Ranges = LD->getRanges()) {
2157 computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
2158 }
2159 break;
2160 }
2161 case ISD::ZERO_EXTEND: {
2162 EVT InVT = Op.getOperand(0).getValueType();
2163 unsigned InBits = InVT.getScalarType().getSizeInBits();
2164 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
2165 KnownZero = KnownZero.trunc(InBits);
2166 KnownOne = KnownOne.trunc(InBits);
2167 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2168 KnownZero = KnownZero.zext(BitWidth);
2169 KnownOne = KnownOne.zext(BitWidth);
2170 KnownZero |= NewBits;
2171 break;
2172 }
2173 case ISD::SIGN_EXTEND: {
2174 EVT InVT = Op.getOperand(0).getValueType();
2175 unsigned InBits = InVT.getScalarType().getSizeInBits();
2176 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
2177
2178 KnownZero = KnownZero.trunc(InBits);
2179 KnownOne = KnownOne.trunc(InBits);
2180 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2181
2182 // Note if the sign bit is known to be zero or one.
2183 bool SignBitKnownZero = KnownZero.isNegative();
2184 bool SignBitKnownOne = KnownOne.isNegative();
2185
2186 KnownZero = KnownZero.zext(BitWidth);
2187 KnownOne = KnownOne.zext(BitWidth);
2188
2189 // If the sign bit is known zero or one, the top bits match.
2190 if (SignBitKnownZero)
2191 KnownZero |= NewBits;
2192 else if (SignBitKnownOne)
2193 KnownOne |= NewBits;
2194 break;
2195 }
2196 case ISD::ANY_EXTEND: {
2197 EVT InVT = Op.getOperand(0).getValueType();
2198 unsigned InBits = InVT.getScalarType().getSizeInBits();
2199 KnownZero = KnownZero.trunc(InBits);
2200 KnownOne = KnownOne.trunc(InBits);
2201 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2202 KnownZero = KnownZero.zext(BitWidth);
2203 KnownOne = KnownOne.zext(BitWidth);
2204 break;
2205 }
2206 case ISD::TRUNCATE: {
2207 EVT InVT = Op.getOperand(0).getValueType();
2208 unsigned InBits = InVT.getScalarType().getSizeInBits();
2209 KnownZero = KnownZero.zext(InBits);
2210 KnownOne = KnownOne.zext(InBits);
2211 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2212 KnownZero = KnownZero.trunc(BitWidth);
2213 KnownOne = KnownOne.trunc(BitWidth);
2214 break;
2215 }
2216 case ISD::AssertZext: {
2217 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2218 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
2219 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2220 KnownZero |= (~InMask);
2221 KnownOne &= (~KnownZero);
2222 break;
2223 }
2224 case ISD::FGETSIGN:
2225 // All bits are zero except the low bit.
2226 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2227 break;
2228
2229 case ISD::SUB: {
2230 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
2231 // We know that the top bits of C-X are clear if X contains less bits
2232 // than C (i.e. no wrap-around can happen). For example, 20-X is
2233 // positive if we can prove that X is >= 0 and < 16.
2234 if (CLHS->getAPIntValue().isNonNegative()) {
2235 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
2236 // NLZ can't be BitWidth with no sign bit
2237 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
2238 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2239
2240 // If all of the MaskV bits are known to be zero, then we know the
2241 // output top bits are zero, because we now know that the output is
2242 // from [0-C].
2243 if ((KnownZero2 & MaskV) == MaskV) {
2244 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
2245 // Top bits known zero.
2246 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
2247 }
2248 }
2249 }
2250 }
2251 // fall through
2252 case ISD::ADD:
2253 case ISD::ADDE: {
2254 // Output known-0 bits are known if clear or set in both the low clear bits
2255 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the
2256 // low 3 bits clear.
2257 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2258 unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
2259
2260 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2261 KnownZeroOut = std::min(KnownZeroOut,
2262 KnownZero2.countTrailingOnes());
2263
2264 if (Op.getOpcode() == ISD::ADD) {
2265 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
2266 break;
2267 }
2268
2269 // With ADDE, a carry bit may be added in, so we can only use this
2270 // information if we know (at least) that the low two bits are clear. We
2271 // then return to the caller that the low bit is unknown but that other bits
2272 // are known zero.
2273 if (KnownZeroOut >= 2) // ADDE
2274 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
2275 break;
2276 }
2277 case ISD::SREM:
2278 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2279 const APInt &RA = Rem->getAPIntValue().abs();
2280 if (RA.isPowerOf2()) {
2281 APInt LowBits = RA - 1;
2282 computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
2283
2284 // The low bits of the first operand are unchanged by the srem.
2285 KnownZero = KnownZero2 & LowBits;
2286 KnownOne = KnownOne2 & LowBits;
2287
2288 // If the first operand is non-negative or has all low bits zero, then
2289 // the upper bits are all zero.
2290 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
2291 KnownZero |= ~LowBits;
2292
2293 // If the first operand is negative and not all low bits are zero, then
2294 // the upper bits are all one.
2295 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
2296 KnownOne |= ~LowBits;
2297 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
2298 }
2299 }
2300 break;
2301 case ISD::UREM: {
2302 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2303 const APInt &RA = Rem->getAPIntValue();
2304 if (RA.isPowerOf2()) {
2305 APInt LowBits = (RA - 1);
2306 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
2307
2308 // The upper bits are all zero, the lower ones are unchanged.
2309 KnownZero = KnownZero2 | ~LowBits;
2310 KnownOne = KnownOne2 & LowBits;
2311 break;
2312 }
2313 }
2314
2315 // Since the result is less than or equal to either operand, any leading
2316 // zero bits in either operand must also exist in the result.
2317 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2318 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2319
2320 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
2321 KnownZero2.countLeadingOnes());
2322 KnownOne.clearAllBits();
2323 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
2324 break;
2325 }
2326 case ISD::FrameIndex:
2327 case ISD::TargetFrameIndex:
2328 if (unsigned Align = InferPtrAlignment(Op)) {
2329 // The low bits are known zero if the pointer is aligned.
2330 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
2331 break;
2332 }
2333 break;
2334
2335 default:
2336 if (Op.getOpcode() < ISD::BUILTIN_OP_END)
2337 break;
2338 // Fallthrough
2339 case ISD::INTRINSIC_WO_CHAIN:
2340 case ISD::INTRINSIC_W_CHAIN:
2341 case ISD::INTRINSIC_VOID:
2342 // Allow the target to implement this method for its nodes.
2343 TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
2344 break;
2345 }
2346
2347 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
2348 }
2349
2350 /// ComputeNumSignBits - Return the number of times the sign bit of the
2351 /// register is replicated into the other bits. We know that at least 1 bit
2352 /// is always equal to the sign bit (itself), but other cases can give us
2353 /// information. For example, immediately after an "SRA X, 2", we know that
2354 /// the top 3 bits are all equal to each other, so we return 3.
ComputeNumSignBits(SDValue Op,unsigned Depth) const2355 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
2356 EVT VT = Op.getValueType();
2357 assert(VT.isInteger() && "Invalid VT!");
2358 unsigned VTBits = VT.getScalarType().getSizeInBits();
2359 unsigned Tmp, Tmp2;
2360 unsigned FirstAnswer = 1;
2361
2362 if (Depth == 6)
2363 return 1; // Limit search depth.
2364
2365 switch (Op.getOpcode()) {
2366 default: break;
2367 case ISD::AssertSext:
2368 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2369 return VTBits-Tmp+1;
2370 case ISD::AssertZext:
2371 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2372 return VTBits-Tmp;
2373
2374 case ISD::Constant: {
2375 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
2376 return Val.getNumSignBits();
2377 }
2378
2379 case ISD::SIGN_EXTEND:
2380 Tmp =
2381 VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
2382 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
2383
2384 case ISD::SIGN_EXTEND_INREG:
2385 // Max of the input and what this extends.
2386 Tmp =
2387 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
2388 Tmp = VTBits-Tmp+1;
2389
2390 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2391 return std::max(Tmp, Tmp2);
2392
2393 case ISD::SRA:
2394 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2395 // SRA X, C -> adds C sign bits.
2396 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2397 Tmp += C->getZExtValue();
2398 if (Tmp > VTBits) Tmp = VTBits;
2399 }
2400 return Tmp;
2401 case ISD::SHL:
2402 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2403 // shl destroys sign bits.
2404 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2405 if (C->getZExtValue() >= VTBits || // Bad shift.
2406 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out.
2407 return Tmp - C->getZExtValue();
2408 }
2409 break;
2410 case ISD::AND:
2411 case ISD::OR:
2412 case ISD::XOR: // NOT is handled here.
2413 // Logical binary ops preserve the number of sign bits at the worst.
2414 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2415 if (Tmp != 1) {
2416 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2417 FirstAnswer = std::min(Tmp, Tmp2);
2418 // We computed what we know about the sign bits as our first
2419 // answer. Now proceed to the generic code that uses
2420 // computeKnownBits, and pick whichever answer is better.
2421 }
2422 break;
2423
2424 case ISD::SELECT:
2425 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2426 if (Tmp == 1) return 1; // Early out.
2427 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
2428 return std::min(Tmp, Tmp2);
2429
2430 case ISD::SADDO:
2431 case ISD::UADDO:
2432 case ISD::SSUBO:
2433 case ISD::USUBO:
2434 case ISD::SMULO:
2435 case ISD::UMULO:
2436 if (Op.getResNo() != 1)
2437 break;
2438 // The boolean result conforms to getBooleanContents. Fall through.
2439 // If setcc returns 0/-1, all bits are sign bits.
2440 // We know that we have an integer-based boolean since these operations
2441 // are only available for integer.
2442 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2443 TargetLowering::ZeroOrNegativeOneBooleanContent)
2444 return VTBits;
2445 break;
2446 case ISD::SETCC:
2447 // If setcc returns 0/-1, all bits are sign bits.
2448 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2449 TargetLowering::ZeroOrNegativeOneBooleanContent)
2450 return VTBits;
2451 break;
2452 case ISD::ROTL:
2453 case ISD::ROTR:
2454 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2455 unsigned RotAmt = C->getZExtValue() & (VTBits-1);
2456
2457 // Handle rotate right by N like a rotate left by 32-N.
2458 if (Op.getOpcode() == ISD::ROTR)
2459 RotAmt = (VTBits-RotAmt) & (VTBits-1);
2460
2461 // If we aren't rotating out all of the known-in sign bits, return the
2462 // number that are left. This handles rotl(sext(x), 1) for example.
2463 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2464 if (Tmp > RotAmt+1) return Tmp-RotAmt;
2465 }
2466 break;
2467 case ISD::ADD:
2468 // Add can have at most one carry bit. Thus we know that the output
2469 // is, at worst, one more bit than the inputs.
2470 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2471 if (Tmp == 1) return 1; // Early out.
2472
2473 // Special case decrementing a value (ADD X, -1):
2474 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2475 if (CRHS->isAllOnesValue()) {
2476 APInt KnownZero, KnownOne;
2477 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2478
2479 // If the input is known to be 0 or 1, the output is 0/-1, which is all
2480 // sign bits set.
2481 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
2482 return VTBits;
2483
2484 // If we are subtracting one from a positive number, there is no carry
2485 // out of the result.
2486 if (KnownZero.isNegative())
2487 return Tmp;
2488 }
2489
2490 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2491 if (Tmp2 == 1) return 1;
2492 return std::min(Tmp, Tmp2)-1;
2493
2494 case ISD::SUB:
2495 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2496 if (Tmp2 == 1) return 1;
2497
2498 // Handle NEG.
2499 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
2500 if (CLHS->isNullValue()) {
2501 APInt KnownZero, KnownOne;
2502 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2503 // If the input is known to be 0 or 1, the output is 0/-1, which is all
2504 // sign bits set.
2505 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
2506 return VTBits;
2507
2508 // If the input is known to be positive (the sign bit is known clear),
2509 // the output of the NEG has the same number of sign bits as the input.
2510 if (KnownZero.isNegative())
2511 return Tmp2;
2512
2513 // Otherwise, we treat this like a SUB.
2514 }
2515
2516 // Sub can have at most one carry bit. Thus we know that the output
2517 // is, at worst, one more bit than the inputs.
2518 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2519 if (Tmp == 1) return 1; // Early out.
2520 return std::min(Tmp, Tmp2)-1;
2521 case ISD::TRUNCATE:
2522 // FIXME: it's tricky to do anything useful for this, but it is an important
2523 // case for targets like X86.
2524 break;
2525 }
2526
2527 // If we are looking at the loaded value of the SDNode.
2528 if (Op.getResNo() == 0) {
2529 // Handle LOADX separately here. EXTLOAD case will fallthrough.
2530 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
2531 unsigned ExtType = LD->getExtensionType();
2532 switch (ExtType) {
2533 default: break;
2534 case ISD::SEXTLOAD: // '17' bits known
2535 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2536 return VTBits-Tmp+1;
2537 case ISD::ZEXTLOAD: // '16' bits known
2538 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2539 return VTBits-Tmp;
2540 }
2541 }
2542 }
2543
2544 // Allow the target to implement this method for its nodes.
2545 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2546 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2547 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2548 Op.getOpcode() == ISD::INTRINSIC_VOID) {
2549 unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
2550 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
2551 }
2552
2553 // Finally, if we can prove that the top bits of the result are 0's or 1's,
2554 // use this information.
2555 APInt KnownZero, KnownOne;
2556 computeKnownBits(Op, KnownZero, KnownOne, Depth);
2557
2558 APInt Mask;
2559 if (KnownZero.isNegative()) { // sign bit is 0
2560 Mask = KnownZero;
2561 } else if (KnownOne.isNegative()) { // sign bit is 1;
2562 Mask = KnownOne;
2563 } else {
2564 // Nothing known.
2565 return FirstAnswer;
2566 }
2567
2568 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
2569 // the number of identical bits in the top of the input value.
2570 Mask = ~Mask;
2571 Mask <<= Mask.getBitWidth()-VTBits;
2572 // Return # leading zeros. We use 'min' here in case Val was zero before
2573 // shifting. We don't want to return '64' as for an i32 "0".
2574 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
2575 }
2576
2577 /// isBaseWithConstantOffset - Return true if the specified operand is an
2578 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
2579 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
2580 /// semantics as an ADD. This handles the equivalence:
2581 /// X|Cst == X+Cst iff X&Cst = 0.
isBaseWithConstantOffset(SDValue Op) const2582 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
2583 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
2584 !isa<ConstantSDNode>(Op.getOperand(1)))
2585 return false;
2586
2587 if (Op.getOpcode() == ISD::OR &&
2588 !MaskedValueIsZero(Op.getOperand(0),
2589 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
2590 return false;
2591
2592 return true;
2593 }
2594
2595
isKnownNeverNaN(SDValue Op) const2596 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
2597 // If we're told that NaNs won't happen, assume they won't.
2598 if (getTarget().Options.NoNaNsFPMath)
2599 return true;
2600
2601 // If the value is a constant, we can obviously see if it is a NaN or not.
2602 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2603 return !C->getValueAPF().isNaN();
2604
2605 // TODO: Recognize more cases here.
2606
2607 return false;
2608 }
2609
isKnownNeverZero(SDValue Op) const2610 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
2611 // If the value is a constant, we can obviously see if it is a zero or not.
2612 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2613 return !C->isZero();
2614
2615 // TODO: Recognize more cases here.
2616 switch (Op.getOpcode()) {
2617 default: break;
2618 case ISD::OR:
2619 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2620 return !C->isNullValue();
2621 break;
2622 }
2623
2624 return false;
2625 }
2626
isEqualTo(SDValue A,SDValue B) const2627 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
2628 // Check the obvious case.
2629 if (A == B) return true;
2630
2631 // For for negative and positive zero.
2632 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
2633 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
2634 if (CA->isZero() && CB->isZero()) return true;
2635
2636 // Otherwise they may not be equal.
2637 return false;
2638 }
2639
2640 /// getNode - Gets or creates the specified node.
2641 ///
getNode(unsigned Opcode,SDLoc DL,EVT VT)2642 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
2643 FoldingSetNodeID ID;
2644 AddNodeIDNode(ID, Opcode, getVTList(VT), None);
2645 void *IP = nullptr;
2646 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
2647 return SDValue(E, 0);
2648
2649 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
2650 DL.getDebugLoc(), getVTList(VT));
2651 CSEMap.InsertNode(N, IP);
2652
2653 InsertNode(N);
2654 return SDValue(N, 0);
2655 }
2656
getNode(unsigned Opcode,SDLoc DL,EVT VT,SDValue Operand)2657 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
2658 EVT VT, SDValue Operand) {
2659 // Constant fold unary operations with an integer constant operand. Even
2660 // opaque constant will be folded, because the folding of unary operations
2661 // doesn't create new constants with different values. Nevertheless, the
2662 // opaque flag is preserved during folding to prevent future folding with
2663 // other constants.
2664 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
2665 const APInt &Val = C->getAPIntValue();
2666 switch (Opcode) {
2667 default: break;
2668 case ISD::SIGN_EXTEND:
2669 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
2670 C->isTargetOpcode(), C->isOpaque());
2671 case ISD::ANY_EXTEND:
2672 case ISD::ZERO_EXTEND:
2673 case ISD::TRUNCATE:
2674 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
2675 C->isTargetOpcode(), C->isOpaque());
2676 case ISD::UINT_TO_FP:
2677 case ISD::SINT_TO_FP: {
2678 APFloat apf(EVTToAPFloatSemantics(VT),
2679 APInt::getNullValue(VT.getSizeInBits()));
2680 (void)apf.convertFromAPInt(Val,
2681 Opcode==ISD::SINT_TO_FP,
2682 APFloat::rmNearestTiesToEven);
2683 return getConstantFP(apf, VT);
2684 }
2685 case ISD::BITCAST:
2686 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
2687 return getConstantFP(APFloat(APFloat::IEEEhalf, Val), VT);
2688 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
2689 return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
2690 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
2691 return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
2692 break;
2693 case ISD::BSWAP:
2694 return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
2695 C->isOpaque());
2696 case ISD::CTPOP:
2697 return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
2698 C->isOpaque());
2699 case ISD::CTLZ:
2700 case ISD::CTLZ_ZERO_UNDEF:
2701 return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
2702 C->isOpaque());
2703 case ISD::CTTZ:
2704 case ISD::CTTZ_ZERO_UNDEF:
2705 return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
2706 C->isOpaque());
2707 }
2708 }
2709
2710 // Constant fold unary operations with a floating point constant operand.
2711 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
2712 APFloat V = C->getValueAPF(); // make copy
2713 switch (Opcode) {
2714 case ISD::FNEG:
2715 V.changeSign();
2716 return getConstantFP(V, VT);
2717 case ISD::FABS:
2718 V.clearSign();
2719 return getConstantFP(V, VT);
2720 case ISD::FCEIL: {
2721 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
2722 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2723 return getConstantFP(V, VT);
2724 break;
2725 }
2726 case ISD::FTRUNC: {
2727 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
2728 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2729 return getConstantFP(V, VT);
2730 break;
2731 }
2732 case ISD::FFLOOR: {
2733 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
2734 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2735 return getConstantFP(V, VT);
2736 break;
2737 }
2738 case ISD::FP_EXTEND: {
2739 bool ignored;
2740 // This can return overflow, underflow, or inexact; we don't care.
2741 // FIXME need to be more flexible about rounding mode.
2742 (void)V.convert(EVTToAPFloatSemantics(VT),
2743 APFloat::rmNearestTiesToEven, &ignored);
2744 return getConstantFP(V, VT);
2745 }
2746 case ISD::FP_TO_SINT:
2747 case ISD::FP_TO_UINT: {
2748 integerPart x[2];
2749 bool ignored;
2750 static_assert(integerPartWidth >= 64, "APFloat parts too small!");
2751 // FIXME need to be more flexible about rounding mode.
2752 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
2753 Opcode==ISD::FP_TO_SINT,
2754 APFloat::rmTowardZero, &ignored);
2755 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual
2756 break;
2757 APInt api(VT.getSizeInBits(), x);
2758 return getConstant(api, VT);
2759 }
2760 case ISD::BITCAST:
2761 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
2762 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), VT);
2763 else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
2764 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
2765 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
2766 return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
2767 break;
2768 }
2769 }
2770
2771 // Constant fold unary operations with a vector integer operand.
2772 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
2773 if (BV->isConstant()) {
2774 switch (Opcode) {
2775 default:
2776 // FIXME: Entirely reasonable to perform folding of other unary
2777 // operations here as the need arises.
2778 break;
2779 case ISD::UINT_TO_FP:
2780 case ISD::SINT_TO_FP: {
2781 SmallVector<SDValue, 8> Ops;
2782 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
2783 SDValue OpN = BV->getOperand(i);
2784 // Let the above scalar folding handle the conversion of each
2785 // element.
2786 OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
2787 OpN);
2788 Ops.push_back(OpN);
2789 }
2790 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
2791 }
2792 }
2793 }
2794 }
2795
2796 unsigned OpOpcode = Operand.getNode()->getOpcode();
2797 switch (Opcode) {
2798 case ISD::TokenFactor:
2799 case ISD::MERGE_VALUES:
2800 case ISD::CONCAT_VECTORS:
2801 return Operand; // Factor, merge or concat of one node? No need.
2802 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
2803 case ISD::FP_EXTEND:
2804 assert(VT.isFloatingPoint() &&
2805 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
2806 if (Operand.getValueType() == VT) return Operand; // noop conversion.
2807 assert((!VT.isVector() ||
2808 VT.getVectorNumElements() ==
2809 Operand.getValueType().getVectorNumElements()) &&
2810 "Vector element count mismatch!");
2811 if (Operand.getOpcode() == ISD::UNDEF)
2812 return getUNDEF(VT);
2813 break;
2814 case ISD::SIGN_EXTEND:
2815 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
2816 "Invalid SIGN_EXTEND!");
2817 if (Operand.getValueType() == VT) return Operand; // noop extension
2818 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2819 "Invalid sext node, dst < src!");
2820 assert((!VT.isVector() ||
2821 VT.getVectorNumElements() ==
2822 Operand.getValueType().getVectorNumElements()) &&
2823 "Vector element count mismatch!");
2824 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
2825 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
2826 else if (OpOpcode == ISD::UNDEF)
2827 // sext(undef) = 0, because the top bits will all be the same.
2828 return getConstant(0, VT);
2829 break;
2830 case ISD::ZERO_EXTEND:
2831 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
2832 "Invalid ZERO_EXTEND!");
2833 if (Operand.getValueType() == VT) return Operand; // noop extension
2834 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2835 "Invalid zext node, dst < src!");
2836 assert((!VT.isVector() ||
2837 VT.getVectorNumElements() ==
2838 Operand.getValueType().getVectorNumElements()) &&
2839 "Vector element count mismatch!");
2840 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
2841 return getNode(ISD::ZERO_EXTEND, DL, VT,
2842 Operand.getNode()->getOperand(0));
2843 else if (OpOpcode == ISD::UNDEF)
2844 // zext(undef) = 0, because the top bits will be zero.
2845 return getConstant(0, VT);
2846 break;
2847 case ISD::ANY_EXTEND:
2848 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
2849 "Invalid ANY_EXTEND!");
2850 if (Operand.getValueType() == VT) return Operand; // noop extension
2851 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2852 "Invalid anyext node, dst < src!");
2853 assert((!VT.isVector() ||
2854 VT.getVectorNumElements() ==
2855 Operand.getValueType().getVectorNumElements()) &&
2856 "Vector element count mismatch!");
2857
2858 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
2859 OpOpcode == ISD::ANY_EXTEND)
2860 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
2861 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
2862 else if (OpOpcode == ISD::UNDEF)
2863 return getUNDEF(VT);
2864
2865 // (ext (trunx x)) -> x
2866 if (OpOpcode == ISD::TRUNCATE) {
2867 SDValue OpOp = Operand.getNode()->getOperand(0);
2868 if (OpOp.getValueType() == VT)
2869 return OpOp;
2870 }
2871 break;
2872 case ISD::TRUNCATE:
2873 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
2874 "Invalid TRUNCATE!");
2875 if (Operand.getValueType() == VT) return Operand; // noop truncate
2876 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
2877 "Invalid truncate node, src < dst!");
2878 assert((!VT.isVector() ||
2879 VT.getVectorNumElements() ==
2880 Operand.getValueType().getVectorNumElements()) &&
2881 "Vector element count mismatch!");
2882 if (OpOpcode == ISD::TRUNCATE)
2883 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
2884 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
2885 OpOpcode == ISD::ANY_EXTEND) {
2886 // If the source is smaller than the dest, we still need an extend.
2887 if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
2888 .bitsLT(VT.getScalarType()))
2889 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
2890 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
2891 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
2892 return Operand.getNode()->getOperand(0);
2893 }
2894 if (OpOpcode == ISD::UNDEF)
2895 return getUNDEF(VT);
2896 break;
2897 case ISD::BITCAST:
2898 // Basic sanity checking.
2899 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
2900 && "Cannot BITCAST between types of different sizes!");
2901 if (VT == Operand.getValueType()) return Operand; // noop conversion.
2902 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
2903 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
2904 if (OpOpcode == ISD::UNDEF)
2905 return getUNDEF(VT);
2906 break;
2907 case ISD::SCALAR_TO_VECTOR:
2908 assert(VT.isVector() && !Operand.getValueType().isVector() &&
2909 (VT.getVectorElementType() == Operand.getValueType() ||
2910 (VT.getVectorElementType().isInteger() &&
2911 Operand.getValueType().isInteger() &&
2912 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
2913 "Illegal SCALAR_TO_VECTOR node!");
2914 if (OpOpcode == ISD::UNDEF)
2915 return getUNDEF(VT);
2916 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
2917 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
2918 isa<ConstantSDNode>(Operand.getOperand(1)) &&
2919 Operand.getConstantOperandVal(1) == 0 &&
2920 Operand.getOperand(0).getValueType() == VT)
2921 return Operand.getOperand(0);
2922 break;
2923 case ISD::FNEG:
2924 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
2925 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
2926 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
2927 Operand.getNode()->getOperand(0));
2928 if (OpOpcode == ISD::FNEG) // --X -> X
2929 return Operand.getNode()->getOperand(0);
2930 break;
2931 case ISD::FABS:
2932 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
2933 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
2934 break;
2935 }
2936
2937 SDNode *N;
2938 SDVTList VTs = getVTList(VT);
2939 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
2940 FoldingSetNodeID ID;
2941 SDValue Ops[1] = { Operand };
2942 AddNodeIDNode(ID, Opcode, VTs, Ops);
2943 void *IP = nullptr;
2944 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
2945 return SDValue(E, 0);
2946
2947 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
2948 DL.getDebugLoc(), VTs, Operand);
2949 CSEMap.InsertNode(N, IP);
2950 } else {
2951 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
2952 DL.getDebugLoc(), VTs, Operand);
2953 }
2954
2955 InsertNode(N);
2956 return SDValue(N, 0);
2957 }
2958
FoldConstantArithmetic(unsigned Opcode,EVT VT,SDNode * Cst1,SDNode * Cst2)2959 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
2960 SDNode *Cst1, SDNode *Cst2) {
2961 // If the opcode is a target-specific ISD node, there's nothing we can
2962 // do here and the operand rules may not line up with the below, so
2963 // bail early.
2964 if (Opcode >= ISD::BUILTIN_OP_END)
2965 return SDValue();
2966
2967 SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
2968 SmallVector<SDValue, 4> Outputs;
2969 EVT SVT = VT.getScalarType();
2970
2971 ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
2972 ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
2973 if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
2974 return SDValue();
2975
2976 if (Scalar1 && Scalar2)
2977 // Scalar instruction.
2978 Inputs.push_back(std::make_pair(Scalar1, Scalar2));
2979 else {
2980 // For vectors extract each constant element into Inputs so we can constant
2981 // fold them individually.
2982 BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
2983 BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
2984 if (!BV1 || !BV2)
2985 return SDValue();
2986
2987 assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
2988
2989 for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
2990 ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
2991 ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
2992 if (!V1 || !V2) // Not a constant, bail.
2993 return SDValue();
2994
2995 if (V1->isOpaque() || V2->isOpaque())
2996 return SDValue();
2997
2998 // Avoid BUILD_VECTOR nodes that perform implicit truncation.
2999 // FIXME: This is valid and could be handled by truncating the APInts.
3000 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
3001 return SDValue();
3002
3003 Inputs.push_back(std::make_pair(V1, V2));
3004 }
3005 }
3006
3007 // We have a number of constant values, constant fold them element by element.
3008 for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
3009 const APInt &C1 = Inputs[I].first->getAPIntValue();
3010 const APInt &C2 = Inputs[I].second->getAPIntValue();
3011
3012 switch (Opcode) {
3013 case ISD::ADD:
3014 Outputs.push_back(getConstant(C1 + C2, SVT));
3015 break;
3016 case ISD::SUB:
3017 Outputs.push_back(getConstant(C1 - C2, SVT));
3018 break;
3019 case ISD::MUL:
3020 Outputs.push_back(getConstant(C1 * C2, SVT));
3021 break;
3022 case ISD::UDIV:
3023 if (!C2.getBoolValue())
3024 return SDValue();
3025 Outputs.push_back(getConstant(C1.udiv(C2), SVT));
3026 break;
3027 case ISD::UREM:
3028 if (!C2.getBoolValue())
3029 return SDValue();
3030 Outputs.push_back(getConstant(C1.urem(C2), SVT));
3031 break;
3032 case ISD::SDIV:
3033 if (!C2.getBoolValue())
3034 return SDValue();
3035 Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
3036 break;
3037 case ISD::SREM:
3038 if (!C2.getBoolValue())
3039 return SDValue();
3040 Outputs.push_back(getConstant(C1.srem(C2), SVT));
3041 break;
3042 case ISD::AND:
3043 Outputs.push_back(getConstant(C1 & C2, SVT));
3044 break;
3045 case ISD::OR:
3046 Outputs.push_back(getConstant(C1 | C2, SVT));
3047 break;
3048 case ISD::XOR:
3049 Outputs.push_back(getConstant(C1 ^ C2, SVT));
3050 break;
3051 case ISD::SHL:
3052 Outputs.push_back(getConstant(C1 << C2, SVT));
3053 break;
3054 case ISD::SRL:
3055 Outputs.push_back(getConstant(C1.lshr(C2), SVT));
3056 break;
3057 case ISD::SRA:
3058 Outputs.push_back(getConstant(C1.ashr(C2), SVT));
3059 break;
3060 case ISD::ROTL:
3061 Outputs.push_back(getConstant(C1.rotl(C2), SVT));
3062 break;
3063 case ISD::ROTR:
3064 Outputs.push_back(getConstant(C1.rotr(C2), SVT));
3065 break;
3066 default:
3067 return SDValue();
3068 }
3069 }
3070
3071 assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
3072 "Expected a scalar or vector!"));
3073
3074 // Handle the scalar case first.
3075 if (!VT.isVector())
3076 return Outputs.back();
3077
3078 // We may have a vector type but a scalar result. Create a splat.
3079 Outputs.resize(VT.getVectorNumElements(), Outputs.back());
3080
3081 // Build a big vector out of the scalar elements we generated.
3082 return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
3083 }
3084
getNode(unsigned Opcode,SDLoc DL,EVT VT,SDValue N1,SDValue N2,bool nuw,bool nsw,bool exact)3085 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
3086 SDValue N2, bool nuw, bool nsw, bool exact) {
3087 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
3088 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
3089 switch (Opcode) {
3090 default: break;
3091 case ISD::TokenFactor:
3092 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
3093 N2.getValueType() == MVT::Other && "Invalid token factor!");
3094 // Fold trivial token factors.
3095 if (N1.getOpcode() == ISD::EntryToken) return N2;
3096 if (N2.getOpcode() == ISD::EntryToken) return N1;
3097 if (N1 == N2) return N1;
3098 break;
3099 case ISD::CONCAT_VECTORS:
3100 // Concat of UNDEFs is UNDEF.
3101 if (N1.getOpcode() == ISD::UNDEF &&
3102 N2.getOpcode() == ISD::UNDEF)
3103 return getUNDEF(VT);
3104
3105 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
3106 // one big BUILD_VECTOR.
3107 if (N1.getOpcode() == ISD::BUILD_VECTOR &&
3108 N2.getOpcode() == ISD::BUILD_VECTOR) {
3109 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
3110 N1.getNode()->op_end());
3111 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
3112 return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
3113 }
3114 break;
3115 case ISD::AND:
3116 assert(VT.isInteger() && "This operator does not apply to FP types!");
3117 assert(N1.getValueType() == N2.getValueType() &&
3118 N1.getValueType() == VT && "Binary operator types must match!");
3119 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
3120 // worth handling here.
3121 if (N2C && N2C->isNullValue())
3122 return N2;
3123 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
3124 return N1;
3125 break;
3126 case ISD::OR:
3127 case ISD::XOR:
3128 case ISD::ADD:
3129 case ISD::SUB:
3130 assert(VT.isInteger() && "This operator does not apply to FP types!");
3131 assert(N1.getValueType() == N2.getValueType() &&
3132 N1.getValueType() == VT && "Binary operator types must match!");
3133 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
3134 // it's worth handling here.
3135 if (N2C && N2C->isNullValue())
3136 return N1;
3137 break;
3138 case ISD::UDIV:
3139 case ISD::UREM:
3140 case ISD::MULHU:
3141 case ISD::MULHS:
3142 case ISD::MUL:
3143 case ISD::SDIV:
3144 case ISD::SREM:
3145 assert(VT.isInteger() && "This operator does not apply to FP types!");
3146 assert(N1.getValueType() == N2.getValueType() &&
3147 N1.getValueType() == VT && "Binary operator types must match!");
3148 break;
3149 case ISD::FADD:
3150 case ISD::FSUB:
3151 case ISD::FMUL:
3152 case ISD::FDIV:
3153 case ISD::FREM:
3154 if (getTarget().Options.UnsafeFPMath) {
3155 if (Opcode == ISD::FADD) {
3156 // 0+x --> x
3157 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
3158 if (CFP->getValueAPF().isZero())
3159 return N2;
3160 // x+0 --> x
3161 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
3162 if (CFP->getValueAPF().isZero())
3163 return N1;
3164 } else if (Opcode == ISD::FSUB) {
3165 // x-0 --> x
3166 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
3167 if (CFP->getValueAPF().isZero())
3168 return N1;
3169 } else if (Opcode == ISD::FMUL) {
3170 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
3171 SDValue V = N2;
3172
3173 // If the first operand isn't the constant, try the second
3174 if (!CFP) {
3175 CFP = dyn_cast<ConstantFPSDNode>(N2);
3176 V = N1;
3177 }
3178
3179 if (CFP) {
3180 // 0*x --> 0
3181 if (CFP->isZero())
3182 return SDValue(CFP,0);
3183 // 1*x --> x
3184 if (CFP->isExactlyValue(1.0))
3185 return V;
3186 }
3187 }
3188 }
3189 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
3190 assert(N1.getValueType() == N2.getValueType() &&
3191 N1.getValueType() == VT && "Binary operator types must match!");
3192 break;
3193 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
3194 assert(N1.getValueType() == VT &&
3195 N1.getValueType().isFloatingPoint() &&
3196 N2.getValueType().isFloatingPoint() &&
3197 "Invalid FCOPYSIGN!");
3198 break;
3199 case ISD::SHL:
3200 case ISD::SRA:
3201 case ISD::SRL:
3202 case ISD::ROTL:
3203 case ISD::ROTR:
3204 assert(VT == N1.getValueType() &&
3205 "Shift operators return type must be the same as their first arg");
3206 assert(VT.isInteger() && N2.getValueType().isInteger() &&
3207 "Shifts only work on integers");
3208 assert((!VT.isVector() || VT == N2.getValueType()) &&
3209 "Vector shift amounts must be in the same as their first arg");
3210 // Verify that the shift amount VT is bit enough to hold valid shift
3211 // amounts. This catches things like trying to shift an i1024 value by an
3212 // i8, which is easy to fall into in generic code that uses
3213 // TLI.getShiftAmount().
3214 assert(N2.getValueType().getSizeInBits() >=
3215 Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
3216 "Invalid use of small shift amount with oversized value!");
3217
3218 // Always fold shifts of i1 values so the code generator doesn't need to
3219 // handle them. Since we know the size of the shift has to be less than the
3220 // size of the value, the shift/rotate count is guaranteed to be zero.
3221 if (VT == MVT::i1)
3222 return N1;
3223 if (N2C && N2C->isNullValue())
3224 return N1;
3225 break;
3226 case ISD::FP_ROUND_INREG: {
3227 EVT EVT = cast<VTSDNode>(N2)->getVT();
3228 assert(VT == N1.getValueType() && "Not an inreg round!");
3229 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
3230 "Cannot FP_ROUND_INREG integer types");
3231 assert(EVT.isVector() == VT.isVector() &&
3232 "FP_ROUND_INREG type should be vector iff the operand "
3233 "type is vector!");
3234 assert((!EVT.isVector() ||
3235 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
3236 "Vector element counts must match in FP_ROUND_INREG");
3237 assert(EVT.bitsLE(VT) && "Not rounding down!");
3238 (void)EVT;
3239 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding.
3240 break;
3241 }
3242 case ISD::FP_ROUND:
3243 assert(VT.isFloatingPoint() &&
3244 N1.getValueType().isFloatingPoint() &&
3245 VT.bitsLE(N1.getValueType()) &&
3246 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
3247 if (N1.getValueType() == VT) return N1; // noop conversion.
3248 break;
3249 case ISD::AssertSext:
3250 case ISD::AssertZext: {
3251 EVT EVT = cast<VTSDNode>(N2)->getVT();
3252 assert(VT == N1.getValueType() && "Not an inreg extend!");
3253 assert(VT.isInteger() && EVT.isInteger() &&
3254 "Cannot *_EXTEND_INREG FP types");
3255 assert(!EVT.isVector() &&
3256 "AssertSExt/AssertZExt type should be the vector element type "
3257 "rather than the vector type!");
3258 assert(EVT.bitsLE(VT) && "Not extending!");
3259 if (VT == EVT) return N1; // noop assertion.
3260 break;
3261 }
3262 case ISD::SIGN_EXTEND_INREG: {
3263 EVT EVT = cast<VTSDNode>(N2)->getVT();
3264 assert(VT == N1.getValueType() && "Not an inreg extend!");
3265 assert(VT.isInteger() && EVT.isInteger() &&
3266 "Cannot *_EXTEND_INREG FP types");
3267 assert(EVT.isVector() == VT.isVector() &&
3268 "SIGN_EXTEND_INREG type should be vector iff the operand "
3269 "type is vector!");
3270 assert((!EVT.isVector() ||
3271 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
3272 "Vector element counts must match in SIGN_EXTEND_INREG");
3273 assert(EVT.bitsLE(VT) && "Not extending!");
3274 if (EVT == VT) return N1; // Not actually extending
3275
3276 if (N1C) {
3277 APInt Val = N1C->getAPIntValue();
3278 unsigned FromBits = EVT.getScalarType().getSizeInBits();
3279 Val <<= Val.getBitWidth()-FromBits;
3280 Val = Val.ashr(Val.getBitWidth()-FromBits);
3281 return getConstant(Val, VT);
3282 }
3283 break;
3284 }
3285 case ISD::EXTRACT_VECTOR_ELT:
3286 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
3287 if (N1.getOpcode() == ISD::UNDEF)
3288 return getUNDEF(VT);
3289
3290 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
3291 // expanding copies of large vectors from registers.
3292 if (N2C &&
3293 N1.getOpcode() == ISD::CONCAT_VECTORS &&
3294 N1.getNumOperands() > 0) {
3295 unsigned Factor =
3296 N1.getOperand(0).getValueType().getVectorNumElements();
3297 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
3298 N1.getOperand(N2C->getZExtValue() / Factor),
3299 getConstant(N2C->getZExtValue() % Factor,
3300 N2.getValueType()));
3301 }
3302
3303 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
3304 // expanding large vector constants.
3305 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
3306 SDValue Elt = N1.getOperand(N2C->getZExtValue());
3307
3308 if (VT != Elt.getValueType())
3309 // If the vector element type is not legal, the BUILD_VECTOR operands
3310 // are promoted and implicitly truncated, and the result implicitly
3311 // extended. Make that explicit here.
3312 Elt = getAnyExtOrTrunc(Elt, DL, VT);
3313
3314 return Elt;
3315 }
3316
3317 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
3318 // operations are lowered to scalars.
3319 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
3320 // If the indices are the same, return the inserted element else
3321 // if the indices are known different, extract the element from
3322 // the original vector.
3323 SDValue N1Op2 = N1.getOperand(2);
3324 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
3325
3326 if (N1Op2C && N2C) {
3327 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
3328 if (VT == N1.getOperand(1).getValueType())
3329 return N1.getOperand(1);
3330 else
3331 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
3332 }
3333
3334 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
3335 }
3336 }
3337 break;
3338 case ISD::EXTRACT_ELEMENT:
3339 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
3340 assert(!N1.getValueType().isVector() && !VT.isVector() &&
3341 (N1.getValueType().isInteger() == VT.isInteger()) &&
3342 N1.getValueType() != VT &&
3343 "Wrong types for EXTRACT_ELEMENT!");
3344
3345 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
3346 // 64-bit integers into 32-bit parts. Instead of building the extract of
3347 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
3348 if (N1.getOpcode() == ISD::BUILD_PAIR)
3349 return N1.getOperand(N2C->getZExtValue());
3350
3351 // EXTRACT_ELEMENT of a constant int is also very common.
3352 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
3353 unsigned ElementSize = VT.getSizeInBits();
3354 unsigned Shift = ElementSize * N2C->getZExtValue();
3355 APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
3356 return getConstant(ShiftedVal.trunc(ElementSize), VT);
3357 }
3358 break;
3359 case ISD::EXTRACT_SUBVECTOR: {
3360 SDValue Index = N2;
3361 if (VT.isSimple() && N1.getValueType().isSimple()) {
3362 assert(VT.isVector() && N1.getValueType().isVector() &&
3363 "Extract subvector VTs must be a vectors!");
3364 assert(VT.getVectorElementType() ==
3365 N1.getValueType().getVectorElementType() &&
3366 "Extract subvector VTs must have the same element type!");
3367 assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
3368 "Extract subvector must be from larger vector to smaller vector!");
3369
3370 if (isa<ConstantSDNode>(Index.getNode())) {
3371 assert((VT.getVectorNumElements() +
3372 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
3373 <= N1.getValueType().getVectorNumElements())
3374 && "Extract subvector overflow!");
3375 }
3376
3377 // Trivial extraction.
3378 if (VT.getSimpleVT() == N1.getSimpleValueType())
3379 return N1;
3380 }
3381 break;
3382 }
3383 }
3384
3385 // Perform trivial constant folding.
3386 if (SDValue SV =
3387 FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()))
3388 return SV;
3389
3390 // Canonicalize constant to RHS if commutative.
3391 if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
3392 std::swap(N1C, N2C);
3393 std::swap(N1, N2);
3394 }
3395
3396 // Constant fold FP operations.
3397 bool HasFPExceptions = TLI->hasFloatingPointExceptions();
3398 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
3399 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
3400 if (N1CFP) {
3401 if (!N2CFP && isCommutativeBinOp(Opcode)) {
3402 // Canonicalize constant to RHS if commutative.
3403 std::swap(N1CFP, N2CFP);
3404 std::swap(N1, N2);
3405 } else if (N2CFP) {
3406 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
3407 APFloat::opStatus s;
3408 switch (Opcode) {
3409 case ISD::FADD:
3410 s = V1.add(V2, APFloat::rmNearestTiesToEven);
3411 if (!HasFPExceptions || s != APFloat::opInvalidOp)
3412 return getConstantFP(V1, VT);
3413 break;
3414 case ISD::FSUB:
3415 s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
3416 if (!HasFPExceptions || s!=APFloat::opInvalidOp)
3417 return getConstantFP(V1, VT);
3418 break;
3419 case ISD::FMUL:
3420 s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
3421 if (!HasFPExceptions || s!=APFloat::opInvalidOp)
3422 return getConstantFP(V1, VT);
3423 break;
3424 case ISD::FDIV:
3425 s = V1.divide(V2, APFloat::rmNearestTiesToEven);
3426 if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
3427 s!=APFloat::opDivByZero)) {
3428 return getConstantFP(V1, VT);
3429 }
3430 break;
3431 case ISD::FREM :
3432 s = V1.mod(V2, APFloat::rmNearestTiesToEven);
3433 if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
3434 s!=APFloat::opDivByZero)) {
3435 return getConstantFP(V1, VT);
3436 }
3437 break;
3438 case ISD::FCOPYSIGN:
3439 V1.copySign(V2);
3440 return getConstantFP(V1, VT);
3441 default: break;
3442 }
3443 }
3444
3445 if (Opcode == ISD::FP_ROUND) {
3446 APFloat V = N1CFP->getValueAPF(); // make copy
3447 bool ignored;
3448 // This can return overflow, underflow, or inexact; we don't care.
3449 // FIXME need to be more flexible about rounding mode.
3450 (void)V.convert(EVTToAPFloatSemantics(VT),
3451 APFloat::rmNearestTiesToEven, &ignored);
3452 return getConstantFP(V, VT);
3453 }
3454 }
3455
3456 // Canonicalize an UNDEF to the RHS, even over a constant.
3457 if (N1.getOpcode() == ISD::UNDEF) {
3458 if (isCommutativeBinOp(Opcode)) {
3459 std::swap(N1, N2);
3460 } else {
3461 switch (Opcode) {
3462 case ISD::FP_ROUND_INREG:
3463 case ISD::SIGN_EXTEND_INREG:
3464 case ISD::SUB:
3465 case ISD::FSUB:
3466 case ISD::FDIV:
3467 case ISD::FREM:
3468 case ISD::SRA:
3469 return N1; // fold op(undef, arg2) -> undef
3470 case ISD::UDIV:
3471 case ISD::SDIV:
3472 case ISD::UREM:
3473 case ISD::SREM:
3474 case ISD::SRL:
3475 case ISD::SHL:
3476 if (!VT.isVector())
3477 return getConstant(0, VT); // fold op(undef, arg2) -> 0
3478 // For vectors, we can't easily build an all zero vector, just return
3479 // the LHS.
3480 return N2;
3481 }
3482 }
3483 }
3484
3485 // Fold a bunch of operators when the RHS is undef.
3486 if (N2.getOpcode() == ISD::UNDEF) {
3487 switch (Opcode) {
3488 case ISD::XOR:
3489 if (N1.getOpcode() == ISD::UNDEF)
3490 // Handle undef ^ undef -> 0 special case. This is a common
3491 // idiom (misuse).
3492 return getConstant(0, VT);
3493 // fallthrough
3494 case ISD::ADD:
3495 case ISD::ADDC:
3496 case ISD::ADDE:
3497 case ISD::SUB:
3498 case ISD::UDIV:
3499 case ISD::SDIV:
3500 case ISD::UREM:
3501 case ISD::SREM:
3502 return N2; // fold op(arg1, undef) -> undef
3503 case ISD::FADD:
3504 case ISD::FSUB:
3505 case ISD::FMUL:
3506 case ISD::FDIV:
3507 case ISD::FREM:
3508 if (getTarget().Options.UnsafeFPMath)
3509 return N2;
3510 break;
3511 case ISD::MUL:
3512 case ISD::AND:
3513 case ISD::SRL:
3514 case ISD::SHL:
3515 if (!VT.isVector())
3516 return getConstant(0, VT); // fold op(arg1, undef) -> 0
3517 // For vectors, we can't easily build an all zero vector, just return
3518 // the LHS.
3519 return N1;
3520 case ISD::OR:
3521 if (!VT.isVector())
3522 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
3523 // For vectors, we can't easily build an all one vector, just return
3524 // the LHS.
3525 return N1;
3526 case ISD::SRA:
3527 return N1;
3528 }
3529 }
3530
3531 // Memoize this node if possible.
3532 BinarySDNode *N;
3533 SDVTList VTs = getVTList(VT);
3534 const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
3535 if (VT != MVT::Glue) {
3536 SDValue Ops[] = {N1, N2};
3537 FoldingSetNodeID ID;
3538 AddNodeIDNode(ID, Opcode, VTs, Ops);
3539 if (BinOpHasFlags)
3540 AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
3541 void *IP = nullptr;
3542 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3543 return SDValue(E, 0);
3544
3545 N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
3546
3547 CSEMap.InsertNode(N, IP);
3548 } else {
3549
3550 N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
3551 }
3552
3553 InsertNode(N);
3554 return SDValue(N, 0);
3555 }
3556
getNode(unsigned Opcode,SDLoc DL,EVT VT,SDValue N1,SDValue N2,SDValue N3)3557 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3558 SDValue N1, SDValue N2, SDValue N3) {
3559 // Perform various simplifications.
3560 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
3561 switch (Opcode) {
3562 case ISD::FMA: {
3563 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
3564 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
3565 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
3566 if (N1CFP && N2CFP && N3CFP) {
3567 APFloat V1 = N1CFP->getValueAPF();
3568 const APFloat &V2 = N2CFP->getValueAPF();
3569 const APFloat &V3 = N3CFP->getValueAPF();
3570 APFloat::opStatus s =
3571 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
3572 if (s != APFloat::opInvalidOp)
3573 return getConstantFP(V1, VT);
3574 }
3575 break;
3576 }
3577 case ISD::CONCAT_VECTORS:
3578 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
3579 // one big BUILD_VECTOR.
3580 if (N1.getOpcode() == ISD::BUILD_VECTOR &&
3581 N2.getOpcode() == ISD::BUILD_VECTOR &&
3582 N3.getOpcode() == ISD::BUILD_VECTOR) {
3583 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
3584 N1.getNode()->op_end());
3585 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
3586 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
3587 return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
3588 }
3589 break;
3590 case ISD::SETCC: {
3591 // Use FoldSetCC to simplify SETCC's.
3592 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
3593 if (Simp.getNode()) return Simp;
3594 break;
3595 }
3596 case ISD::SELECT:
3597 if (N1C) {
3598 if (N1C->getZExtValue())
3599 return N2; // select true, X, Y -> X
3600 return N3; // select false, X, Y -> Y
3601 }
3602
3603 if (N2 == N3) return N2; // select C, X, X -> X
3604 break;
3605 case ISD::VECTOR_SHUFFLE:
3606 llvm_unreachable("should use getVectorShuffle constructor!");
3607 case ISD::INSERT_SUBVECTOR: {
3608 SDValue Index = N3;
3609 if (VT.isSimple() && N1.getValueType().isSimple()
3610 && N2.getValueType().isSimple()) {
3611 assert(VT.isVector() && N1.getValueType().isVector() &&
3612 N2.getValueType().isVector() &&
3613 "Insert subvector VTs must be a vectors");
3614 assert(VT == N1.getValueType() &&
3615 "Dest and insert subvector source types must match!");
3616 assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
3617 "Insert subvector must be from smaller vector to larger vector!");
3618 if (isa<ConstantSDNode>(Index.getNode())) {
3619 assert((N2.getValueType().getVectorNumElements() +
3620 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
3621 <= VT.getVectorNumElements())
3622 && "Insert subvector overflow!");
3623 }
3624
3625 // Trivial insertion.
3626 if (VT.getSimpleVT() == N2.getSimpleValueType())
3627 return N2;
3628 }
3629 break;
3630 }
3631 case ISD::BITCAST:
3632 // Fold bit_convert nodes from a type to themselves.
3633 if (N1.getValueType() == VT)
3634 return N1;
3635 break;
3636 }
3637
3638 // Memoize node if it doesn't produce a flag.
3639 SDNode *N;
3640 SDVTList VTs = getVTList(VT);
3641 if (VT != MVT::Glue) {
3642 SDValue Ops[] = { N1, N2, N3 };
3643 FoldingSetNodeID ID;
3644 AddNodeIDNode(ID, Opcode, VTs, Ops);
3645 void *IP = nullptr;
3646 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3647 return SDValue(E, 0);
3648
3649 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
3650 DL.getDebugLoc(), VTs, N1, N2, N3);
3651 CSEMap.InsertNode(N, IP);
3652 } else {
3653 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
3654 DL.getDebugLoc(), VTs, N1, N2, N3);
3655 }
3656
3657 InsertNode(N);
3658 return SDValue(N, 0);
3659 }
3660
getNode(unsigned Opcode,SDLoc DL,EVT VT,SDValue N1,SDValue N2,SDValue N3,SDValue N4)3661 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3662 SDValue N1, SDValue N2, SDValue N3,
3663 SDValue N4) {
3664 SDValue Ops[] = { N1, N2, N3, N4 };
3665 return getNode(Opcode, DL, VT, Ops);
3666 }
3667
getNode(unsigned Opcode,SDLoc DL,EVT VT,SDValue N1,SDValue N2,SDValue N3,SDValue N4,SDValue N5)3668 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3669 SDValue N1, SDValue N2, SDValue N3,
3670 SDValue N4, SDValue N5) {
3671 SDValue Ops[] = { N1, N2, N3, N4, N5 };
3672 return getNode(Opcode, DL, VT, Ops);
3673 }
3674
3675 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
3676 /// the incoming stack arguments to be loaded from the stack.
getStackArgumentTokenFactor(SDValue Chain)3677 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
3678 SmallVector<SDValue, 8> ArgChains;
3679
3680 // Include the original chain at the beginning of the list. When this is
3681 // used by target LowerCall hooks, this helps legalize find the
3682 // CALLSEQ_BEGIN node.
3683 ArgChains.push_back(Chain);
3684
3685 // Add a chain value for each stack argument.
3686 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
3687 UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
3688 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
3689 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
3690 if (FI->getIndex() < 0)
3691 ArgChains.push_back(SDValue(L, 1));
3692
3693 // Build a tokenfactor for all the chains.
3694 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
3695 }
3696
3697 /// getMemsetValue - Vectorized representation of the memset value
3698 /// operand.
getMemsetValue(SDValue Value,EVT VT,SelectionDAG & DAG,SDLoc dl)3699 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
3700 SDLoc dl) {
3701 assert(Value.getOpcode() != ISD::UNDEF);
3702
3703 unsigned NumBits = VT.getScalarType().getSizeInBits();
3704 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
3705 assert(C->getAPIntValue().getBitWidth() == 8);
3706 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
3707 if (VT.isInteger())
3708 return DAG.getConstant(Val, VT);
3709 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
3710 }
3711
3712 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
3713 if (NumBits > 8) {
3714 // Use a multiplication with 0x010101... to extend the input to the
3715 // required length.
3716 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
3717 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
3718 }
3719
3720 return Value;
3721 }
3722
3723 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
3724 /// used when a memcpy is turned into a memset when the source is a constant
3725 /// string ptr.
getMemsetStringVal(EVT VT,SDLoc dl,SelectionDAG & DAG,const TargetLowering & TLI,StringRef Str)3726 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
3727 const TargetLowering &TLI, StringRef Str) {
3728 // Handle vector with all elements zero.
3729 if (Str.empty()) {
3730 if (VT.isInteger())
3731 return DAG.getConstant(0, VT);
3732 else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
3733 return DAG.getConstantFP(0.0, VT);
3734 else if (VT.isVector()) {
3735 unsigned NumElts = VT.getVectorNumElements();
3736 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
3737 return DAG.getNode(ISD::BITCAST, dl, VT,
3738 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
3739 EltVT, NumElts)));
3740 } else
3741 llvm_unreachable("Expected type!");
3742 }
3743
3744 assert(!VT.isVector() && "Can't handle vector type here!");
3745 unsigned NumVTBits = VT.getSizeInBits();
3746 unsigned NumVTBytes = NumVTBits / 8;
3747 unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
3748
3749 APInt Val(NumVTBits, 0);
3750 if (TLI.isLittleEndian()) {
3751 for (unsigned i = 0; i != NumBytes; ++i)
3752 Val |= (uint64_t)(unsigned char)Str[i] << i*8;
3753 } else {
3754 for (unsigned i = 0; i != NumBytes; ++i)
3755 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
3756 }
3757
3758 // If the "cost" of materializing the integer immediate is less than the cost
3759 // of a load, then it is cost effective to turn the load into the immediate.
3760 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
3761 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
3762 return DAG.getConstant(Val, VT);
3763 return SDValue(nullptr, 0);
3764 }
3765
3766 /// getMemBasePlusOffset - Returns base and offset node for the
3767 ///
getMemBasePlusOffset(SDValue Base,unsigned Offset,SDLoc dl,SelectionDAG & DAG)3768 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
3769 SelectionDAG &DAG) {
3770 EVT VT = Base.getValueType();
3771 return DAG.getNode(ISD::ADD, dl,
3772 VT, Base, DAG.getConstant(Offset, VT));
3773 }
3774
3775 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
3776 ///
isMemSrcFromString(SDValue Src,StringRef & Str)3777 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
3778 unsigned SrcDelta = 0;
3779 GlobalAddressSDNode *G = nullptr;
3780 if (Src.getOpcode() == ISD::GlobalAddress)
3781 G = cast<GlobalAddressSDNode>(Src);
3782 else if (Src.getOpcode() == ISD::ADD &&
3783 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
3784 Src.getOperand(1).getOpcode() == ISD::Constant) {
3785 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
3786 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
3787 }
3788 if (!G)
3789 return false;
3790
3791 return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
3792 }
3793
3794 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
3795 /// to replace the memset / memcpy. Return true if the number of memory ops
3796 /// is below the threshold. It returns the types of the sequence of
3797 /// memory ops to perform memset / memcpy by reference.
FindOptimalMemOpLowering(std::vector<EVT> & MemOps,unsigned Limit,uint64_t Size,unsigned DstAlign,unsigned SrcAlign,bool IsMemset,bool ZeroMemset,bool MemcpyStrSrc,bool AllowOverlap,SelectionDAG & DAG,const TargetLowering & TLI)3798 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
3799 unsigned Limit, uint64_t Size,
3800 unsigned DstAlign, unsigned SrcAlign,
3801 bool IsMemset,
3802 bool ZeroMemset,
3803 bool MemcpyStrSrc,
3804 bool AllowOverlap,
3805 SelectionDAG &DAG,
3806 const TargetLowering &TLI) {
3807 assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
3808 "Expecting memcpy / memset source to meet alignment requirement!");
3809 // If 'SrcAlign' is zero, that means the memory operation does not need to
3810 // load the value, i.e. memset or memcpy from constant string. Otherwise,
3811 // it's the inferred alignment of the source. 'DstAlign', on the other hand,
3812 // is the specified alignment of the memory operation. If it is zero, that
3813 // means it's possible to change the alignment of the destination.
3814 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
3815 // not need to be loaded.
3816 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
3817 IsMemset, ZeroMemset, MemcpyStrSrc,
3818 DAG.getMachineFunction());
3819
3820 if (VT == MVT::Other) {
3821 unsigned AS = 0;
3822 if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
3823 TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
3824 VT = TLI.getPointerTy();
3825 } else {
3826 switch (DstAlign & 7) {
3827 case 0: VT = MVT::i64; break;
3828 case 4: VT = MVT::i32; break;
3829 case 2: VT = MVT::i16; break;
3830 default: VT = MVT::i8; break;
3831 }
3832 }
3833
3834 MVT LVT = MVT::i64;
3835 while (!TLI.isTypeLegal(LVT))
3836 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
3837 assert(LVT.isInteger());
3838
3839 if (VT.bitsGT(LVT))
3840 VT = LVT;
3841 }
3842
3843 unsigned NumMemOps = 0;
3844 while (Size != 0) {
3845 unsigned VTSize = VT.getSizeInBits() / 8;
3846 while (VTSize > Size) {
3847 // For now, only use non-vector load / store's for the left-over pieces.
3848 EVT NewVT = VT;
3849 unsigned NewVTSize;
3850
3851 bool Found = false;
3852 if (VT.isVector() || VT.isFloatingPoint()) {
3853 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
3854 if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
3855 TLI.isSafeMemOpType(NewVT.getSimpleVT()))
3856 Found = true;
3857 else if (NewVT == MVT::i64 &&
3858 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
3859 TLI.isSafeMemOpType(MVT::f64)) {
3860 // i64 is usually not legal on 32-bit targets, but f64 may be.
3861 NewVT = MVT::f64;
3862 Found = true;
3863 }
3864 }
3865
3866 if (!Found) {
3867 do {
3868 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
3869 if (NewVT == MVT::i8)
3870 break;
3871 } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
3872 }
3873 NewVTSize = NewVT.getSizeInBits() / 8;
3874
3875 // If the new VT cannot cover all of the remaining bits, then consider
3876 // issuing a (or a pair of) unaligned and overlapping load / store.
3877 // FIXME: Only does this for 64-bit or more since we don't have proper
3878 // cost model for unaligned load / store.
3879 bool Fast;
3880 unsigned AS = 0;
3881 if (NumMemOps && AllowOverlap &&
3882 VTSize >= 8 && NewVTSize < Size &&
3883 TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
3884 VTSize = Size;
3885 else {
3886 VT = NewVT;
3887 VTSize = NewVTSize;
3888 }
3889 }
3890
3891 if (++NumMemOps > Limit)
3892 return false;
3893
3894 MemOps.push_back(VT);
3895 Size -= VTSize;
3896 }
3897
3898 return true;
3899 }
3900
getMemcpyLoadsAndStores(SelectionDAG & DAG,SDLoc dl,SDValue Chain,SDValue Dst,SDValue Src,uint64_t Size,unsigned Align,bool isVol,bool AlwaysInline,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo)3901 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
3902 SDValue Chain, SDValue Dst,
3903 SDValue Src, uint64_t Size,
3904 unsigned Align, bool isVol,
3905 bool AlwaysInline,
3906 MachinePointerInfo DstPtrInfo,
3907 MachinePointerInfo SrcPtrInfo) {
3908 // Turn a memcpy of undef to nop.
3909 if (Src.getOpcode() == ISD::UNDEF)
3910 return Chain;
3911
3912 // Expand memcpy to a series of load and store ops if the size operand falls
3913 // below a certain threshold.
3914 // TODO: In the AlwaysInline case, if the size is big then generate a loop
3915 // rather than maybe a humongous number of loads and stores.
3916 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3917 std::vector<EVT> MemOps;
3918 bool DstAlignCanChange = false;
3919 MachineFunction &MF = DAG.getMachineFunction();
3920 MachineFrameInfo *MFI = MF.getFrameInfo();
3921 bool OptSize =
3922 MF.getFunction()->getAttributes().
3923 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
3924 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
3925 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
3926 DstAlignCanChange = true;
3927 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
3928 if (Align > SrcAlign)
3929 SrcAlign = Align;
3930 StringRef Str;
3931 bool CopyFromStr = isMemSrcFromString(Src, Str);
3932 bool isZeroStr = CopyFromStr && Str.empty();
3933 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
3934
3935 if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
3936 (DstAlignCanChange ? 0 : Align),
3937 (isZeroStr ? 0 : SrcAlign),
3938 false, false, CopyFromStr, true, DAG, TLI))
3939 return SDValue();
3940
3941 if (DstAlignCanChange) {
3942 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
3943 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
3944
3945 // Don't promote to an alignment that would require dynamic stack
3946 // realignment.
3947 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
3948 if (!TRI->needsStackRealignment(MF))
3949 while (NewAlign > Align &&
3950 TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
3951 NewAlign /= 2;
3952
3953 if (NewAlign > Align) {
3954 // Give the stack frame object a larger alignment if needed.
3955 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
3956 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
3957 Align = NewAlign;
3958 }
3959 }
3960
3961 SmallVector<SDValue, 8> OutChains;
3962 unsigned NumMemOps = MemOps.size();
3963 uint64_t SrcOff = 0, DstOff = 0;
3964 for (unsigned i = 0; i != NumMemOps; ++i) {
3965 EVT VT = MemOps[i];
3966 unsigned VTSize = VT.getSizeInBits() / 8;
3967 SDValue Value, Store;
3968
3969 if (VTSize > Size) {
3970 // Issuing an unaligned load / store pair that overlaps with the previous
3971 // pair. Adjust the offset accordingly.
3972 assert(i == NumMemOps-1 && i != 0);
3973 SrcOff -= VTSize - Size;
3974 DstOff -= VTSize - Size;
3975 }
3976
3977 if (CopyFromStr &&
3978 (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
3979 // It's unlikely a store of a vector immediate can be done in a single
3980 // instruction. It would require a load from a constantpool first.
3981 // We only handle zero vectors here.
3982 // FIXME: Handle other cases where store of vector immediate is done in
3983 // a single instruction.
3984 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
3985 if (Value.getNode())
3986 Store = DAG.getStore(Chain, dl, Value,
3987 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
3988 DstPtrInfo.getWithOffset(DstOff), isVol,
3989 false, Align);
3990 }
3991
3992 if (!Store.getNode()) {
3993 // The type might not be legal for the target. This should only happen
3994 // if the type is smaller than a legal type, as on PPC, so the right
3995 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
3996 // to Load/Store if NVT==VT.
3997 // FIXME does the case above also need this?
3998 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3999 assert(NVT.bitsGE(VT));
4000 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
4001 getMemBasePlusOffset(Src, SrcOff, dl, DAG),
4002 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
4003 false, MinAlign(SrcAlign, SrcOff));
4004 Store = DAG.getTruncStore(Chain, dl, Value,
4005 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4006 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
4007 false, Align);
4008 }
4009 OutChains.push_back(Store);
4010 SrcOff += VTSize;
4011 DstOff += VTSize;
4012 Size -= VTSize;
4013 }
4014
4015 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4016 }
4017
getMemmoveLoadsAndStores(SelectionDAG & DAG,SDLoc dl,SDValue Chain,SDValue Dst,SDValue Src,uint64_t Size,unsigned Align,bool isVol,bool AlwaysInline,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo)4018 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
4019 SDValue Chain, SDValue Dst,
4020 SDValue Src, uint64_t Size,
4021 unsigned Align, bool isVol,
4022 bool AlwaysInline,
4023 MachinePointerInfo DstPtrInfo,
4024 MachinePointerInfo SrcPtrInfo) {
4025 // Turn a memmove of undef to nop.
4026 if (Src.getOpcode() == ISD::UNDEF)
4027 return Chain;
4028
4029 // Expand memmove to a series of load and store ops if the size operand falls
4030 // below a certain threshold.
4031 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4032 std::vector<EVT> MemOps;
4033 bool DstAlignCanChange = false;
4034 MachineFunction &MF = DAG.getMachineFunction();
4035 MachineFrameInfo *MFI = MF.getFrameInfo();
4036 bool OptSize = MF.getFunction()->getAttributes().
4037 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
4038 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
4039 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
4040 DstAlignCanChange = true;
4041 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
4042 if (Align > SrcAlign)
4043 SrcAlign = Align;
4044 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
4045
4046 if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
4047 (DstAlignCanChange ? 0 : Align), SrcAlign,
4048 false, false, false, false, DAG, TLI))
4049 return SDValue();
4050
4051 if (DstAlignCanChange) {
4052 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
4053 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
4054 if (NewAlign > Align) {
4055 // Give the stack frame object a larger alignment if needed.
4056 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
4057 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
4058 Align = NewAlign;
4059 }
4060 }
4061
4062 uint64_t SrcOff = 0, DstOff = 0;
4063 SmallVector<SDValue, 8> LoadValues;
4064 SmallVector<SDValue, 8> LoadChains;
4065 SmallVector<SDValue, 8> OutChains;
4066 unsigned NumMemOps = MemOps.size();
4067 for (unsigned i = 0; i < NumMemOps; i++) {
4068 EVT VT = MemOps[i];
4069 unsigned VTSize = VT.getSizeInBits() / 8;
4070 SDValue Value;
4071
4072 Value = DAG.getLoad(VT, dl, Chain,
4073 getMemBasePlusOffset(Src, SrcOff, dl, DAG),
4074 SrcPtrInfo.getWithOffset(SrcOff), isVol,
4075 false, false, SrcAlign);
4076 LoadValues.push_back(Value);
4077 LoadChains.push_back(Value.getValue(1));
4078 SrcOff += VTSize;
4079 }
4080 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
4081 OutChains.clear();
4082 for (unsigned i = 0; i < NumMemOps; i++) {
4083 EVT VT = MemOps[i];
4084 unsigned VTSize = VT.getSizeInBits() / 8;
4085 SDValue Store;
4086
4087 Store = DAG.getStore(Chain, dl, LoadValues[i],
4088 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4089 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
4090 OutChains.push_back(Store);
4091 DstOff += VTSize;
4092 }
4093
4094 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4095 }
4096
4097 /// \brief Lower the call to 'memset' intrinsic function into a series of store
4098 /// operations.
4099 ///
4100 /// \param DAG Selection DAG where lowered code is placed.
4101 /// \param dl Link to corresponding IR location.
4102 /// \param Chain Control flow dependency.
4103 /// \param Dst Pointer to destination memory location.
4104 /// \param Src Value of byte to write into the memory.
4105 /// \param Size Number of bytes to write.
4106 /// \param Align Alignment of the destination in bytes.
4107 /// \param isVol True if destination is volatile.
4108 /// \param DstPtrInfo IR information on the memory pointer.
4109 /// \returns New head in the control flow, if lowering was successful, empty
4110 /// SDValue otherwise.
4111 ///
4112 /// The function tries to replace 'llvm.memset' intrinsic with several store
4113 /// operations and value calculation code. This is usually profitable for small
4114 /// memory size.
getMemsetStores(SelectionDAG & DAG,SDLoc dl,SDValue Chain,SDValue Dst,SDValue Src,uint64_t Size,unsigned Align,bool isVol,MachinePointerInfo DstPtrInfo)4115 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
4116 SDValue Chain, SDValue Dst,
4117 SDValue Src, uint64_t Size,
4118 unsigned Align, bool isVol,
4119 MachinePointerInfo DstPtrInfo) {
4120 // Turn a memset of undef to nop.
4121 if (Src.getOpcode() == ISD::UNDEF)
4122 return Chain;
4123
4124 // Expand memset to a series of load/store ops if the size operand
4125 // falls below a certain threshold.
4126 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4127 std::vector<EVT> MemOps;
4128 bool DstAlignCanChange = false;
4129 MachineFunction &MF = DAG.getMachineFunction();
4130 MachineFrameInfo *MFI = MF.getFrameInfo();
4131 bool OptSize = MF.getFunction()->getAttributes().
4132 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
4133 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
4134 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
4135 DstAlignCanChange = true;
4136 bool IsZeroVal =
4137 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
4138 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
4139 Size, (DstAlignCanChange ? 0 : Align), 0,
4140 true, IsZeroVal, false, true, DAG, TLI))
4141 return SDValue();
4142
4143 if (DstAlignCanChange) {
4144 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
4145 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
4146 if (NewAlign > Align) {
4147 // Give the stack frame object a larger alignment if needed.
4148 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
4149 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
4150 Align = NewAlign;
4151 }
4152 }
4153
4154 SmallVector<SDValue, 8> OutChains;
4155 uint64_t DstOff = 0;
4156 unsigned NumMemOps = MemOps.size();
4157
4158 // Find the largest store and generate the bit pattern for it.
4159 EVT LargestVT = MemOps[0];
4160 for (unsigned i = 1; i < NumMemOps; i++)
4161 if (MemOps[i].bitsGT(LargestVT))
4162 LargestVT = MemOps[i];
4163 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
4164
4165 for (unsigned i = 0; i < NumMemOps; i++) {
4166 EVT VT = MemOps[i];
4167 unsigned VTSize = VT.getSizeInBits() / 8;
4168 if (VTSize > Size) {
4169 // Issuing an unaligned load / store pair that overlaps with the previous
4170 // pair. Adjust the offset accordingly.
4171 assert(i == NumMemOps-1 && i != 0);
4172 DstOff -= VTSize - Size;
4173 }
4174
4175 // If this store is smaller than the largest store see whether we can get
4176 // the smaller value for free with a truncate.
4177 SDValue Value = MemSetValue;
4178 if (VT.bitsLT(LargestVT)) {
4179 if (!LargestVT.isVector() && !VT.isVector() &&
4180 TLI.isTruncateFree(LargestVT, VT))
4181 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
4182 else
4183 Value = getMemsetValue(Src, VT, DAG, dl);
4184 }
4185 assert(Value.getValueType() == VT && "Value with wrong type.");
4186 SDValue Store = DAG.getStore(Chain, dl, Value,
4187 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4188 DstPtrInfo.getWithOffset(DstOff),
4189 isVol, false, Align);
4190 OutChains.push_back(Store);
4191 DstOff += VT.getSizeInBits() / 8;
4192 Size -= VTSize;
4193 }
4194
4195 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4196 }
4197
getMemcpy(SDValue Chain,SDLoc dl,SDValue Dst,SDValue Src,SDValue Size,unsigned Align,bool isVol,bool AlwaysInline,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo)4198 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
4199 SDValue Src, SDValue Size,
4200 unsigned Align, bool isVol, bool AlwaysInline,
4201 MachinePointerInfo DstPtrInfo,
4202 MachinePointerInfo SrcPtrInfo) {
4203 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4204
4205 // Check to see if we should lower the memcpy to loads and stores first.
4206 // For cases within the target-specified limits, this is the best choice.
4207 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4208 if (ConstantSize) {
4209 // Memcpy with size zero? Just return the original chain.
4210 if (ConstantSize->isNullValue())
4211 return Chain;
4212
4213 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
4214 ConstantSize->getZExtValue(),Align,
4215 isVol, false, DstPtrInfo, SrcPtrInfo);
4216 if (Result.getNode())
4217 return Result;
4218 }
4219
4220 // Then check to see if we should lower the memcpy with target-specific
4221 // code. If the target chooses to do this, this is the next best.
4222 SDValue Result =
4223 TSI->EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
4224 isVol, AlwaysInline, DstPtrInfo, SrcPtrInfo);
4225 if (Result.getNode())
4226 return Result;
4227
4228 // If we really need inline code and the target declined to provide it,
4229 // use a (potentially long) sequence of loads and stores.
4230 if (AlwaysInline) {
4231 assert(ConstantSize && "AlwaysInline requires a constant size!");
4232 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
4233 ConstantSize->getZExtValue(), Align, isVol,
4234 true, DstPtrInfo, SrcPtrInfo);
4235 }
4236
4237 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
4238 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
4239 // respect volatile, so they may do things like read or write memory
4240 // beyond the given memory regions. But fixing this isn't easy, and most
4241 // people don't care.
4242
4243 // Emit a library call.
4244 TargetLowering::ArgListTy Args;
4245 TargetLowering::ArgListEntry Entry;
4246 Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
4247 Entry.Node = Dst; Args.push_back(Entry);
4248 Entry.Node = Src; Args.push_back(Entry);
4249 Entry.Node = Size; Args.push_back(Entry);
4250 // FIXME: pass in SDLoc
4251 TargetLowering::CallLoweringInfo CLI(*this);
4252 CLI.setDebugLoc(dl).setChain(Chain)
4253 .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
4254 Type::getVoidTy(*getContext()),
4255 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
4256 TLI->getPointerTy()), std::move(Args), 0)
4257 .setDiscardResult();
4258 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4259
4260 return CallResult.second;
4261 }
4262
getMemmove(SDValue Chain,SDLoc dl,SDValue Dst,SDValue Src,SDValue Size,unsigned Align,bool isVol,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo)4263 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
4264 SDValue Src, SDValue Size,
4265 unsigned Align, bool isVol,
4266 MachinePointerInfo DstPtrInfo,
4267 MachinePointerInfo SrcPtrInfo) {
4268 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4269
4270 // Check to see if we should lower the memmove to loads and stores first.
4271 // For cases within the target-specified limits, this is the best choice.
4272 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4273 if (ConstantSize) {
4274 // Memmove with size zero? Just return the original chain.
4275 if (ConstantSize->isNullValue())
4276 return Chain;
4277
4278 SDValue Result =
4279 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
4280 ConstantSize->getZExtValue(), Align, isVol,
4281 false, DstPtrInfo, SrcPtrInfo);
4282 if (Result.getNode())
4283 return Result;
4284 }
4285
4286 // Then check to see if we should lower the memmove with target-specific
4287 // code. If the target chooses to do this, this is the next best.
4288 SDValue Result = TSI->EmitTargetCodeForMemmove(
4289 *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
4290 if (Result.getNode())
4291 return Result;
4292
4293 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
4294 // not be safe. See memcpy above for more details.
4295
4296 // Emit a library call.
4297 TargetLowering::ArgListTy Args;
4298 TargetLowering::ArgListEntry Entry;
4299 Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
4300 Entry.Node = Dst; Args.push_back(Entry);
4301 Entry.Node = Src; Args.push_back(Entry);
4302 Entry.Node = Size; Args.push_back(Entry);
4303 // FIXME: pass in SDLoc
4304 TargetLowering::CallLoweringInfo CLI(*this);
4305 CLI.setDebugLoc(dl).setChain(Chain)
4306 .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
4307 Type::getVoidTy(*getContext()),
4308 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
4309 TLI->getPointerTy()), std::move(Args), 0)
4310 .setDiscardResult();
4311 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4312
4313 return CallResult.second;
4314 }
4315
getMemset(SDValue Chain,SDLoc dl,SDValue Dst,SDValue Src,SDValue Size,unsigned Align,bool isVol,MachinePointerInfo DstPtrInfo)4316 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
4317 SDValue Src, SDValue Size,
4318 unsigned Align, bool isVol,
4319 MachinePointerInfo DstPtrInfo) {
4320 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4321
4322 // Check to see if we should lower the memset to stores first.
4323 // For cases within the target-specified limits, this is the best choice.
4324 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4325 if (ConstantSize) {
4326 // Memset with size zero? Just return the original chain.
4327 if (ConstantSize->isNullValue())
4328 return Chain;
4329
4330 SDValue Result =
4331 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
4332 Align, isVol, DstPtrInfo);
4333
4334 if (Result.getNode())
4335 return Result;
4336 }
4337
4338 // Then check to see if we should lower the memset with target-specific
4339 // code. If the target chooses to do this, this is the next best.
4340 SDValue Result = TSI->EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src,
4341 Size, Align, isVol, DstPtrInfo);
4342 if (Result.getNode())
4343 return Result;
4344
4345 // Emit a library call.
4346 Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
4347 TargetLowering::ArgListTy Args;
4348 TargetLowering::ArgListEntry Entry;
4349 Entry.Node = Dst; Entry.Ty = IntPtrTy;
4350 Args.push_back(Entry);
4351 Entry.Node = Src;
4352 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
4353 Args.push_back(Entry);
4354 Entry.Node = Size;
4355 Entry.Ty = IntPtrTy;
4356 Args.push_back(Entry);
4357
4358 // FIXME: pass in SDLoc
4359 TargetLowering::CallLoweringInfo CLI(*this);
4360 CLI.setDebugLoc(dl).setChain(Chain)
4361 .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
4362 Type::getVoidTy(*getContext()),
4363 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
4364 TLI->getPointerTy()), std::move(Args), 0)
4365 .setDiscardResult();
4366
4367 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4368 return CallResult.second;
4369 }
4370
getAtomic(unsigned Opcode,SDLoc dl,EVT MemVT,SDVTList VTList,ArrayRef<SDValue> Ops,MachineMemOperand * MMO,AtomicOrdering SuccessOrdering,AtomicOrdering FailureOrdering,SynchronizationScope SynchScope)4371 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4372 SDVTList VTList, ArrayRef<SDValue> Ops,
4373 MachineMemOperand *MMO,
4374 AtomicOrdering SuccessOrdering,
4375 AtomicOrdering FailureOrdering,
4376 SynchronizationScope SynchScope) {
4377 FoldingSetNodeID ID;
4378 ID.AddInteger(MemVT.getRawBits());
4379 AddNodeIDNode(ID, Opcode, VTList, Ops);
4380 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4381 void* IP = nullptr;
4382 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4383 cast<AtomicSDNode>(E)->refineAlignment(MMO);
4384 return SDValue(E, 0);
4385 }
4386
4387 // Allocate the operands array for the node out of the BumpPtrAllocator, since
4388 // SDNode doesn't have access to it. This memory will be "leaked" when
4389 // the node is deallocated, but recovered when the allocator is released.
4390 // If the number of operands is less than 5 we use AtomicSDNode's internal
4391 // storage.
4392 unsigned NumOps = Ops.size();
4393 SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
4394 : nullptr;
4395
4396 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
4397 dl.getDebugLoc(), VTList, MemVT,
4398 Ops.data(), DynOps, NumOps, MMO,
4399 SuccessOrdering, FailureOrdering,
4400 SynchScope);
4401 CSEMap.InsertNode(N, IP);
4402 InsertNode(N);
4403 return SDValue(N, 0);
4404 }
4405
getAtomic(unsigned Opcode,SDLoc dl,EVT MemVT,SDVTList VTList,ArrayRef<SDValue> Ops,MachineMemOperand * MMO,AtomicOrdering Ordering,SynchronizationScope SynchScope)4406 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4407 SDVTList VTList, ArrayRef<SDValue> Ops,
4408 MachineMemOperand *MMO,
4409 AtomicOrdering Ordering,
4410 SynchronizationScope SynchScope) {
4411 return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
4412 Ordering, SynchScope);
4413 }
4414
getAtomicCmpSwap(unsigned Opcode,SDLoc dl,EVT MemVT,SDVTList VTs,SDValue Chain,SDValue Ptr,SDValue Cmp,SDValue Swp,MachinePointerInfo PtrInfo,unsigned Alignment,AtomicOrdering SuccessOrdering,AtomicOrdering FailureOrdering,SynchronizationScope SynchScope)4415 SDValue SelectionDAG::getAtomicCmpSwap(
4416 unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
4417 SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
4418 unsigned Alignment, AtomicOrdering SuccessOrdering,
4419 AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
4420 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
4421 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
4422 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4423
4424 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4425 Alignment = getEVTAlignment(MemVT);
4426
4427 MachineFunction &MF = getMachineFunction();
4428
4429 // FIXME: Volatile isn't really correct; we should keep track of atomic
4430 // orderings in the memoperand.
4431 unsigned Flags = MachineMemOperand::MOVolatile;
4432 Flags |= MachineMemOperand::MOLoad;
4433 Flags |= MachineMemOperand::MOStore;
4434
4435 MachineMemOperand *MMO =
4436 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
4437
4438 return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
4439 SuccessOrdering, FailureOrdering, SynchScope);
4440 }
4441
getAtomicCmpSwap(unsigned Opcode,SDLoc dl,EVT MemVT,SDVTList VTs,SDValue Chain,SDValue Ptr,SDValue Cmp,SDValue Swp,MachineMemOperand * MMO,AtomicOrdering SuccessOrdering,AtomicOrdering FailureOrdering,SynchronizationScope SynchScope)4442 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
4443 SDVTList VTs, SDValue Chain, SDValue Ptr,
4444 SDValue Cmp, SDValue Swp,
4445 MachineMemOperand *MMO,
4446 AtomicOrdering SuccessOrdering,
4447 AtomicOrdering FailureOrdering,
4448 SynchronizationScope SynchScope) {
4449 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
4450 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
4451 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4452
4453 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4454 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
4455 SuccessOrdering, FailureOrdering, SynchScope);
4456 }
4457
getAtomic(unsigned Opcode,SDLoc dl,EVT MemVT,SDValue Chain,SDValue Ptr,SDValue Val,const Value * PtrVal,unsigned Alignment,AtomicOrdering Ordering,SynchronizationScope SynchScope)4458 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4459 SDValue Chain,
4460 SDValue Ptr, SDValue Val,
4461 const Value* PtrVal,
4462 unsigned Alignment,
4463 AtomicOrdering Ordering,
4464 SynchronizationScope SynchScope) {
4465 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4466 Alignment = getEVTAlignment(MemVT);
4467
4468 MachineFunction &MF = getMachineFunction();
4469 // An atomic store does not load. An atomic load does not store.
4470 // (An atomicrmw obviously both loads and stores.)
4471 // For now, atomics are considered to be volatile always, and they are
4472 // chained as such.
4473 // FIXME: Volatile isn't really correct; we should keep track of atomic
4474 // orderings in the memoperand.
4475 unsigned Flags = MachineMemOperand::MOVolatile;
4476 if (Opcode != ISD::ATOMIC_STORE)
4477 Flags |= MachineMemOperand::MOLoad;
4478 if (Opcode != ISD::ATOMIC_LOAD)
4479 Flags |= MachineMemOperand::MOStore;
4480
4481 MachineMemOperand *MMO =
4482 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
4483 MemVT.getStoreSize(), Alignment);
4484
4485 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
4486 Ordering, SynchScope);
4487 }
4488
getAtomic(unsigned Opcode,SDLoc dl,EVT MemVT,SDValue Chain,SDValue Ptr,SDValue Val,MachineMemOperand * MMO,AtomicOrdering Ordering,SynchronizationScope SynchScope)4489 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4490 SDValue Chain,
4491 SDValue Ptr, SDValue Val,
4492 MachineMemOperand *MMO,
4493 AtomicOrdering Ordering,
4494 SynchronizationScope SynchScope) {
4495 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
4496 Opcode == ISD::ATOMIC_LOAD_SUB ||
4497 Opcode == ISD::ATOMIC_LOAD_AND ||
4498 Opcode == ISD::ATOMIC_LOAD_OR ||
4499 Opcode == ISD::ATOMIC_LOAD_XOR ||
4500 Opcode == ISD::ATOMIC_LOAD_NAND ||
4501 Opcode == ISD::ATOMIC_LOAD_MIN ||
4502 Opcode == ISD::ATOMIC_LOAD_MAX ||
4503 Opcode == ISD::ATOMIC_LOAD_UMIN ||
4504 Opcode == ISD::ATOMIC_LOAD_UMAX ||
4505 Opcode == ISD::ATOMIC_SWAP ||
4506 Opcode == ISD::ATOMIC_STORE) &&
4507 "Invalid Atomic Op");
4508
4509 EVT VT = Val.getValueType();
4510
4511 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
4512 getVTList(VT, MVT::Other);
4513 SDValue Ops[] = {Chain, Ptr, Val};
4514 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
4515 }
4516
getAtomic(unsigned Opcode,SDLoc dl,EVT MemVT,EVT VT,SDValue Chain,SDValue Ptr,MachineMemOperand * MMO,AtomicOrdering Ordering,SynchronizationScope SynchScope)4517 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4518 EVT VT, SDValue Chain,
4519 SDValue Ptr,
4520 MachineMemOperand *MMO,
4521 AtomicOrdering Ordering,
4522 SynchronizationScope SynchScope) {
4523 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
4524
4525 SDVTList VTs = getVTList(VT, MVT::Other);
4526 SDValue Ops[] = {Chain, Ptr};
4527 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
4528 }
4529
4530 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
getMergeValues(ArrayRef<SDValue> Ops,SDLoc dl)4531 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
4532 if (Ops.size() == 1)
4533 return Ops[0];
4534
4535 SmallVector<EVT, 4> VTs;
4536 VTs.reserve(Ops.size());
4537 for (unsigned i = 0; i < Ops.size(); ++i)
4538 VTs.push_back(Ops[i].getValueType());
4539 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
4540 }
4541
4542 SDValue
getMemIntrinsicNode(unsigned Opcode,SDLoc dl,SDVTList VTList,ArrayRef<SDValue> Ops,EVT MemVT,MachinePointerInfo PtrInfo,unsigned Align,bool Vol,bool ReadMem,bool WriteMem,unsigned Size)4543 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
4544 ArrayRef<SDValue> Ops,
4545 EVT MemVT, MachinePointerInfo PtrInfo,
4546 unsigned Align, bool Vol,
4547 bool ReadMem, bool WriteMem, unsigned Size) {
4548 if (Align == 0) // Ensure that codegen never sees alignment 0
4549 Align = getEVTAlignment(MemVT);
4550
4551 MachineFunction &MF = getMachineFunction();
4552 unsigned Flags = 0;
4553 if (WriteMem)
4554 Flags |= MachineMemOperand::MOStore;
4555 if (ReadMem)
4556 Flags |= MachineMemOperand::MOLoad;
4557 if (Vol)
4558 Flags |= MachineMemOperand::MOVolatile;
4559 if (!Size)
4560 Size = MemVT.getStoreSize();
4561 MachineMemOperand *MMO =
4562 MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
4563
4564 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
4565 }
4566
4567 SDValue
getMemIntrinsicNode(unsigned Opcode,SDLoc dl,SDVTList VTList,ArrayRef<SDValue> Ops,EVT MemVT,MachineMemOperand * MMO)4568 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
4569 ArrayRef<SDValue> Ops, EVT MemVT,
4570 MachineMemOperand *MMO) {
4571 assert((Opcode == ISD::INTRINSIC_VOID ||
4572 Opcode == ISD::INTRINSIC_W_CHAIN ||
4573 Opcode == ISD::PREFETCH ||
4574 Opcode == ISD::LIFETIME_START ||
4575 Opcode == ISD::LIFETIME_END ||
4576 (Opcode <= INT_MAX &&
4577 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
4578 "Opcode is not a memory-accessing opcode!");
4579
4580 // Memoize the node unless it returns a flag.
4581 MemIntrinsicSDNode *N;
4582 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
4583 FoldingSetNodeID ID;
4584 AddNodeIDNode(ID, Opcode, VTList, Ops);
4585 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4586 void *IP = nullptr;
4587 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4588 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
4589 return SDValue(E, 0);
4590 }
4591
4592 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
4593 dl.getDebugLoc(), VTList, Ops,
4594 MemVT, MMO);
4595 CSEMap.InsertNode(N, IP);
4596 } else {
4597 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
4598 dl.getDebugLoc(), VTList, Ops,
4599 MemVT, MMO);
4600 }
4601 InsertNode(N);
4602 return SDValue(N, 0);
4603 }
4604
4605 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
4606 /// MachinePointerInfo record from it. This is particularly useful because the
4607 /// code generator has many cases where it doesn't bother passing in a
4608 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
InferPointerInfo(SDValue Ptr,int64_t Offset=0)4609 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
4610 // If this is FI+Offset, we can model it.
4611 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
4612 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
4613
4614 // If this is (FI+Offset1)+Offset2, we can model it.
4615 if (Ptr.getOpcode() != ISD::ADD ||
4616 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
4617 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
4618 return MachinePointerInfo();
4619
4620 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
4621 return MachinePointerInfo::getFixedStack(FI, Offset+
4622 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
4623 }
4624
4625 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
4626 /// MachinePointerInfo record from it. This is particularly useful because the
4627 /// code generator has many cases where it doesn't bother passing in a
4628 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
InferPointerInfo(SDValue Ptr,SDValue OffsetOp)4629 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
4630 // If the 'Offset' value isn't a constant, we can't handle this.
4631 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
4632 return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
4633 if (OffsetOp.getOpcode() == ISD::UNDEF)
4634 return InferPointerInfo(Ptr);
4635 return MachinePointerInfo();
4636 }
4637
4638
4639 SDValue
getLoad(ISD::MemIndexedMode AM,ISD::LoadExtType ExtType,EVT VT,SDLoc dl,SDValue Chain,SDValue Ptr,SDValue Offset,MachinePointerInfo PtrInfo,EVT MemVT,bool isVolatile,bool isNonTemporal,bool isInvariant,unsigned Alignment,const AAMDNodes & AAInfo,const MDNode * Ranges)4640 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
4641 EVT VT, SDLoc dl, SDValue Chain,
4642 SDValue Ptr, SDValue Offset,
4643 MachinePointerInfo PtrInfo, EVT MemVT,
4644 bool isVolatile, bool isNonTemporal, bool isInvariant,
4645 unsigned Alignment, const AAMDNodes &AAInfo,
4646 const MDNode *Ranges) {
4647 assert(Chain.getValueType() == MVT::Other &&
4648 "Invalid chain type");
4649 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4650 Alignment = getEVTAlignment(VT);
4651
4652 unsigned Flags = MachineMemOperand::MOLoad;
4653 if (isVolatile)
4654 Flags |= MachineMemOperand::MOVolatile;
4655 if (isNonTemporal)
4656 Flags |= MachineMemOperand::MONonTemporal;
4657 if (isInvariant)
4658 Flags |= MachineMemOperand::MOInvariant;
4659
4660 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
4661 // clients.
4662 if (PtrInfo.V.isNull())
4663 PtrInfo = InferPointerInfo(Ptr, Offset);
4664
4665 MachineFunction &MF = getMachineFunction();
4666 MachineMemOperand *MMO =
4667 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
4668 AAInfo, Ranges);
4669 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
4670 }
4671
4672 SDValue
getLoad(ISD::MemIndexedMode AM,ISD::LoadExtType ExtType,EVT VT,SDLoc dl,SDValue Chain,SDValue Ptr,SDValue Offset,EVT MemVT,MachineMemOperand * MMO)4673 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
4674 EVT VT, SDLoc dl, SDValue Chain,
4675 SDValue Ptr, SDValue Offset, EVT MemVT,
4676 MachineMemOperand *MMO) {
4677 if (VT == MemVT) {
4678 ExtType = ISD::NON_EXTLOAD;
4679 } else if (ExtType == ISD::NON_EXTLOAD) {
4680 assert(VT == MemVT && "Non-extending load from different memory type!");
4681 } else {
4682 // Extending load.
4683 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
4684 "Should only be an extending load, not truncating!");
4685 assert(VT.isInteger() == MemVT.isInteger() &&
4686 "Cannot convert from FP to Int or Int -> FP!");
4687 assert(VT.isVector() == MemVT.isVector() &&
4688 "Cannot use trunc store to convert to or from a vector!");
4689 assert((!VT.isVector() ||
4690 VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
4691 "Cannot use trunc store to change the number of vector elements!");
4692 }
4693
4694 bool Indexed = AM != ISD::UNINDEXED;
4695 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
4696 "Unindexed load with an offset!");
4697
4698 SDVTList VTs = Indexed ?
4699 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
4700 SDValue Ops[] = { Chain, Ptr, Offset };
4701 FoldingSetNodeID ID;
4702 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
4703 ID.AddInteger(MemVT.getRawBits());
4704 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
4705 MMO->isNonTemporal(),
4706 MMO->isInvariant()));
4707 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4708 void *IP = nullptr;
4709 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4710 cast<LoadSDNode>(E)->refineAlignment(MMO);
4711 return SDValue(E, 0);
4712 }
4713 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
4714 dl.getDebugLoc(), VTs, AM, ExtType,
4715 MemVT, MMO);
4716 CSEMap.InsertNode(N, IP);
4717 InsertNode(N);
4718 return SDValue(N, 0);
4719 }
4720
getLoad(EVT VT,SDLoc dl,SDValue Chain,SDValue Ptr,MachinePointerInfo PtrInfo,bool isVolatile,bool isNonTemporal,bool isInvariant,unsigned Alignment,const AAMDNodes & AAInfo,const MDNode * Ranges)4721 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
4722 SDValue Chain, SDValue Ptr,
4723 MachinePointerInfo PtrInfo,
4724 bool isVolatile, bool isNonTemporal,
4725 bool isInvariant, unsigned Alignment,
4726 const AAMDNodes &AAInfo,
4727 const MDNode *Ranges) {
4728 SDValue Undef = getUNDEF(Ptr.getValueType());
4729 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
4730 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
4731 AAInfo, Ranges);
4732 }
4733
getLoad(EVT VT,SDLoc dl,SDValue Chain,SDValue Ptr,MachineMemOperand * MMO)4734 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
4735 SDValue Chain, SDValue Ptr,
4736 MachineMemOperand *MMO) {
4737 SDValue Undef = getUNDEF(Ptr.getValueType());
4738 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
4739 VT, MMO);
4740 }
4741
getExtLoad(ISD::LoadExtType ExtType,SDLoc dl,EVT VT,SDValue Chain,SDValue Ptr,MachinePointerInfo PtrInfo,EVT MemVT,bool isVolatile,bool isNonTemporal,bool isInvariant,unsigned Alignment,const AAMDNodes & AAInfo)4742 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
4743 SDValue Chain, SDValue Ptr,
4744 MachinePointerInfo PtrInfo, EVT MemVT,
4745 bool isVolatile, bool isNonTemporal,
4746 bool isInvariant, unsigned Alignment,
4747 const AAMDNodes &AAInfo) {
4748 SDValue Undef = getUNDEF(Ptr.getValueType());
4749 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
4750 PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
4751 Alignment, AAInfo);
4752 }
4753
4754
getExtLoad(ISD::LoadExtType ExtType,SDLoc dl,EVT VT,SDValue Chain,SDValue Ptr,EVT MemVT,MachineMemOperand * MMO)4755 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
4756 SDValue Chain, SDValue Ptr, EVT MemVT,
4757 MachineMemOperand *MMO) {
4758 SDValue Undef = getUNDEF(Ptr.getValueType());
4759 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
4760 MemVT, MMO);
4761 }
4762
4763 SDValue
getIndexedLoad(SDValue OrigLoad,SDLoc dl,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)4764 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
4765 SDValue Offset, ISD::MemIndexedMode AM) {
4766 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
4767 assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
4768 "Load is already a indexed load!");
4769 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
4770 LD->getChain(), Base, Offset, LD->getPointerInfo(),
4771 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
4772 false, LD->getAlignment());
4773 }
4774
getStore(SDValue Chain,SDLoc dl,SDValue Val,SDValue Ptr,MachinePointerInfo PtrInfo,bool isVolatile,bool isNonTemporal,unsigned Alignment,const AAMDNodes & AAInfo)4775 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
4776 SDValue Ptr, MachinePointerInfo PtrInfo,
4777 bool isVolatile, bool isNonTemporal,
4778 unsigned Alignment, const AAMDNodes &AAInfo) {
4779 assert(Chain.getValueType() == MVT::Other &&
4780 "Invalid chain type");
4781 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4782 Alignment = getEVTAlignment(Val.getValueType());
4783
4784 unsigned Flags = MachineMemOperand::MOStore;
4785 if (isVolatile)
4786 Flags |= MachineMemOperand::MOVolatile;
4787 if (isNonTemporal)
4788 Flags |= MachineMemOperand::MONonTemporal;
4789
4790 if (PtrInfo.V.isNull())
4791 PtrInfo = InferPointerInfo(Ptr);
4792
4793 MachineFunction &MF = getMachineFunction();
4794 MachineMemOperand *MMO =
4795 MF.getMachineMemOperand(PtrInfo, Flags,
4796 Val.getValueType().getStoreSize(), Alignment,
4797 AAInfo);
4798
4799 return getStore(Chain, dl, Val, Ptr, MMO);
4800 }
4801
getStore(SDValue Chain,SDLoc dl,SDValue Val,SDValue Ptr,MachineMemOperand * MMO)4802 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
4803 SDValue Ptr, MachineMemOperand *MMO) {
4804 assert(Chain.getValueType() == MVT::Other &&
4805 "Invalid chain type");
4806 EVT VT = Val.getValueType();
4807 SDVTList VTs = getVTList(MVT::Other);
4808 SDValue Undef = getUNDEF(Ptr.getValueType());
4809 SDValue Ops[] = { Chain, Val, Ptr, Undef };
4810 FoldingSetNodeID ID;
4811 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
4812 ID.AddInteger(VT.getRawBits());
4813 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
4814 MMO->isNonTemporal(), MMO->isInvariant()));
4815 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4816 void *IP = nullptr;
4817 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4818 cast<StoreSDNode>(E)->refineAlignment(MMO);
4819 return SDValue(E, 0);
4820 }
4821 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
4822 dl.getDebugLoc(), VTs,
4823 ISD::UNINDEXED, false, VT, MMO);
4824 CSEMap.InsertNode(N, IP);
4825 InsertNode(N);
4826 return SDValue(N, 0);
4827 }
4828
getTruncStore(SDValue Chain,SDLoc dl,SDValue Val,SDValue Ptr,MachinePointerInfo PtrInfo,EVT SVT,bool isVolatile,bool isNonTemporal,unsigned Alignment,const AAMDNodes & AAInfo)4829 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
4830 SDValue Ptr, MachinePointerInfo PtrInfo,
4831 EVT SVT,bool isVolatile, bool isNonTemporal,
4832 unsigned Alignment,
4833 const AAMDNodes &AAInfo) {
4834 assert(Chain.getValueType() == MVT::Other &&
4835 "Invalid chain type");
4836 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4837 Alignment = getEVTAlignment(SVT);
4838
4839 unsigned Flags = MachineMemOperand::MOStore;
4840 if (isVolatile)
4841 Flags |= MachineMemOperand::MOVolatile;
4842 if (isNonTemporal)
4843 Flags |= MachineMemOperand::MONonTemporal;
4844
4845 if (PtrInfo.V.isNull())
4846 PtrInfo = InferPointerInfo(Ptr);
4847
4848 MachineFunction &MF = getMachineFunction();
4849 MachineMemOperand *MMO =
4850 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
4851 AAInfo);
4852
4853 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
4854 }
4855
getTruncStore(SDValue Chain,SDLoc dl,SDValue Val,SDValue Ptr,EVT SVT,MachineMemOperand * MMO)4856 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
4857 SDValue Ptr, EVT SVT,
4858 MachineMemOperand *MMO) {
4859 EVT VT = Val.getValueType();
4860
4861 assert(Chain.getValueType() == MVT::Other &&
4862 "Invalid chain type");
4863 if (VT == SVT)
4864 return getStore(Chain, dl, Val, Ptr, MMO);
4865
4866 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
4867 "Should only be a truncating store, not extending!");
4868 assert(VT.isInteger() == SVT.isInteger() &&
4869 "Can't do FP-INT conversion!");
4870 assert(VT.isVector() == SVT.isVector() &&
4871 "Cannot use trunc store to convert to or from a vector!");
4872 assert((!VT.isVector() ||
4873 VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
4874 "Cannot use trunc store to change the number of vector elements!");
4875
4876 SDVTList VTs = getVTList(MVT::Other);
4877 SDValue Undef = getUNDEF(Ptr.getValueType());
4878 SDValue Ops[] = { Chain, Val, Ptr, Undef };
4879 FoldingSetNodeID ID;
4880 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
4881 ID.AddInteger(SVT.getRawBits());
4882 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
4883 MMO->isNonTemporal(), MMO->isInvariant()));
4884 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4885 void *IP = nullptr;
4886 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4887 cast<StoreSDNode>(E)->refineAlignment(MMO);
4888 return SDValue(E, 0);
4889 }
4890 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
4891 dl.getDebugLoc(), VTs,
4892 ISD::UNINDEXED, true, SVT, MMO);
4893 CSEMap.InsertNode(N, IP);
4894 InsertNode(N);
4895 return SDValue(N, 0);
4896 }
4897
4898 SDValue
getIndexedStore(SDValue OrigStore,SDLoc dl,SDValue Base,SDValue Offset,ISD::MemIndexedMode AM)4899 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
4900 SDValue Offset, ISD::MemIndexedMode AM) {
4901 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
4902 assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
4903 "Store is already a indexed store!");
4904 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
4905 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
4906 FoldingSetNodeID ID;
4907 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
4908 ID.AddInteger(ST->getMemoryVT().getRawBits());
4909 ID.AddInteger(ST->getRawSubclassData());
4910 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
4911 void *IP = nullptr;
4912 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
4913 return SDValue(E, 0);
4914
4915 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
4916 dl.getDebugLoc(), VTs, AM,
4917 ST->isTruncatingStore(),
4918 ST->getMemoryVT(),
4919 ST->getMemOperand());
4920 CSEMap.InsertNode(N, IP);
4921 InsertNode(N);
4922 return SDValue(N, 0);
4923 }
4924
4925 SDValue
getMaskedLoad(EVT VT,SDLoc dl,SDValue Chain,SDValue Ptr,SDValue Mask,SDValue Src0,EVT MemVT,MachineMemOperand * MMO,ISD::LoadExtType ExtTy)4926 SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
4927 SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT,
4928 MachineMemOperand *MMO, ISD::LoadExtType ExtTy) {
4929
4930 SDVTList VTs = getVTList(VT, MVT::Other);
4931 SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
4932 FoldingSetNodeID ID;
4933 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
4934 ID.AddInteger(VT.getRawBits());
4935 ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED,
4936 MMO->isVolatile(),
4937 MMO->isNonTemporal(),
4938 MMO->isInvariant()));
4939 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4940 void *IP = nullptr;
4941 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4942 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
4943 return SDValue(E, 0);
4944 }
4945 SDNode *N = new (NodeAllocator) MaskedLoadSDNode(dl.getIROrder(),
4946 dl.getDebugLoc(), Ops, 4, VTs,
4947 ExtTy, MemVT, MMO);
4948 CSEMap.InsertNode(N, IP);
4949 InsertNode(N);
4950 return SDValue(N, 0);
4951 }
4952
getMaskedStore(SDValue Chain,SDLoc dl,SDValue Val,SDValue Ptr,SDValue Mask,EVT MemVT,MachineMemOperand * MMO,bool isTrunc)4953 SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
4954 SDValue Ptr, SDValue Mask, EVT MemVT,
4955 MachineMemOperand *MMO, bool isTrunc) {
4956 assert(Chain.getValueType() == MVT::Other &&
4957 "Invalid chain type");
4958 EVT VT = Val.getValueType();
4959 SDVTList VTs = getVTList(MVT::Other);
4960 SDValue Ops[] = { Chain, Ptr, Mask, Val };
4961 FoldingSetNodeID ID;
4962 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
4963 ID.AddInteger(VT.getRawBits());
4964 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
4965 MMO->isNonTemporal(), MMO->isInvariant()));
4966 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4967 void *IP = nullptr;
4968 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4969 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
4970 return SDValue(E, 0);
4971 }
4972 SDNode *N = new (NodeAllocator) MaskedStoreSDNode(dl.getIROrder(),
4973 dl.getDebugLoc(), Ops, 4,
4974 VTs, isTrunc, MemVT, MMO);
4975 CSEMap.InsertNode(N, IP);
4976 InsertNode(N);
4977 return SDValue(N, 0);
4978 }
4979
getVAArg(EVT VT,SDLoc dl,SDValue Chain,SDValue Ptr,SDValue SV,unsigned Align)4980 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
4981 SDValue Chain, SDValue Ptr,
4982 SDValue SV,
4983 unsigned Align) {
4984 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
4985 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
4986 }
4987
getNode(unsigned Opcode,SDLoc DL,EVT VT,ArrayRef<SDUse> Ops)4988 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
4989 ArrayRef<SDUse> Ops) {
4990 switch (Ops.size()) {
4991 case 0: return getNode(Opcode, DL, VT);
4992 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
4993 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
4994 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
4995 default: break;
4996 }
4997
4998 // Copy from an SDUse array into an SDValue array for use with
4999 // the regular getNode logic.
5000 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
5001 return getNode(Opcode, DL, VT, NewOps);
5002 }
5003
getNode(unsigned Opcode,SDLoc DL,EVT VT,ArrayRef<SDValue> Ops)5004 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
5005 ArrayRef<SDValue> Ops) {
5006 unsigned NumOps = Ops.size();
5007 switch (NumOps) {
5008 case 0: return getNode(Opcode, DL, VT);
5009 case 1: return getNode(Opcode, DL, VT, Ops[0]);
5010 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
5011 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
5012 default: break;
5013 }
5014
5015 switch (Opcode) {
5016 default: break;
5017 case ISD::SELECT_CC: {
5018 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
5019 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
5020 "LHS and RHS of condition must have same type!");
5021 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
5022 "True and False arms of SelectCC must have same type!");
5023 assert(Ops[2].getValueType() == VT &&
5024 "select_cc node must be of same type as true and false value!");
5025 break;
5026 }
5027 case ISD::BR_CC: {
5028 assert(NumOps == 5 && "BR_CC takes 5 operands!");
5029 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
5030 "LHS/RHS of comparison should match types!");
5031 break;
5032 }
5033 }
5034
5035 // Memoize nodes.
5036 SDNode *N;
5037 SDVTList VTs = getVTList(VT);
5038
5039 if (VT != MVT::Glue) {
5040 FoldingSetNodeID ID;
5041 AddNodeIDNode(ID, Opcode, VTs, Ops);
5042 void *IP = nullptr;
5043
5044 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
5045 return SDValue(E, 0);
5046
5047 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5048 VTs, Ops);
5049 CSEMap.InsertNode(N, IP);
5050 } else {
5051 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5052 VTs, Ops);
5053 }
5054
5055 InsertNode(N);
5056 return SDValue(N, 0);
5057 }
5058
getNode(unsigned Opcode,SDLoc DL,ArrayRef<EVT> ResultTys,ArrayRef<SDValue> Ops)5059 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
5060 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
5061 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
5062 }
5063
getNode(unsigned Opcode,SDLoc DL,SDVTList VTList,ArrayRef<SDValue> Ops)5064 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5065 ArrayRef<SDValue> Ops) {
5066 if (VTList.NumVTs == 1)
5067 return getNode(Opcode, DL, VTList.VTs[0], Ops);
5068
5069 #if 0
5070 switch (Opcode) {
5071 // FIXME: figure out how to safely handle things like
5072 // int foo(int x) { return 1 << (x & 255); }
5073 // int bar() { return foo(256); }
5074 case ISD::SRA_PARTS:
5075 case ISD::SRL_PARTS:
5076 case ISD::SHL_PARTS:
5077 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5078 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
5079 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
5080 else if (N3.getOpcode() == ISD::AND)
5081 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
5082 // If the and is only masking out bits that cannot effect the shift,
5083 // eliminate the and.
5084 unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
5085 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
5086 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
5087 }
5088 break;
5089 }
5090 #endif
5091
5092 // Memoize the node unless it returns a flag.
5093 SDNode *N;
5094 unsigned NumOps = Ops.size();
5095 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
5096 FoldingSetNodeID ID;
5097 AddNodeIDNode(ID, Opcode, VTList, Ops);
5098 void *IP = nullptr;
5099 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
5100 return SDValue(E, 0);
5101
5102 if (NumOps == 1) {
5103 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
5104 DL.getDebugLoc(), VTList, Ops[0]);
5105 } else if (NumOps == 2) {
5106 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
5107 DL.getDebugLoc(), VTList, Ops[0],
5108 Ops[1]);
5109 } else if (NumOps == 3) {
5110 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
5111 DL.getDebugLoc(), VTList, Ops[0],
5112 Ops[1], Ops[2]);
5113 } else {
5114 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5115 VTList, Ops);
5116 }
5117 CSEMap.InsertNode(N, IP);
5118 } else {
5119 if (NumOps == 1) {
5120 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
5121 DL.getDebugLoc(), VTList, Ops[0]);
5122 } else if (NumOps == 2) {
5123 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
5124 DL.getDebugLoc(), VTList, Ops[0],
5125 Ops[1]);
5126 } else if (NumOps == 3) {
5127 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
5128 DL.getDebugLoc(), VTList, Ops[0],
5129 Ops[1], Ops[2]);
5130 } else {
5131 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5132 VTList, Ops);
5133 }
5134 }
5135 InsertNode(N);
5136 return SDValue(N, 0);
5137 }
5138
getNode(unsigned Opcode,SDLoc DL,SDVTList VTList)5139 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
5140 return getNode(Opcode, DL, VTList, None);
5141 }
5142
getNode(unsigned Opcode,SDLoc DL,SDVTList VTList,SDValue N1)5143 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5144 SDValue N1) {
5145 SDValue Ops[] = { N1 };
5146 return getNode(Opcode, DL, VTList, Ops);
5147 }
5148
getNode(unsigned Opcode,SDLoc DL,SDVTList VTList,SDValue N1,SDValue N2)5149 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5150 SDValue N1, SDValue N2) {
5151 SDValue Ops[] = { N1, N2 };
5152 return getNode(Opcode, DL, VTList, Ops);
5153 }
5154
getNode(unsigned Opcode,SDLoc DL,SDVTList VTList,SDValue N1,SDValue N2,SDValue N3)5155 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5156 SDValue N1, SDValue N2, SDValue N3) {
5157 SDValue Ops[] = { N1, N2, N3 };
5158 return getNode(Opcode, DL, VTList, Ops);
5159 }
5160
getNode(unsigned Opcode,SDLoc DL,SDVTList VTList,SDValue N1,SDValue N2,SDValue N3,SDValue N4)5161 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5162 SDValue N1, SDValue N2, SDValue N3,
5163 SDValue N4) {
5164 SDValue Ops[] = { N1, N2, N3, N4 };
5165 return getNode(Opcode, DL, VTList, Ops);
5166 }
5167
getNode(unsigned Opcode,SDLoc DL,SDVTList VTList,SDValue N1,SDValue N2,SDValue N3,SDValue N4,SDValue N5)5168 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5169 SDValue N1, SDValue N2, SDValue N3,
5170 SDValue N4, SDValue N5) {
5171 SDValue Ops[] = { N1, N2, N3, N4, N5 };
5172 return getNode(Opcode, DL, VTList, Ops);
5173 }
5174
getVTList(EVT VT)5175 SDVTList SelectionDAG::getVTList(EVT VT) {
5176 return makeVTList(SDNode::getValueTypeList(VT), 1);
5177 }
5178
getVTList(EVT VT1,EVT VT2)5179 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
5180 FoldingSetNodeID ID;
5181 ID.AddInteger(2U);
5182 ID.AddInteger(VT1.getRawBits());
5183 ID.AddInteger(VT2.getRawBits());
5184
5185 void *IP = nullptr;
5186 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5187 if (!Result) {
5188 EVT *Array = Allocator.Allocate<EVT>(2);
5189 Array[0] = VT1;
5190 Array[1] = VT2;
5191 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
5192 VTListMap.InsertNode(Result, IP);
5193 }
5194 return Result->getSDVTList();
5195 }
5196
getVTList(EVT VT1,EVT VT2,EVT VT3)5197 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
5198 FoldingSetNodeID ID;
5199 ID.AddInteger(3U);
5200 ID.AddInteger(VT1.getRawBits());
5201 ID.AddInteger(VT2.getRawBits());
5202 ID.AddInteger(VT3.getRawBits());
5203
5204 void *IP = nullptr;
5205 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5206 if (!Result) {
5207 EVT *Array = Allocator.Allocate<EVT>(3);
5208 Array[0] = VT1;
5209 Array[1] = VT2;
5210 Array[2] = VT3;
5211 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
5212 VTListMap.InsertNode(Result, IP);
5213 }
5214 return Result->getSDVTList();
5215 }
5216
getVTList(EVT VT1,EVT VT2,EVT VT3,EVT VT4)5217 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
5218 FoldingSetNodeID ID;
5219 ID.AddInteger(4U);
5220 ID.AddInteger(VT1.getRawBits());
5221 ID.AddInteger(VT2.getRawBits());
5222 ID.AddInteger(VT3.getRawBits());
5223 ID.AddInteger(VT4.getRawBits());
5224
5225 void *IP = nullptr;
5226 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5227 if (!Result) {
5228 EVT *Array = Allocator.Allocate<EVT>(4);
5229 Array[0] = VT1;
5230 Array[1] = VT2;
5231 Array[2] = VT3;
5232 Array[3] = VT4;
5233 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
5234 VTListMap.InsertNode(Result, IP);
5235 }
5236 return Result->getSDVTList();
5237 }
5238
getVTList(ArrayRef<EVT> VTs)5239 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
5240 unsigned NumVTs = VTs.size();
5241 FoldingSetNodeID ID;
5242 ID.AddInteger(NumVTs);
5243 for (unsigned index = 0; index < NumVTs; index++) {
5244 ID.AddInteger(VTs[index].getRawBits());
5245 }
5246
5247 void *IP = nullptr;
5248 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5249 if (!Result) {
5250 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
5251 std::copy(VTs.begin(), VTs.end(), Array);
5252 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
5253 VTListMap.InsertNode(Result, IP);
5254 }
5255 return Result->getSDVTList();
5256 }
5257
5258
5259 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
5260 /// specified operands. If the resultant node already exists in the DAG,
5261 /// this does not modify the specified node, instead it returns the node that
5262 /// already exists. If the resultant node does not exist in the DAG, the
5263 /// input node is returned. As a degenerate case, if you specify the same
5264 /// input operands as the node already has, the input node is returned.
UpdateNodeOperands(SDNode * N,SDValue Op)5265 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
5266 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
5267
5268 // Check to see if there is no change.
5269 if (Op == N->getOperand(0)) return N;
5270
5271 // See if the modified node already exists.
5272 void *InsertPos = nullptr;
5273 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
5274 return Existing;
5275
5276 // Nope it doesn't. Remove the node from its current place in the maps.
5277 if (InsertPos)
5278 if (!RemoveNodeFromCSEMaps(N))
5279 InsertPos = nullptr;
5280
5281 // Now we update the operands.
5282 N->OperandList[0].set(Op);
5283
5284 // If this gets put into a CSE map, add it.
5285 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5286 return N;
5287 }
5288
UpdateNodeOperands(SDNode * N,SDValue Op1,SDValue Op2)5289 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
5290 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
5291
5292 // Check to see if there is no change.
5293 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
5294 return N; // No operands changed, just return the input node.
5295
5296 // See if the modified node already exists.
5297 void *InsertPos = nullptr;
5298 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
5299 return Existing;
5300
5301 // Nope it doesn't. Remove the node from its current place in the maps.
5302 if (InsertPos)
5303 if (!RemoveNodeFromCSEMaps(N))
5304 InsertPos = nullptr;
5305
5306 // Now we update the operands.
5307 if (N->OperandList[0] != Op1)
5308 N->OperandList[0].set(Op1);
5309 if (N->OperandList[1] != Op2)
5310 N->OperandList[1].set(Op2);
5311
5312 // If this gets put into a CSE map, add it.
5313 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5314 return N;
5315 }
5316
5317 SDNode *SelectionDAG::
UpdateNodeOperands(SDNode * N,SDValue Op1,SDValue Op2,SDValue Op3)5318 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
5319 SDValue Ops[] = { Op1, Op2, Op3 };
5320 return UpdateNodeOperands(N, Ops);
5321 }
5322
5323 SDNode *SelectionDAG::
UpdateNodeOperands(SDNode * N,SDValue Op1,SDValue Op2,SDValue Op3,SDValue Op4)5324 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
5325 SDValue Op3, SDValue Op4) {
5326 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
5327 return UpdateNodeOperands(N, Ops);
5328 }
5329
5330 SDNode *SelectionDAG::
UpdateNodeOperands(SDNode * N,SDValue Op1,SDValue Op2,SDValue Op3,SDValue Op4,SDValue Op5)5331 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
5332 SDValue Op3, SDValue Op4, SDValue Op5) {
5333 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
5334 return UpdateNodeOperands(N, Ops);
5335 }
5336
5337 SDNode *SelectionDAG::
UpdateNodeOperands(SDNode * N,ArrayRef<SDValue> Ops)5338 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
5339 unsigned NumOps = Ops.size();
5340 assert(N->getNumOperands() == NumOps &&
5341 "Update with wrong number of operands");
5342
5343 // Check to see if there is no change.
5344 bool AnyChange = false;
5345 for (unsigned i = 0; i != NumOps; ++i) {
5346 if (Ops[i] != N->getOperand(i)) {
5347 AnyChange = true;
5348 break;
5349 }
5350 }
5351
5352 // No operands changed, just return the input node.
5353 if (!AnyChange) return N;
5354
5355 // See if the modified node already exists.
5356 void *InsertPos = nullptr;
5357 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
5358 return Existing;
5359
5360 // Nope it doesn't. Remove the node from its current place in the maps.
5361 if (InsertPos)
5362 if (!RemoveNodeFromCSEMaps(N))
5363 InsertPos = nullptr;
5364
5365 // Now we update the operands.
5366 for (unsigned i = 0; i != NumOps; ++i)
5367 if (N->OperandList[i] != Ops[i])
5368 N->OperandList[i].set(Ops[i]);
5369
5370 // If this gets put into a CSE map, add it.
5371 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5372 return N;
5373 }
5374
5375 /// DropOperands - Release the operands and set this node to have
5376 /// zero operands.
DropOperands()5377 void SDNode::DropOperands() {
5378 // Unlike the code in MorphNodeTo that does this, we don't need to
5379 // watch for dead nodes here.
5380 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
5381 SDUse &Use = *I++;
5382 Use.set(SDValue());
5383 }
5384 }
5385
5386 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
5387 /// machine opcode.
5388 ///
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT)5389 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5390 EVT VT) {
5391 SDVTList VTs = getVTList(VT);
5392 return SelectNodeTo(N, MachineOpc, VTs, None);
5393 }
5394
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT,SDValue Op1)5395 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5396 EVT VT, SDValue Op1) {
5397 SDVTList VTs = getVTList(VT);
5398 SDValue Ops[] = { Op1 };
5399 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5400 }
5401
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT,SDValue Op1,SDValue Op2)5402 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5403 EVT VT, SDValue Op1,
5404 SDValue Op2) {
5405 SDVTList VTs = getVTList(VT);
5406 SDValue Ops[] = { Op1, Op2 };
5407 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5408 }
5409
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT,SDValue Op1,SDValue Op2,SDValue Op3)5410 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5411 EVT VT, SDValue Op1,
5412 SDValue Op2, SDValue Op3) {
5413 SDVTList VTs = getVTList(VT);
5414 SDValue Ops[] = { Op1, Op2, Op3 };
5415 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5416 }
5417
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT,ArrayRef<SDValue> Ops)5418 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5419 EVT VT, ArrayRef<SDValue> Ops) {
5420 SDVTList VTs = getVTList(VT);
5421 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5422 }
5423
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,ArrayRef<SDValue> Ops)5424 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5425 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
5426 SDVTList VTs = getVTList(VT1, VT2);
5427 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5428 }
5429
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2)5430 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5431 EVT VT1, EVT VT2) {
5432 SDVTList VTs = getVTList(VT1, VT2);
5433 return SelectNodeTo(N, MachineOpc, VTs, None);
5434 }
5435
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,EVT VT3,ArrayRef<SDValue> Ops)5436 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5437 EVT VT1, EVT VT2, EVT VT3,
5438 ArrayRef<SDValue> Ops) {
5439 SDVTList VTs = getVTList(VT1, VT2, VT3);
5440 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5441 }
5442
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,EVT VT3,EVT VT4,ArrayRef<SDValue> Ops)5443 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5444 EVT VT1, EVT VT2, EVT VT3, EVT VT4,
5445 ArrayRef<SDValue> Ops) {
5446 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
5447 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5448 }
5449
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,SDValue Op1)5450 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5451 EVT VT1, EVT VT2,
5452 SDValue Op1) {
5453 SDVTList VTs = getVTList(VT1, VT2);
5454 SDValue Ops[] = { Op1 };
5455 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5456 }
5457
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,SDValue Op1,SDValue Op2)5458 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5459 EVT VT1, EVT VT2,
5460 SDValue Op1, SDValue Op2) {
5461 SDVTList VTs = getVTList(VT1, VT2);
5462 SDValue Ops[] = { Op1, Op2 };
5463 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5464 }
5465
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,SDValue Op1,SDValue Op2,SDValue Op3)5466 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5467 EVT VT1, EVT VT2,
5468 SDValue Op1, SDValue Op2,
5469 SDValue Op3) {
5470 SDVTList VTs = getVTList(VT1, VT2);
5471 SDValue Ops[] = { Op1, Op2, Op3 };
5472 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5473 }
5474
SelectNodeTo(SDNode * N,unsigned MachineOpc,EVT VT1,EVT VT2,EVT VT3,SDValue Op1,SDValue Op2,SDValue Op3)5475 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5476 EVT VT1, EVT VT2, EVT VT3,
5477 SDValue Op1, SDValue Op2,
5478 SDValue Op3) {
5479 SDVTList VTs = getVTList(VT1, VT2, VT3);
5480 SDValue Ops[] = { Op1, Op2, Op3 };
5481 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5482 }
5483
SelectNodeTo(SDNode * N,unsigned MachineOpc,SDVTList VTs,ArrayRef<SDValue> Ops)5484 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5485 SDVTList VTs,ArrayRef<SDValue> Ops) {
5486 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
5487 // Reset the NodeID to -1.
5488 N->setNodeId(-1);
5489 return N;
5490 }
5491
5492 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
5493 /// the line number information on the merged node since it is not possible to
5494 /// preserve the information that operation is associated with multiple lines.
5495 /// This will make the debugger working better at -O0, were there is a higher
5496 /// probability having other instructions associated with that line.
5497 ///
5498 /// For IROrder, we keep the smaller of the two
UpdadeSDLocOnMergedSDNode(SDNode * N,SDLoc OLoc)5499 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
5500 DebugLoc NLoc = N->getDebugLoc();
5501 if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
5502 (OLoc.getDebugLoc() != NLoc)) {
5503 N->setDebugLoc(DebugLoc());
5504 }
5505 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
5506 N->setIROrder(Order);
5507 return N;
5508 }
5509
5510 /// MorphNodeTo - This *mutates* the specified node to have the specified
5511 /// return type, opcode, and operands.
5512 ///
5513 /// Note that MorphNodeTo returns the resultant node. If there is already a
5514 /// node of the specified opcode and operands, it returns that node instead of
5515 /// the current one. Note that the SDLoc need not be the same.
5516 ///
5517 /// Using MorphNodeTo is faster than creating a new node and swapping it in
5518 /// with ReplaceAllUsesWith both because it often avoids allocating a new
5519 /// node, and because it doesn't require CSE recalculation for any of
5520 /// the node's users.
5521 ///
5522 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
5523 /// As a consequence it isn't appropriate to use from within the DAG combiner or
5524 /// the legalizer which maintain worklists that would need to be updated when
5525 /// deleting things.
MorphNodeTo(SDNode * N,unsigned Opc,SDVTList VTs,ArrayRef<SDValue> Ops)5526 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
5527 SDVTList VTs, ArrayRef<SDValue> Ops) {
5528 unsigned NumOps = Ops.size();
5529 // If an identical node already exists, use it.
5530 void *IP = nullptr;
5531 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
5532 FoldingSetNodeID ID;
5533 AddNodeIDNode(ID, Opc, VTs, Ops);
5534 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
5535 return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
5536 }
5537
5538 if (!RemoveNodeFromCSEMaps(N))
5539 IP = nullptr;
5540
5541 // Start the morphing.
5542 N->NodeType = Opc;
5543 N->ValueList = VTs.VTs;
5544 N->NumValues = VTs.NumVTs;
5545
5546 // Clear the operands list, updating used nodes to remove this from their
5547 // use list. Keep track of any operands that become dead as a result.
5548 SmallPtrSet<SDNode*, 16> DeadNodeSet;
5549 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
5550 SDUse &Use = *I++;
5551 SDNode *Used = Use.getNode();
5552 Use.set(SDValue());
5553 if (Used->use_empty())
5554 DeadNodeSet.insert(Used);
5555 }
5556
5557 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
5558 // Initialize the memory references information.
5559 MN->setMemRefs(nullptr, nullptr);
5560 // If NumOps is larger than the # of operands we can have in a
5561 // MachineSDNode, reallocate the operand list.
5562 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
5563 if (MN->OperandsNeedDelete)
5564 delete[] MN->OperandList;
5565 if (NumOps > array_lengthof(MN->LocalOperands))
5566 // We're creating a final node that will live unmorphed for the
5567 // remainder of the current SelectionDAG iteration, so we can allocate
5568 // the operands directly out of a pool with no recycling metadata.
5569 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
5570 Ops.data(), NumOps);
5571 else
5572 MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
5573 MN->OperandsNeedDelete = false;
5574 } else
5575 MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
5576 } else {
5577 // If NumOps is larger than the # of operands we currently have, reallocate
5578 // the operand list.
5579 if (NumOps > N->NumOperands) {
5580 if (N->OperandsNeedDelete)
5581 delete[] N->OperandList;
5582 N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
5583 N->OperandsNeedDelete = true;
5584 } else
5585 N->InitOperands(N->OperandList, Ops.data(), NumOps);
5586 }
5587
5588 // Delete any nodes that are still dead after adding the uses for the
5589 // new operands.
5590 if (!DeadNodeSet.empty()) {
5591 SmallVector<SDNode *, 16> DeadNodes;
5592 for (SDNode *N : DeadNodeSet)
5593 if (N->use_empty())
5594 DeadNodes.push_back(N);
5595 RemoveDeadNodes(DeadNodes);
5596 }
5597
5598 if (IP)
5599 CSEMap.InsertNode(N, IP); // Memoize the new node.
5600 return N;
5601 }
5602
5603
5604 /// getMachineNode - These are used for target selectors to create a new node
5605 /// with specified return type(s), MachineInstr opcode, and operands.
5606 ///
5607 /// Note that getMachineNode returns the resultant node. If there is already a
5608 /// node of the specified opcode and operands, it returns that node instead of
5609 /// the current one.
5610 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT)5611 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
5612 SDVTList VTs = getVTList(VT);
5613 return getMachineNode(Opcode, dl, VTs, None);
5614 }
5615
5616 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT,SDValue Op1)5617 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
5618 SDVTList VTs = getVTList(VT);
5619 SDValue Ops[] = { Op1 };
5620 return getMachineNode(Opcode, dl, VTs, Ops);
5621 }
5622
5623 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT,SDValue Op1,SDValue Op2)5624 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5625 SDValue Op1, SDValue Op2) {
5626 SDVTList VTs = getVTList(VT);
5627 SDValue Ops[] = { Op1, Op2 };
5628 return getMachineNode(Opcode, dl, VTs, Ops);
5629 }
5630
5631 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT,SDValue Op1,SDValue Op2,SDValue Op3)5632 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5633 SDValue Op1, SDValue Op2, SDValue Op3) {
5634 SDVTList VTs = getVTList(VT);
5635 SDValue Ops[] = { Op1, Op2, Op3 };
5636 return getMachineNode(Opcode, dl, VTs, Ops);
5637 }
5638
5639 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT,ArrayRef<SDValue> Ops)5640 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5641 ArrayRef<SDValue> Ops) {
5642 SDVTList VTs = getVTList(VT);
5643 return getMachineNode(Opcode, dl, VTs, Ops);
5644 }
5645
5646 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2)5647 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
5648 SDVTList VTs = getVTList(VT1, VT2);
5649 return getMachineNode(Opcode, dl, VTs, None);
5650 }
5651
5652 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2,SDValue Op1)5653 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5654 EVT VT1, EVT VT2, SDValue Op1) {
5655 SDVTList VTs = getVTList(VT1, VT2);
5656 SDValue Ops[] = { Op1 };
5657 return getMachineNode(Opcode, dl, VTs, Ops);
5658 }
5659
5660 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2,SDValue Op1,SDValue Op2)5661 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5662 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
5663 SDVTList VTs = getVTList(VT1, VT2);
5664 SDValue Ops[] = { Op1, Op2 };
5665 return getMachineNode(Opcode, dl, VTs, Ops);
5666 }
5667
5668 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2,SDValue Op1,SDValue Op2,SDValue Op3)5669 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5670 EVT VT1, EVT VT2, SDValue Op1,
5671 SDValue Op2, SDValue Op3) {
5672 SDVTList VTs = getVTList(VT1, VT2);
5673 SDValue Ops[] = { Op1, Op2, Op3 };
5674 return getMachineNode(Opcode, dl, VTs, Ops);
5675 }
5676
5677 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2,ArrayRef<SDValue> Ops)5678 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5679 EVT VT1, EVT VT2,
5680 ArrayRef<SDValue> Ops) {
5681 SDVTList VTs = getVTList(VT1, VT2);
5682 return getMachineNode(Opcode, dl, VTs, Ops);
5683 }
5684
5685 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2,EVT VT3,SDValue Op1,SDValue Op2)5686 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5687 EVT VT1, EVT VT2, EVT VT3,
5688 SDValue Op1, SDValue Op2) {
5689 SDVTList VTs = getVTList(VT1, VT2, VT3);
5690 SDValue Ops[] = { Op1, Op2 };
5691 return getMachineNode(Opcode, dl, VTs, Ops);
5692 }
5693
5694 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2,EVT VT3,SDValue Op1,SDValue Op2,SDValue Op3)5695 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5696 EVT VT1, EVT VT2, EVT VT3,
5697 SDValue Op1, SDValue Op2, SDValue Op3) {
5698 SDVTList VTs = getVTList(VT1, VT2, VT3);
5699 SDValue Ops[] = { Op1, Op2, Op3 };
5700 return getMachineNode(Opcode, dl, VTs, Ops);
5701 }
5702
5703 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2,EVT VT3,ArrayRef<SDValue> Ops)5704 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5705 EVT VT1, EVT VT2, EVT VT3,
5706 ArrayRef<SDValue> Ops) {
5707 SDVTList VTs = getVTList(VT1, VT2, VT3);
5708 return getMachineNode(Opcode, dl, VTs, Ops);
5709 }
5710
5711 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,EVT VT1,EVT VT2,EVT VT3,EVT VT4,ArrayRef<SDValue> Ops)5712 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
5713 EVT VT2, EVT VT3, EVT VT4,
5714 ArrayRef<SDValue> Ops) {
5715 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
5716 return getMachineNode(Opcode, dl, VTs, Ops);
5717 }
5718
5719 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc dl,ArrayRef<EVT> ResultTys,ArrayRef<SDValue> Ops)5720 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5721 ArrayRef<EVT> ResultTys,
5722 ArrayRef<SDValue> Ops) {
5723 SDVTList VTs = getVTList(ResultTys);
5724 return getMachineNode(Opcode, dl, VTs, Ops);
5725 }
5726
5727 MachineSDNode *
getMachineNode(unsigned Opcode,SDLoc DL,SDVTList VTs,ArrayRef<SDValue> OpsArray)5728 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
5729 ArrayRef<SDValue> OpsArray) {
5730 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
5731 MachineSDNode *N;
5732 void *IP = nullptr;
5733 const SDValue *Ops = OpsArray.data();
5734 unsigned NumOps = OpsArray.size();
5735
5736 if (DoCSE) {
5737 FoldingSetNodeID ID;
5738 AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
5739 IP = nullptr;
5740 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
5741 return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
5742 }
5743 }
5744
5745 // Allocate a new MachineSDNode.
5746 N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
5747 DL.getDebugLoc(), VTs);
5748
5749 // Initialize the operands list.
5750 if (NumOps > array_lengthof(N->LocalOperands))
5751 // We're creating a final node that will live unmorphed for the
5752 // remainder of the current SelectionDAG iteration, so we can allocate
5753 // the operands directly out of a pool with no recycling metadata.
5754 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
5755 Ops, NumOps);
5756 else
5757 N->InitOperands(N->LocalOperands, Ops, NumOps);
5758 N->OperandsNeedDelete = false;
5759
5760 if (DoCSE)
5761 CSEMap.InsertNode(N, IP);
5762
5763 InsertNode(N);
5764 return N;
5765 }
5766
5767 /// getTargetExtractSubreg - A convenience function for creating
5768 /// TargetOpcode::EXTRACT_SUBREG nodes.
5769 SDValue
getTargetExtractSubreg(int SRIdx,SDLoc DL,EVT VT,SDValue Operand)5770 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
5771 SDValue Operand) {
5772 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
5773 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
5774 VT, Operand, SRIdxVal);
5775 return SDValue(Subreg, 0);
5776 }
5777
5778 /// getTargetInsertSubreg - A convenience function for creating
5779 /// TargetOpcode::INSERT_SUBREG nodes.
5780 SDValue
getTargetInsertSubreg(int SRIdx,SDLoc DL,EVT VT,SDValue Operand,SDValue Subreg)5781 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
5782 SDValue Operand, SDValue Subreg) {
5783 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
5784 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
5785 VT, Operand, Subreg, SRIdxVal);
5786 return SDValue(Result, 0);
5787 }
5788
5789 /// getNodeIfExists - Get the specified node if it's already available, or
5790 /// else return NULL.
getNodeIfExists(unsigned Opcode,SDVTList VTList,ArrayRef<SDValue> Ops,bool nuw,bool nsw,bool exact)5791 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
5792 ArrayRef<SDValue> Ops, bool nuw, bool nsw,
5793 bool exact) {
5794 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
5795 FoldingSetNodeID ID;
5796 AddNodeIDNode(ID, Opcode, VTList, Ops);
5797 if (isBinOpWithFlags(Opcode))
5798 AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
5799 void *IP = nullptr;
5800 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
5801 return E;
5802 }
5803 return nullptr;
5804 }
5805
5806 /// getDbgValue - Creates a SDDbgValue node.
5807 ///
5808 /// SDNode
getDbgValue(MDNode * Var,MDNode * Expr,SDNode * N,unsigned R,bool IsIndirect,uint64_t Off,DebugLoc DL,unsigned O)5809 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
5810 unsigned R, bool IsIndirect, uint64_t Off,
5811 DebugLoc DL, unsigned O) {
5812 return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
5813 }
5814
5815 /// Constant
getConstantDbgValue(MDNode * Var,MDNode * Expr,const Value * C,uint64_t Off,DebugLoc DL,unsigned O)5816 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
5817 const Value *C, uint64_t Off,
5818 DebugLoc DL, unsigned O) {
5819 return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
5820 }
5821
5822 /// FrameIndex
getFrameIndexDbgValue(MDNode * Var,MDNode * Expr,unsigned FI,uint64_t Off,DebugLoc DL,unsigned O)5823 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
5824 unsigned FI, uint64_t Off,
5825 DebugLoc DL, unsigned O) {
5826 return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
5827 }
5828
5829 namespace {
5830
5831 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
5832 /// pointed to by a use iterator is deleted, increment the use iterator
5833 /// so that it doesn't dangle.
5834 ///
5835 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
5836 SDNode::use_iterator &UI;
5837 SDNode::use_iterator &UE;
5838
NodeDeleted(SDNode * N,SDNode * E)5839 void NodeDeleted(SDNode *N, SDNode *E) override {
5840 // Increment the iterator as needed.
5841 while (UI != UE && N == *UI)
5842 ++UI;
5843 }
5844
5845 public:
RAUWUpdateListener(SelectionDAG & d,SDNode::use_iterator & ui,SDNode::use_iterator & ue)5846 RAUWUpdateListener(SelectionDAG &d,
5847 SDNode::use_iterator &ui,
5848 SDNode::use_iterator &ue)
5849 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
5850 };
5851
5852 }
5853
5854 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
5855 /// This can cause recursive merging of nodes in the DAG.
5856 ///
5857 /// This version assumes From has a single result value.
5858 ///
ReplaceAllUsesWith(SDValue FromN,SDValue To)5859 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
5860 SDNode *From = FromN.getNode();
5861 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
5862 "Cannot replace with this method!");
5863 assert(From != To.getNode() && "Cannot replace uses of with self");
5864
5865 // Iterate over all the existing uses of From. New uses will be added
5866 // to the beginning of the use list, which we avoid visiting.
5867 // This specifically avoids visiting uses of From that arise while the
5868 // replacement is happening, because any such uses would be the result
5869 // of CSE: If an existing node looks like From after one of its operands
5870 // is replaced by To, we don't want to replace of all its users with To
5871 // too. See PR3018 for more info.
5872 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
5873 RAUWUpdateListener Listener(*this, UI, UE);
5874 while (UI != UE) {
5875 SDNode *User = *UI;
5876
5877 // This node is about to morph, remove its old self from the CSE maps.
5878 RemoveNodeFromCSEMaps(User);
5879
5880 // A user can appear in a use list multiple times, and when this
5881 // happens the uses are usually next to each other in the list.
5882 // To help reduce the number of CSE recomputations, process all
5883 // the uses of this user that we can find this way.
5884 do {
5885 SDUse &Use = UI.getUse();
5886 ++UI;
5887 Use.set(To);
5888 } while (UI != UE && *UI == User);
5889
5890 // Now that we have modified User, add it back to the CSE maps. If it
5891 // already exists there, recursively merge the results together.
5892 AddModifiedNodeToCSEMaps(User);
5893 }
5894
5895 // If we just RAUW'd the root, take note.
5896 if (FromN == getRoot())
5897 setRoot(To);
5898 }
5899
5900 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
5901 /// This can cause recursive merging of nodes in the DAG.
5902 ///
5903 /// This version assumes that for each value of From, there is a
5904 /// corresponding value in To in the same position with the same type.
5905 ///
ReplaceAllUsesWith(SDNode * From,SDNode * To)5906 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
5907 #ifndef NDEBUG
5908 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
5909 assert((!From->hasAnyUseOfValue(i) ||
5910 From->getValueType(i) == To->getValueType(i)) &&
5911 "Cannot use this version of ReplaceAllUsesWith!");
5912 #endif
5913
5914 // Handle the trivial case.
5915 if (From == To)
5916 return;
5917
5918 // Iterate over just the existing users of From. See the comments in
5919 // the ReplaceAllUsesWith above.
5920 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
5921 RAUWUpdateListener Listener(*this, UI, UE);
5922 while (UI != UE) {
5923 SDNode *User = *UI;
5924
5925 // This node is about to morph, remove its old self from the CSE maps.
5926 RemoveNodeFromCSEMaps(User);
5927
5928 // A user can appear in a use list multiple times, and when this
5929 // happens the uses are usually next to each other in the list.
5930 // To help reduce the number of CSE recomputations, process all
5931 // the uses of this user that we can find this way.
5932 do {
5933 SDUse &Use = UI.getUse();
5934 ++UI;
5935 Use.setNode(To);
5936 } while (UI != UE && *UI == User);
5937
5938 // Now that we have modified User, add it back to the CSE maps. If it
5939 // already exists there, recursively merge the results together.
5940 AddModifiedNodeToCSEMaps(User);
5941 }
5942
5943 // If we just RAUW'd the root, take note.
5944 if (From == getRoot().getNode())
5945 setRoot(SDValue(To, getRoot().getResNo()));
5946 }
5947
5948 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
5949 /// This can cause recursive merging of nodes in the DAG.
5950 ///
5951 /// This version can replace From with any result values. To must match the
5952 /// number and types of values returned by From.
ReplaceAllUsesWith(SDNode * From,const SDValue * To)5953 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
5954 if (From->getNumValues() == 1) // Handle the simple case efficiently.
5955 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
5956
5957 // Iterate over just the existing users of From. See the comments in
5958 // the ReplaceAllUsesWith above.
5959 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
5960 RAUWUpdateListener Listener(*this, UI, UE);
5961 while (UI != UE) {
5962 SDNode *User = *UI;
5963
5964 // This node is about to morph, remove its old self from the CSE maps.
5965 RemoveNodeFromCSEMaps(User);
5966
5967 // A user can appear in a use list multiple times, and when this
5968 // happens the uses are usually next to each other in the list.
5969 // To help reduce the number of CSE recomputations, process all
5970 // the uses of this user that we can find this way.
5971 do {
5972 SDUse &Use = UI.getUse();
5973 const SDValue &ToOp = To[Use.getResNo()];
5974 ++UI;
5975 Use.set(ToOp);
5976 } while (UI != UE && *UI == User);
5977
5978 // Now that we have modified User, add it back to the CSE maps. If it
5979 // already exists there, recursively merge the results together.
5980 AddModifiedNodeToCSEMaps(User);
5981 }
5982
5983 // If we just RAUW'd the root, take note.
5984 if (From == getRoot().getNode())
5985 setRoot(SDValue(To[getRoot().getResNo()]));
5986 }
5987
5988 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
5989 /// uses of other values produced by From.getNode() alone. The Deleted
5990 /// vector is handled the same way as for ReplaceAllUsesWith.
ReplaceAllUsesOfValueWith(SDValue From,SDValue To)5991 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
5992 // Handle the really simple, really trivial case efficiently.
5993 if (From == To) return;
5994
5995 // Handle the simple, trivial, case efficiently.
5996 if (From.getNode()->getNumValues() == 1) {
5997 ReplaceAllUsesWith(From, To);
5998 return;
5999 }
6000
6001 // Iterate over just the existing users of From. See the comments in
6002 // the ReplaceAllUsesWith above.
6003 SDNode::use_iterator UI = From.getNode()->use_begin(),
6004 UE = From.getNode()->use_end();
6005 RAUWUpdateListener Listener(*this, UI, UE);
6006 while (UI != UE) {
6007 SDNode *User = *UI;
6008 bool UserRemovedFromCSEMaps = false;
6009
6010 // A user can appear in a use list multiple times, and when this
6011 // happens the uses are usually next to each other in the list.
6012 // To help reduce the number of CSE recomputations, process all
6013 // the uses of this user that we can find this way.
6014 do {
6015 SDUse &Use = UI.getUse();
6016
6017 // Skip uses of different values from the same node.
6018 if (Use.getResNo() != From.getResNo()) {
6019 ++UI;
6020 continue;
6021 }
6022
6023 // If this node hasn't been modified yet, it's still in the CSE maps,
6024 // so remove its old self from the CSE maps.
6025 if (!UserRemovedFromCSEMaps) {
6026 RemoveNodeFromCSEMaps(User);
6027 UserRemovedFromCSEMaps = true;
6028 }
6029
6030 ++UI;
6031 Use.set(To);
6032 } while (UI != UE && *UI == User);
6033
6034 // We are iterating over all uses of the From node, so if a use
6035 // doesn't use the specific value, no changes are made.
6036 if (!UserRemovedFromCSEMaps)
6037 continue;
6038
6039 // Now that we have modified User, add it back to the CSE maps. If it
6040 // already exists there, recursively merge the results together.
6041 AddModifiedNodeToCSEMaps(User);
6042 }
6043
6044 // If we just RAUW'd the root, take note.
6045 if (From == getRoot())
6046 setRoot(To);
6047 }
6048
6049 namespace {
6050 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
6051 /// to record information about a use.
6052 struct UseMemo {
6053 SDNode *User;
6054 unsigned Index;
6055 SDUse *Use;
6056 };
6057
6058 /// operator< - Sort Memos by User.
operator <(const UseMemo & L,const UseMemo & R)6059 bool operator<(const UseMemo &L, const UseMemo &R) {
6060 return (intptr_t)L.User < (intptr_t)R.User;
6061 }
6062 }
6063
6064 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
6065 /// uses of other values produced by From.getNode() alone. The same value
6066 /// may appear in both the From and To list. The Deleted vector is
6067 /// handled the same way as for ReplaceAllUsesWith.
ReplaceAllUsesOfValuesWith(const SDValue * From,const SDValue * To,unsigned Num)6068 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
6069 const SDValue *To,
6070 unsigned Num){
6071 // Handle the simple, trivial case efficiently.
6072 if (Num == 1)
6073 return ReplaceAllUsesOfValueWith(*From, *To);
6074
6075 // Read up all the uses and make records of them. This helps
6076 // processing new uses that are introduced during the
6077 // replacement process.
6078 SmallVector<UseMemo, 4> Uses;
6079 for (unsigned i = 0; i != Num; ++i) {
6080 unsigned FromResNo = From[i].getResNo();
6081 SDNode *FromNode = From[i].getNode();
6082 for (SDNode::use_iterator UI = FromNode->use_begin(),
6083 E = FromNode->use_end(); UI != E; ++UI) {
6084 SDUse &Use = UI.getUse();
6085 if (Use.getResNo() == FromResNo) {
6086 UseMemo Memo = { *UI, i, &Use };
6087 Uses.push_back(Memo);
6088 }
6089 }
6090 }
6091
6092 // Sort the uses, so that all the uses from a given User are together.
6093 std::sort(Uses.begin(), Uses.end());
6094
6095 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
6096 UseIndex != UseIndexEnd; ) {
6097 // We know that this user uses some value of From. If it is the right
6098 // value, update it.
6099 SDNode *User = Uses[UseIndex].User;
6100
6101 // This node is about to morph, remove its old self from the CSE maps.
6102 RemoveNodeFromCSEMaps(User);
6103
6104 // The Uses array is sorted, so all the uses for a given User
6105 // are next to each other in the list.
6106 // To help reduce the number of CSE recomputations, process all
6107 // the uses of this user that we can find this way.
6108 do {
6109 unsigned i = Uses[UseIndex].Index;
6110 SDUse &Use = *Uses[UseIndex].Use;
6111 ++UseIndex;
6112
6113 Use.set(To[i]);
6114 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
6115
6116 // Now that we have modified User, add it back to the CSE maps. If it
6117 // already exists there, recursively merge the results together.
6118 AddModifiedNodeToCSEMaps(User);
6119 }
6120 }
6121
6122 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
6123 /// based on their topological order. It returns the maximum id and a vector
6124 /// of the SDNodes* in assigned order by reference.
AssignTopologicalOrder()6125 unsigned SelectionDAG::AssignTopologicalOrder() {
6126
6127 unsigned DAGSize = 0;
6128
6129 // SortedPos tracks the progress of the algorithm. Nodes before it are
6130 // sorted, nodes after it are unsorted. When the algorithm completes
6131 // it is at the end of the list.
6132 allnodes_iterator SortedPos = allnodes_begin();
6133
6134 // Visit all the nodes. Move nodes with no operands to the front of
6135 // the list immediately. Annotate nodes that do have operands with their
6136 // operand count. Before we do this, the Node Id fields of the nodes
6137 // may contain arbitrary values. After, the Node Id fields for nodes
6138 // before SortedPos will contain the topological sort index, and the
6139 // Node Id fields for nodes At SortedPos and after will contain the
6140 // count of outstanding operands.
6141 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
6142 SDNode *N = I++;
6143 checkForCycles(N, this);
6144 unsigned Degree = N->getNumOperands();
6145 if (Degree == 0) {
6146 // A node with no uses, add it to the result array immediately.
6147 N->setNodeId(DAGSize++);
6148 allnodes_iterator Q = N;
6149 if (Q != SortedPos)
6150 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
6151 assert(SortedPos != AllNodes.end() && "Overran node list");
6152 ++SortedPos;
6153 } else {
6154 // Temporarily use the Node Id as scratch space for the degree count.
6155 N->setNodeId(Degree);
6156 }
6157 }
6158
6159 // Visit all the nodes. As we iterate, move nodes into sorted order,
6160 // such that by the time the end is reached all nodes will be sorted.
6161 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
6162 SDNode *N = I;
6163 checkForCycles(N, this);
6164 // N is in sorted position, so all its uses have one less operand
6165 // that needs to be sorted.
6166 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
6167 UI != UE; ++UI) {
6168 SDNode *P = *UI;
6169 unsigned Degree = P->getNodeId();
6170 assert(Degree != 0 && "Invalid node degree");
6171 --Degree;
6172 if (Degree == 0) {
6173 // All of P's operands are sorted, so P may sorted now.
6174 P->setNodeId(DAGSize++);
6175 if (P != SortedPos)
6176 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
6177 assert(SortedPos != AllNodes.end() && "Overran node list");
6178 ++SortedPos;
6179 } else {
6180 // Update P's outstanding operand count.
6181 P->setNodeId(Degree);
6182 }
6183 }
6184 if (I == SortedPos) {
6185 #ifndef NDEBUG
6186 SDNode *S = ++I;
6187 dbgs() << "Overran sorted position:\n";
6188 S->dumprFull(this); dbgs() << "\n";
6189 dbgs() << "Checking if this is due to cycles\n";
6190 checkForCycles(this, true);
6191 #endif
6192 llvm_unreachable(nullptr);
6193 }
6194 }
6195
6196 assert(SortedPos == AllNodes.end() &&
6197 "Topological sort incomplete!");
6198 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
6199 "First node in topological sort is not the entry token!");
6200 assert(AllNodes.front().getNodeId() == 0 &&
6201 "First node in topological sort has non-zero id!");
6202 assert(AllNodes.front().getNumOperands() == 0 &&
6203 "First node in topological sort has operands!");
6204 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
6205 "Last node in topologic sort has unexpected id!");
6206 assert(AllNodes.back().use_empty() &&
6207 "Last node in topologic sort has users!");
6208 assert(DAGSize == allnodes_size() && "Node count mismatch!");
6209 return DAGSize;
6210 }
6211
6212 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
6213 /// value is produced by SD.
AddDbgValue(SDDbgValue * DB,SDNode * SD,bool isParameter)6214 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
6215 if (SD) {
6216 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
6217 SD->setHasDebugValue(true);
6218 }
6219 DbgInfo->add(DB, SD, isParameter);
6220 }
6221
6222 /// TransferDbgValues - Transfer SDDbgValues.
TransferDbgValues(SDValue From,SDValue To)6223 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
6224 if (From == To || !From.getNode()->getHasDebugValue())
6225 return;
6226 SDNode *FromNode = From.getNode();
6227 SDNode *ToNode = To.getNode();
6228 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
6229 SmallVector<SDDbgValue *, 2> ClonedDVs;
6230 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
6231 I != E; ++I) {
6232 SDDbgValue *Dbg = *I;
6233 if (Dbg->getKind() == SDDbgValue::SDNODE) {
6234 SDDbgValue *Clone =
6235 getDbgValue(Dbg->getVariable(), Dbg->getExpression(), ToNode,
6236 To.getResNo(), Dbg->isIndirect(), Dbg->getOffset(),
6237 Dbg->getDebugLoc(), Dbg->getOrder());
6238 ClonedDVs.push_back(Clone);
6239 }
6240 }
6241 for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
6242 E = ClonedDVs.end(); I != E; ++I)
6243 AddDbgValue(*I, ToNode, false);
6244 }
6245
6246 //===----------------------------------------------------------------------===//
6247 // SDNode Class
6248 //===----------------------------------------------------------------------===//
6249
~HandleSDNode()6250 HandleSDNode::~HandleSDNode() {
6251 DropOperands();
6252 }
6253
GlobalAddressSDNode(unsigned Opc,unsigned Order,DebugLoc DL,const GlobalValue * GA,EVT VT,int64_t o,unsigned char TF)6254 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
6255 DebugLoc DL, const GlobalValue *GA,
6256 EVT VT, int64_t o, unsigned char TF)
6257 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
6258 TheGlobal = GA;
6259 }
6260
AddrSpaceCastSDNode(unsigned Order,DebugLoc dl,EVT VT,SDValue X,unsigned SrcAS,unsigned DestAS)6261 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
6262 SDValue X, unsigned SrcAS,
6263 unsigned DestAS)
6264 : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
6265 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
6266
MemSDNode(unsigned Opc,unsigned Order,DebugLoc dl,SDVTList VTs,EVT memvt,MachineMemOperand * mmo)6267 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
6268 EVT memvt, MachineMemOperand *mmo)
6269 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
6270 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
6271 MMO->isNonTemporal(), MMO->isInvariant());
6272 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
6273 assert(isNonTemporal() == MMO->isNonTemporal() &&
6274 "Non-temporal encoding error!");
6275 // We check here that the size of the memory operand fits within the size of
6276 // the MMO. This is because the MMO might indicate only a possible address
6277 // range instead of specifying the affected memory addresses precisely.
6278 assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
6279 }
6280
MemSDNode(unsigned Opc,unsigned Order,DebugLoc dl,SDVTList VTs,ArrayRef<SDValue> Ops,EVT memvt,MachineMemOperand * mmo)6281 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
6282 ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
6283 : SDNode(Opc, Order, dl, VTs, Ops),
6284 MemoryVT(memvt), MMO(mmo) {
6285 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
6286 MMO->isNonTemporal(), MMO->isInvariant());
6287 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
6288 assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
6289 }
6290
6291 /// Profile - Gather unique data for the node.
6292 ///
Profile(FoldingSetNodeID & ID) const6293 void SDNode::Profile(FoldingSetNodeID &ID) const {
6294 AddNodeIDNode(ID, this);
6295 }
6296
6297 namespace {
6298 struct EVTArray {
6299 std::vector<EVT> VTs;
6300
EVTArray__anon22af3c720311::EVTArray6301 EVTArray() {
6302 VTs.reserve(MVT::LAST_VALUETYPE);
6303 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
6304 VTs.push_back(MVT((MVT::SimpleValueType)i));
6305 }
6306 };
6307 }
6308
6309 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
6310 static ManagedStatic<EVTArray> SimpleVTArray;
6311 static ManagedStatic<sys::SmartMutex<true> > VTMutex;
6312
6313 /// getValueTypeList - Return a pointer to the specified value type.
6314 ///
getValueTypeList(EVT VT)6315 const EVT *SDNode::getValueTypeList(EVT VT) {
6316 if (VT.isExtended()) {
6317 sys::SmartScopedLock<true> Lock(*VTMutex);
6318 return &(*EVTs->insert(VT).first);
6319 } else {
6320 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
6321 "Value type out of range!");
6322 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
6323 }
6324 }
6325
6326 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
6327 /// indicated value. This method ignores uses of other values defined by this
6328 /// operation.
hasNUsesOfValue(unsigned NUses,unsigned Value) const6329 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
6330 assert(Value < getNumValues() && "Bad value!");
6331
6332 // TODO: Only iterate over uses of a given value of the node
6333 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
6334 if (UI.getUse().getResNo() == Value) {
6335 if (NUses == 0)
6336 return false;
6337 --NUses;
6338 }
6339 }
6340
6341 // Found exactly the right number of uses?
6342 return NUses == 0;
6343 }
6344
6345
6346 /// hasAnyUseOfValue - Return true if there are any use of the indicated
6347 /// value. This method ignores uses of other values defined by this operation.
hasAnyUseOfValue(unsigned Value) const6348 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
6349 assert(Value < getNumValues() && "Bad value!");
6350
6351 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
6352 if (UI.getUse().getResNo() == Value)
6353 return true;
6354
6355 return false;
6356 }
6357
6358
6359 /// isOnlyUserOf - Return true if this node is the only use of N.
6360 ///
isOnlyUserOf(SDNode * N) const6361 bool SDNode::isOnlyUserOf(SDNode *N) const {
6362 bool Seen = false;
6363 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
6364 SDNode *User = *I;
6365 if (User == this)
6366 Seen = true;
6367 else
6368 return false;
6369 }
6370
6371 return Seen;
6372 }
6373
6374 /// isOperand - Return true if this node is an operand of N.
6375 ///
isOperandOf(SDNode * N) const6376 bool SDValue::isOperandOf(SDNode *N) const {
6377 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
6378 if (*this == N->getOperand(i))
6379 return true;
6380 return false;
6381 }
6382
isOperandOf(SDNode * N) const6383 bool SDNode::isOperandOf(SDNode *N) const {
6384 for (unsigned i = 0, e = N->NumOperands; i != e; ++i)
6385 if (this == N->OperandList[i].getNode())
6386 return true;
6387 return false;
6388 }
6389
6390 /// reachesChainWithoutSideEffects - Return true if this operand (which must
6391 /// be a chain) reaches the specified operand without crossing any
6392 /// side-effecting instructions on any chain path. In practice, this looks
6393 /// through token factors and non-volatile loads. In order to remain efficient,
6394 /// this only looks a couple of nodes in, it does not do an exhaustive search.
reachesChainWithoutSideEffects(SDValue Dest,unsigned Depth) const6395 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
6396 unsigned Depth) const {
6397 if (*this == Dest) return true;
6398
6399 // Don't search too deeply, we just want to be able to see through
6400 // TokenFactor's etc.
6401 if (Depth == 0) return false;
6402
6403 // If this is a token factor, all inputs to the TF happen in parallel. If any
6404 // of the operands of the TF does not reach dest, then we cannot do the xform.
6405 if (getOpcode() == ISD::TokenFactor) {
6406 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
6407 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1))
6408 return false;
6409 return true;
6410 }
6411
6412 // Loads don't have side effects, look through them.
6413 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
6414 if (!Ld->isVolatile())
6415 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
6416 }
6417 return false;
6418 }
6419
6420 /// hasPredecessor - Return true if N is a predecessor of this node.
6421 /// N is either an operand of this node, or can be reached by recursively
6422 /// traversing up the operands.
6423 /// NOTE: This is an expensive method. Use it carefully.
hasPredecessor(const SDNode * N) const6424 bool SDNode::hasPredecessor(const SDNode *N) const {
6425 SmallPtrSet<const SDNode *, 32> Visited;
6426 SmallVector<const SDNode *, 16> Worklist;
6427 return hasPredecessorHelper(N, Visited, Worklist);
6428 }
6429
6430 bool
hasPredecessorHelper(const SDNode * N,SmallPtrSetImpl<const SDNode * > & Visited,SmallVectorImpl<const SDNode * > & Worklist) const6431 SDNode::hasPredecessorHelper(const SDNode *N,
6432 SmallPtrSetImpl<const SDNode *> &Visited,
6433 SmallVectorImpl<const SDNode *> &Worklist) const {
6434 if (Visited.empty()) {
6435 Worklist.push_back(this);
6436 } else {
6437 // Take a look in the visited set. If we've already encountered this node
6438 // we needn't search further.
6439 if (Visited.count(N))
6440 return true;
6441 }
6442
6443 // Haven't visited N yet. Continue the search.
6444 while (!Worklist.empty()) {
6445 const SDNode *M = Worklist.pop_back_val();
6446 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
6447 SDNode *Op = M->getOperand(i).getNode();
6448 if (Visited.insert(Op).second)
6449 Worklist.push_back(Op);
6450 if (Op == N)
6451 return true;
6452 }
6453 }
6454
6455 return false;
6456 }
6457
getConstantOperandVal(unsigned Num) const6458 uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
6459 assert(Num < NumOperands && "Invalid child # of SDNode!");
6460 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue();
6461 }
6462
UnrollVectorOp(SDNode * N,unsigned ResNE)6463 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
6464 assert(N->getNumValues() == 1 &&
6465 "Can't unroll a vector with multiple results!");
6466
6467 EVT VT = N->getValueType(0);
6468 unsigned NE = VT.getVectorNumElements();
6469 EVT EltVT = VT.getVectorElementType();
6470 SDLoc dl(N);
6471
6472 SmallVector<SDValue, 8> Scalars;
6473 SmallVector<SDValue, 4> Operands(N->getNumOperands());
6474
6475 // If ResNE is 0, fully unroll the vector op.
6476 if (ResNE == 0)
6477 ResNE = NE;
6478 else if (NE > ResNE)
6479 NE = ResNE;
6480
6481 unsigned i;
6482 for (i= 0; i != NE; ++i) {
6483 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
6484 SDValue Operand = N->getOperand(j);
6485 EVT OperandVT = Operand.getValueType();
6486 if (OperandVT.isVector()) {
6487 // A vector operand; extract a single element.
6488 EVT OperandEltVT = OperandVT.getVectorElementType();
6489 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl,
6490 OperandEltVT,
6491 Operand,
6492 getConstant(i, TLI->getVectorIdxTy()));
6493 } else {
6494 // A scalar operand; just use it as is.
6495 Operands[j] = Operand;
6496 }
6497 }
6498
6499 switch (N->getOpcode()) {
6500 default:
6501 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands));
6502 break;
6503 case ISD::VSELECT:
6504 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
6505 break;
6506 case ISD::SHL:
6507 case ISD::SRA:
6508 case ISD::SRL:
6509 case ISD::ROTL:
6510 case ISD::ROTR:
6511 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
6512 getShiftAmountOperand(Operands[0].getValueType(),
6513 Operands[1])));
6514 break;
6515 case ISD::SIGN_EXTEND_INREG:
6516 case ISD::FP_ROUND_INREG: {
6517 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
6518 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
6519 Operands[0],
6520 getValueType(ExtVT)));
6521 }
6522 }
6523 }
6524
6525 for (; i < ResNE; ++i)
6526 Scalars.push_back(getUNDEF(EltVT));
6527
6528 return getNode(ISD::BUILD_VECTOR, dl,
6529 EVT::getVectorVT(*getContext(), EltVT, ResNE), Scalars);
6530 }
6531
6532
6533 /// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a
6534 /// location that is 'Dist' units away from the location that the 'Base' load
6535 /// is loading from.
isConsecutiveLoad(LoadSDNode * LD,LoadSDNode * Base,unsigned Bytes,int Dist) const6536 bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base,
6537 unsigned Bytes, int Dist) const {
6538 if (LD->getChain() != Base->getChain())
6539 return false;
6540 EVT VT = LD->getValueType(0);
6541 if (VT.getSizeInBits() / 8 != Bytes)
6542 return false;
6543
6544 SDValue Loc = LD->getOperand(1);
6545 SDValue BaseLoc = Base->getOperand(1);
6546 if (Loc.getOpcode() == ISD::FrameIndex) {
6547 if (BaseLoc.getOpcode() != ISD::FrameIndex)
6548 return false;
6549 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo();
6550 int FI = cast<FrameIndexSDNode>(Loc)->getIndex();
6551 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
6552 int FS = MFI->getObjectSize(FI);
6553 int BFS = MFI->getObjectSize(BFI);
6554 if (FS != BFS || FS != (int)Bytes) return false;
6555 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes);
6556 }
6557
6558 // Handle X + C.
6559 if (isBaseWithConstantOffset(Loc)) {
6560 int64_t LocOffset = cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue();
6561 if (Loc.getOperand(0) == BaseLoc) {
6562 // If the base location is a simple address with no offset itself, then
6563 // the second load's first add operand should be the base address.
6564 if (LocOffset == Dist * (int)Bytes)
6565 return true;
6566 } else if (isBaseWithConstantOffset(BaseLoc)) {
6567 // The base location itself has an offset, so subtract that value from the
6568 // second load's offset before comparing to distance * size.
6569 int64_t BOffset =
6570 cast<ConstantSDNode>(BaseLoc.getOperand(1))->getSExtValue();
6571 if (Loc.getOperand(0) == BaseLoc.getOperand(0)) {
6572 if ((LocOffset - BOffset) == Dist * (int)Bytes)
6573 return true;
6574 }
6575 }
6576 }
6577 const GlobalValue *GV1 = nullptr;
6578 const GlobalValue *GV2 = nullptr;
6579 int64_t Offset1 = 0;
6580 int64_t Offset2 = 0;
6581 bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1);
6582 bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2);
6583 if (isGA1 && isGA2 && GV1 == GV2)
6584 return Offset1 == (Offset2 + Dist*Bytes);
6585 return false;
6586 }
6587
6588
6589 /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
6590 /// it cannot be inferred.
InferPtrAlignment(SDValue Ptr) const6591 unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
6592 // If this is a GlobalAddress + cst, return the alignment.
6593 const GlobalValue *GV;
6594 int64_t GVOffset = 0;
6595 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
6596 unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType());
6597 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0);
6598 llvm::computeKnownBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne,
6599 TLI->getDataLayout());
6600 unsigned AlignBits = KnownZero.countTrailingOnes();
6601 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
6602 if (Align)
6603 return MinAlign(Align, GVOffset);
6604 }
6605
6606 // If this is a direct reference to a stack slot, use information about the
6607 // stack slot's alignment.
6608 int FrameIdx = 1 << 31;
6609 int64_t FrameOffset = 0;
6610 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
6611 FrameIdx = FI->getIndex();
6612 } else if (isBaseWithConstantOffset(Ptr) &&
6613 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
6614 // Handle FI+Cst
6615 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6616 FrameOffset = Ptr.getConstantOperandVal(1);
6617 }
6618
6619 if (FrameIdx != (1 << 31)) {
6620 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo();
6621 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
6622 FrameOffset);
6623 return FIInfoAlign;
6624 }
6625
6626 return 0;
6627 }
6628
6629 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
6630 /// which is split (or expanded) into two not necessarily identical pieces.
GetSplitDestVTs(const EVT & VT) const6631 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
6632 // Currently all types are split in half.
6633 EVT LoVT, HiVT;
6634 if (!VT.isVector()) {
6635 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
6636 } else {
6637 unsigned NumElements = VT.getVectorNumElements();
6638 assert(!(NumElements & 1) && "Splitting vector, but not in half!");
6639 LoVT = HiVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
6640 NumElements/2);
6641 }
6642 return std::make_pair(LoVT, HiVT);
6643 }
6644
6645 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
6646 /// low/high part.
6647 std::pair<SDValue, SDValue>
SplitVector(const SDValue & N,const SDLoc & DL,const EVT & LoVT,const EVT & HiVT)6648 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
6649 const EVT &HiVT) {
6650 assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
6651 N.getValueType().getVectorNumElements() &&
6652 "More vector elements requested than available!");
6653 SDValue Lo, Hi;
6654 Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
6655 getConstant(0, TLI->getVectorIdxTy()));
6656 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
6657 getConstant(LoVT.getVectorNumElements(), TLI->getVectorIdxTy()));
6658 return std::make_pair(Lo, Hi);
6659 }
6660
ExtractVectorElements(SDValue Op,SmallVectorImpl<SDValue> & Args,unsigned Start,unsigned Count)6661 void SelectionDAG::ExtractVectorElements(SDValue Op,
6662 SmallVectorImpl<SDValue> &Args,
6663 unsigned Start, unsigned Count) {
6664 EVT VT = Op.getValueType();
6665 if (Count == 0)
6666 Count = VT.getVectorNumElements();
6667
6668 EVT EltVT = VT.getVectorElementType();
6669 EVT IdxTy = TLI->getVectorIdxTy();
6670 SDLoc SL(Op);
6671 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
6672 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
6673 Op, getConstant(i, IdxTy)));
6674 }
6675 }
6676
6677 // getAddressSpace - Return the address space this GlobalAddress belongs to.
getAddressSpace() const6678 unsigned GlobalAddressSDNode::getAddressSpace() const {
6679 return getGlobal()->getType()->getAddressSpace();
6680 }
6681
6682
getType() const6683 Type *ConstantPoolSDNode::getType() const {
6684 if (isMachineConstantPoolEntry())
6685 return Val.MachineCPVal->getType();
6686 return Val.ConstVal->getType();
6687 }
6688
isConstantSplat(APInt & SplatValue,APInt & SplatUndef,unsigned & SplatBitSize,bool & HasAnyUndefs,unsigned MinSplatBits,bool isBigEndian) const6689 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue,
6690 APInt &SplatUndef,
6691 unsigned &SplatBitSize,
6692 bool &HasAnyUndefs,
6693 unsigned MinSplatBits,
6694 bool isBigEndian) const {
6695 EVT VT = getValueType(0);
6696 assert(VT.isVector() && "Expected a vector type");
6697 unsigned sz = VT.getSizeInBits();
6698 if (MinSplatBits > sz)
6699 return false;
6700
6701 SplatValue = APInt(sz, 0);
6702 SplatUndef = APInt(sz, 0);
6703
6704 // Get the bits. Bits with undefined values (when the corresponding element
6705 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
6706 // in SplatValue. If any of the values are not constant, give up and return
6707 // false.
6708 unsigned int nOps = getNumOperands();
6709 assert(nOps > 0 && "isConstantSplat has 0-size build vector");
6710 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits();
6711
6712 for (unsigned j = 0; j < nOps; ++j) {
6713 unsigned i = isBigEndian ? nOps-1-j : j;
6714 SDValue OpVal = getOperand(i);
6715 unsigned BitPos = j * EltBitSize;
6716
6717 if (OpVal.getOpcode() == ISD::UNDEF)
6718 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize);
6719 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal))
6720 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize).
6721 zextOrTrunc(sz) << BitPos;
6722 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal))
6723 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos;
6724 else
6725 return false;
6726 }
6727
6728 // The build_vector is all constants or undefs. Find the smallest element
6729 // size that splats the vector.
6730
6731 HasAnyUndefs = (SplatUndef != 0);
6732 while (sz > 8) {
6733
6734 unsigned HalfSize = sz / 2;
6735 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
6736 APInt LowValue = SplatValue.trunc(HalfSize);
6737 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
6738 APInt LowUndef = SplatUndef.trunc(HalfSize);
6739
6740 // If the two halves do not match (ignoring undef bits), stop here.
6741 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
6742 MinSplatBits > HalfSize)
6743 break;
6744
6745 SplatValue = HighValue | LowValue;
6746 SplatUndef = HighUndef & LowUndef;
6747
6748 sz = HalfSize;
6749 }
6750
6751 SplatBitSize = sz;
6752 return true;
6753 }
6754
getSplatValue(BitVector * UndefElements) const6755 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
6756 if (UndefElements) {
6757 UndefElements->clear();
6758 UndefElements->resize(getNumOperands());
6759 }
6760 SDValue Splatted;
6761 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
6762 SDValue Op = getOperand(i);
6763 if (Op.getOpcode() == ISD::UNDEF) {
6764 if (UndefElements)
6765 (*UndefElements)[i] = true;
6766 } else if (!Splatted) {
6767 Splatted = Op;
6768 } else if (Splatted != Op) {
6769 return SDValue();
6770 }
6771 }
6772
6773 if (!Splatted) {
6774 assert(getOperand(0).getOpcode() == ISD::UNDEF &&
6775 "Can only have a splat without a constant for all undefs.");
6776 return getOperand(0);
6777 }
6778
6779 return Splatted;
6780 }
6781
6782 ConstantSDNode *
getConstantSplatNode(BitVector * UndefElements) const6783 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
6784 return dyn_cast_or_null<ConstantSDNode>(
6785 getSplatValue(UndefElements).getNode());
6786 }
6787
6788 ConstantFPSDNode *
getConstantFPSplatNode(BitVector * UndefElements) const6789 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
6790 return dyn_cast_or_null<ConstantFPSDNode>(
6791 getSplatValue(UndefElements).getNode());
6792 }
6793
isConstant() const6794 bool BuildVectorSDNode::isConstant() const {
6795 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
6796 unsigned Opc = getOperand(i).getOpcode();
6797 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
6798 return false;
6799 }
6800 return true;
6801 }
6802
isSplatMask(const int * Mask,EVT VT)6803 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
6804 // Find the first non-undef value in the shuffle mask.
6805 unsigned i, e;
6806 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
6807 /* search */;
6808
6809 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
6810
6811 // Make sure all remaining elements are either undef or the same as the first
6812 // non-undef value.
6813 for (int Idx = Mask[i]; i != e; ++i)
6814 if (Mask[i] >= 0 && Mask[i] != Idx)
6815 return false;
6816 return true;
6817 }
6818
6819 #ifndef NDEBUG
checkForCyclesHelper(const SDNode * N,SmallPtrSetImpl<const SDNode * > & Visited,SmallPtrSetImpl<const SDNode * > & Checked,const llvm::SelectionDAG * DAG)6820 static void checkForCyclesHelper(const SDNode *N,
6821 SmallPtrSetImpl<const SDNode*> &Visited,
6822 SmallPtrSetImpl<const SDNode*> &Checked,
6823 const llvm::SelectionDAG *DAG) {
6824 // If this node has already been checked, don't check it again.
6825 if (Checked.count(N))
6826 return;
6827
6828 // If a node has already been visited on this depth-first walk, reject it as
6829 // a cycle.
6830 if (!Visited.insert(N).second) {
6831 errs() << "Detected cycle in SelectionDAG\n";
6832 dbgs() << "Offending node:\n";
6833 N->dumprFull(DAG); dbgs() << "\n";
6834 abort();
6835 }
6836
6837 for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
6838 checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked, DAG);
6839
6840 Checked.insert(N);
6841 Visited.erase(N);
6842 }
6843 #endif
6844
checkForCycles(const llvm::SDNode * N,const llvm::SelectionDAG * DAG,bool force)6845 void llvm::checkForCycles(const llvm::SDNode *N,
6846 const llvm::SelectionDAG *DAG,
6847 bool force) {
6848 #ifndef NDEBUG
6849 bool check = force;
6850 #ifdef XDEBUG
6851 check = true;
6852 #endif // XDEBUG
6853 if (check) {
6854 assert(N && "Checking nonexistent SDNode");
6855 SmallPtrSet<const SDNode*, 32> visited;
6856 SmallPtrSet<const SDNode*, 32> checked;
6857 checkForCyclesHelper(N, visited, checked, DAG);
6858 }
6859 #endif // !NDEBUG
6860 }
6861
checkForCycles(const llvm::SelectionDAG * DAG,bool force)6862 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
6863 checkForCycles(DAG->getRoot().getNode(), DAG, force);
6864 }
6865