1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::Legalize method.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ADT/APFloat.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/FloatingPointMode.h"
17 #include "llvm/ADT/SetVector.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/Analysis/ConstantFolding.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/CodeGen/ISDOpcodes.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/MachineJumpTableInfo.h"
26 #include "llvm/CodeGen/MachineMemOperand.h"
27 #include "llvm/CodeGen/MachineValueType.h"
28 #include "llvm/CodeGen/RuntimeLibcalls.h"
29 #include "llvm/CodeGen/SelectionDAG.h"
30 #include "llvm/CodeGen/SelectionDAGNodes.h"
31 #include "llvm/CodeGen/TargetFrameLowering.h"
32 #include "llvm/CodeGen/TargetLowering.h"
33 #include "llvm/CodeGen/TargetSubtargetInfo.h"
34 #include "llvm/CodeGen/ValueTypes.h"
35 #include "llvm/IR/CallingConv.h"
36 #include "llvm/IR/Constants.h"
37 #include "llvm/IR/DataLayout.h"
38 #include "llvm/IR/DerivedTypes.h"
39 #include "llvm/IR/Function.h"
40 #include "llvm/IR/Metadata.h"
41 #include "llvm/IR/Type.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/Compiler.h"
44 #include "llvm/Support/Debug.h"
45 #include "llvm/Support/ErrorHandling.h"
46 #include "llvm/Support/MathExtras.h"
47 #include "llvm/Support/raw_ostream.h"
48 #include "llvm/Target/TargetMachine.h"
49 #include "llvm/Target/TargetOptions.h"
50 #include <cassert>
51 #include <cstdint>
52 #include <tuple>
53 #include <utility>
54
55 using namespace llvm;
56
57 #define DEBUG_TYPE "legalizedag"
58
59 namespace {
60
61 /// Keeps track of state when getting the sign of a floating-point value as an
62 /// integer.
63 struct FloatSignAsInt {
64 EVT FloatVT;
65 SDValue Chain;
66 SDValue FloatPtr;
67 SDValue IntPtr;
68 MachinePointerInfo IntPointerInfo;
69 MachinePointerInfo FloatPointerInfo;
70 SDValue IntValue;
71 APInt SignMask;
72 uint8_t SignBit;
73 };
74
75 //===----------------------------------------------------------------------===//
76 /// This takes an arbitrary SelectionDAG as input and
77 /// hacks on it until the target machine can handle it. This involves
78 /// eliminating value sizes the machine cannot handle (promoting small sizes to
79 /// large sizes or splitting up large values into small values) as well as
80 /// eliminating operations the machine cannot handle.
81 ///
82 /// This code also does a small amount of optimization and recognition of idioms
83 /// as part of its processing. For example, if a target does not support a
84 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
85 /// will attempt merge setcc and brc instructions into brcc's.
86 class SelectionDAGLegalize {
87 const TargetMachine &TM;
88 const TargetLowering &TLI;
89 SelectionDAG &DAG;
90
91 /// The set of nodes which have already been legalized. We hold a
92 /// reference to it in order to update as necessary on node deletion.
93 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
94
95 /// A set of all the nodes updated during legalization.
96 SmallSetVector<SDNode *, 16> *UpdatedNodes;
97
getSetCCResultType(EVT VT) const98 EVT getSetCCResultType(EVT VT) const {
99 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
100 }
101
102 // Libcall insertion helpers.
103
104 public:
SelectionDAGLegalize(SelectionDAG & DAG,SmallPtrSetImpl<SDNode * > & LegalizedNodes,SmallSetVector<SDNode *,16> * UpdatedNodes=nullptr)105 SelectionDAGLegalize(SelectionDAG &DAG,
106 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
107 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
108 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
109 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
110
111 /// Legalizes the given operation.
112 void LegalizeOp(SDNode *Node);
113
114 private:
115 SDValue OptimizeFloatStore(StoreSDNode *ST);
116
117 void LegalizeLoadOps(SDNode *Node);
118 void LegalizeStoreOps(SDNode *Node);
119
120 /// Some targets cannot handle a variable
121 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
122 /// is necessary to spill the vector being inserted into to memory, perform
123 /// the insert there, and then read the result back.
124 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
125 const SDLoc &dl);
126 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
127 const SDLoc &dl);
128
129 /// Return a vector shuffle operation which
130 /// performs the same shuffe in terms of order or result bytes, but on a type
131 /// whose vector element type is narrower than the original shuffle type.
132 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
133 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
134 SDValue N1, SDValue N2,
135 ArrayRef<int> Mask) const;
136
137 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
138 TargetLowering::ArgListTy &&Args, bool isSigned);
139 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140
141 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
143 SmallVectorImpl<SDValue> &Results);
144 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
145 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
146 RTLIB::Libcall Call_F128,
147 RTLIB::Libcall Call_PPCF128,
148 SmallVectorImpl<SDValue> &Results);
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 RTLIB::Libcall Call_I8,
151 RTLIB::Libcall Call_I16,
152 RTLIB::Libcall Call_I32,
153 RTLIB::Libcall Call_I64,
154 RTLIB::Libcall Call_I128);
155 void ExpandArgFPLibCall(SDNode *Node,
156 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
157 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
158 RTLIB::Libcall Call_PPCF128,
159 SmallVectorImpl<SDValue> &Results);
160 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162
163 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
164 const SDLoc &dl);
165 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
166 const SDLoc &dl, SDValue ChainIn);
167 SDValue ExpandBUILD_VECTOR(SDNode *Node);
168 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
169 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
170 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
171 SmallVectorImpl<SDValue> &Results);
172 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
173 SDValue Value) const;
174 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
175 SDValue NewIntValue) const;
176 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
177 SDValue ExpandFABS(SDNode *Node) const;
178 SDValue ExpandFNEG(SDNode *Node) const;
179 SDValue expandLdexp(SDNode *Node) const;
180 SDValue expandFrexp(SDNode *Node) const;
181
182 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
183 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
184 SmallVectorImpl<SDValue> &Results);
185 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
186 SmallVectorImpl<SDValue> &Results);
187 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
188
189 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
190
191 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
192 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
193 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
194
195 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
196 SDValue ExpandConstant(ConstantSDNode *CP);
197
198 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
199 bool ExpandNode(SDNode *Node);
200 void ConvertNodeToLibcall(SDNode *Node);
201 void PromoteNode(SDNode *Node);
202
203 public:
204 // Node replacement helpers
205
ReplacedNode(SDNode * N)206 void ReplacedNode(SDNode *N) {
207 LegalizedNodes.erase(N);
208 if (UpdatedNodes)
209 UpdatedNodes->insert(N);
210 }
211
ReplaceNode(SDNode * Old,SDNode * New)212 void ReplaceNode(SDNode *Old, SDNode *New) {
213 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
214 dbgs() << " with: "; New->dump(&DAG));
215
216 assert(Old->getNumValues() == New->getNumValues() &&
217 "Replacing one node with another that produces a different number "
218 "of values!");
219 DAG.ReplaceAllUsesWith(Old, New);
220 if (UpdatedNodes)
221 UpdatedNodes->insert(New);
222 ReplacedNode(Old);
223 }
224
ReplaceNode(SDValue Old,SDValue New)225 void ReplaceNode(SDValue Old, SDValue New) {
226 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
227 dbgs() << " with: "; New->dump(&DAG));
228
229 DAG.ReplaceAllUsesWith(Old, New);
230 if (UpdatedNodes)
231 UpdatedNodes->insert(New.getNode());
232 ReplacedNode(Old.getNode());
233 }
234
ReplaceNode(SDNode * Old,const SDValue * New)235 void ReplaceNode(SDNode *Old, const SDValue *New) {
236 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
237
238 DAG.ReplaceAllUsesWith(Old, New);
239 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
240 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
241 New[i]->dump(&DAG));
242 if (UpdatedNodes)
243 UpdatedNodes->insert(New[i].getNode());
244 }
245 ReplacedNode(Old);
246 }
247
ReplaceNodeWithValue(SDValue Old,SDValue New)248 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
249 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
250 dbgs() << " with: "; New->dump(&DAG));
251
252 DAG.ReplaceAllUsesOfValueWith(Old, New);
253 if (UpdatedNodes)
254 UpdatedNodes->insert(New.getNode());
255 ReplacedNode(Old.getNode());
256 }
257 };
258
259 } // end anonymous namespace
260
261 /// Return a vector shuffle operation which
262 /// performs the same shuffle in terms of order or result bytes, but on a type
263 /// whose vector element type is narrower than the original shuffle type.
264 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
ShuffleWithNarrowerEltType(EVT NVT,EVT VT,const SDLoc & dl,SDValue N1,SDValue N2,ArrayRef<int> Mask) const265 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
266 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
267 ArrayRef<int> Mask) const {
268 unsigned NumMaskElts = VT.getVectorNumElements();
269 unsigned NumDestElts = NVT.getVectorNumElements();
270 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
271
272 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
273
274 if (NumEltsGrowth == 1)
275 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
276
277 SmallVector<int, 8> NewMask;
278 for (unsigned i = 0; i != NumMaskElts; ++i) {
279 int Idx = Mask[i];
280 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
281 if (Idx < 0)
282 NewMask.push_back(-1);
283 else
284 NewMask.push_back(Idx * NumEltsGrowth + j);
285 }
286 }
287 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
288 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
289 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
290 }
291
292 /// Expands the ConstantFP node to an integer constant or
293 /// a load from the constant pool.
294 SDValue
ExpandConstantFP(ConstantFPSDNode * CFP,bool UseCP)295 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
296 bool Extend = false;
297 SDLoc dl(CFP);
298
299 // If a FP immediate is precise when represented as a float and if the
300 // target can do an extending load from float to double, we put it into
301 // the constant pool as a float, even if it's is statically typed as a
302 // double. This shrinks FP constants and canonicalizes them for targets where
303 // an FP extending load is the same cost as a normal load (such as on the x87
304 // fp stack or PPC FP unit).
305 EVT VT = CFP->getValueType(0);
306 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
307 if (!UseCP) {
308 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
309 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
310 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
311 }
312
313 APFloat APF = CFP->getValueAPF();
314 EVT OrigVT = VT;
315 EVT SVT = VT;
316
317 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
318 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
319 if (!APF.isSignaling()) {
320 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
321 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
322 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
323 // Only do this if the target has a native EXTLOAD instruction from
324 // smaller type.
325 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
326 TLI.ShouldShrinkFPConstant(OrigVT)) {
327 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
328 LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(
329 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
330 VT = SVT;
331 Extend = true;
332 }
333 }
334 }
335
336 SDValue CPIdx =
337 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
338 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
339 if (Extend) {
340 SDValue Result = DAG.getExtLoad(
341 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
342 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
343 Alignment);
344 return Result;
345 }
346 SDValue Result = DAG.getLoad(
347 OrigVT, dl, DAG.getEntryNode(), CPIdx,
348 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
349 return Result;
350 }
351
352 /// Expands the Constant node to a load from the constant pool.
ExpandConstant(ConstantSDNode * CP)353 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
354 SDLoc dl(CP);
355 EVT VT = CP->getValueType(0);
356 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
357 TLI.getPointerTy(DAG.getDataLayout()));
358 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
359 SDValue Result = DAG.getLoad(
360 VT, dl, DAG.getEntryNode(), CPIdx,
361 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
362 return Result;
363 }
364
365 /// Some target cannot handle a variable insertion index for the
366 /// INSERT_VECTOR_ELT instruction. In this case, it
367 /// is necessary to spill the vector being inserted into to memory, perform
368 /// the insert there, and then read the result back.
PerformInsertVectorEltInMemory(SDValue Vec,SDValue Val,SDValue Idx,const SDLoc & dl)369 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
370 SDValue Val,
371 SDValue Idx,
372 const SDLoc &dl) {
373 SDValue Tmp1 = Vec;
374 SDValue Tmp2 = Val;
375 SDValue Tmp3 = Idx;
376
377 // If the target doesn't support this, we have to spill the input vector
378 // to a temporary stack slot, update the element, then reload it. This is
379 // badness. We could also load the value into a vector register (either
380 // with a "move to register" or "extload into register" instruction, then
381 // permute it into place, if the idx is a constant and if the idx is
382 // supported by the target.
383 EVT VT = Tmp1.getValueType();
384 EVT EltVT = VT.getVectorElementType();
385 SDValue StackPtr = DAG.CreateStackTemporary(VT);
386
387 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
388
389 // Store the vector.
390 SDValue Ch = DAG.getStore(
391 DAG.getEntryNode(), dl, Tmp1, StackPtr,
392 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
393
394 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
395
396 // Store the scalar value.
397 Ch = DAG.getTruncStore(
398 Ch, dl, Tmp2, StackPtr2,
399 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
400 // Load the updated vector.
401 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
402 DAG.getMachineFunction(), SPFI));
403 }
404
ExpandINSERT_VECTOR_ELT(SDValue Vec,SDValue Val,SDValue Idx,const SDLoc & dl)405 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
406 SDValue Idx,
407 const SDLoc &dl) {
408 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
409 // SCALAR_TO_VECTOR requires that the type of the value being inserted
410 // match the element type of the vector being created, except for
411 // integers in which case the inserted value can be over width.
412 EVT EltVT = Vec.getValueType().getVectorElementType();
413 if (Val.getValueType() == EltVT ||
414 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
415 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
416 Vec.getValueType(), Val);
417
418 unsigned NumElts = Vec.getValueType().getVectorNumElements();
419 // We generate a shuffle of InVec and ScVec, so the shuffle mask
420 // should be 0,1,2,3,4,5... with the appropriate element replaced with
421 // elt 0 of the RHS.
422 SmallVector<int, 8> ShufOps;
423 for (unsigned i = 0; i != NumElts; ++i)
424 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
425
426 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
427 }
428 }
429 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
430 }
431
OptimizeFloatStore(StoreSDNode * ST)432 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
433 if (!ISD::isNormalStore(ST))
434 return SDValue();
435
436 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
437 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
438 // FIXME: move this to the DAG Combiner! Note that we can't regress due
439 // to phase ordering between legalized code and the dag combiner. This
440 // probably means that we need to integrate dag combiner and legalizer
441 // together.
442 // We generally can't do this one for long doubles.
443 SDValue Chain = ST->getChain();
444 SDValue Ptr = ST->getBasePtr();
445 SDValue Value = ST->getValue();
446 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
447 AAMDNodes AAInfo = ST->getAAInfo();
448 SDLoc dl(ST);
449
450 // Don't optimise TargetConstantFP
451 if (Value.getOpcode() == ISD::TargetConstantFP)
452 return SDValue();
453
454 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
455 if (CFP->getValueType(0) == MVT::f32 &&
456 TLI.isTypeLegal(MVT::i32)) {
457 SDValue Con = DAG.getConstant(CFP->getValueAPF().
458 bitcastToAPInt().zextOrTrunc(32),
459 SDLoc(CFP), MVT::i32);
460 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
461 ST->getOriginalAlign(), MMOFlags, AAInfo);
462 }
463
464 if (CFP->getValueType(0) == MVT::f64 &&
465 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
466 // If this target supports 64-bit registers, do a single 64-bit store.
467 if (TLI.isTypeLegal(MVT::i64)) {
468 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
469 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
470 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
471 ST->getOriginalAlign(), MMOFlags, AAInfo);
472 }
473
474 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
475 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
476 // stores. If the target supports neither 32- nor 64-bits, this
477 // xform is certainly not worth it.
478 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
479 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
480 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
481 if (DAG.getDataLayout().isBigEndian())
482 std::swap(Lo, Hi);
483
484 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
485 ST->getOriginalAlign(), MMOFlags, AAInfo);
486 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl);
487 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
488 ST->getPointerInfo().getWithOffset(4),
489 ST->getOriginalAlign(), MMOFlags, AAInfo);
490
491 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
492 }
493 }
494 }
495 return SDValue();
496 }
497
LegalizeStoreOps(SDNode * Node)498 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
499 StoreSDNode *ST = cast<StoreSDNode>(Node);
500 SDValue Chain = ST->getChain();
501 SDValue Ptr = ST->getBasePtr();
502 SDLoc dl(Node);
503
504 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
505 AAMDNodes AAInfo = ST->getAAInfo();
506
507 if (!ST->isTruncatingStore()) {
508 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
509 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
510 ReplaceNode(ST, OptStore);
511 return;
512 }
513
514 SDValue Value = ST->getValue();
515 MVT VT = Value.getSimpleValueType();
516 switch (TLI.getOperationAction(ISD::STORE, VT)) {
517 default: llvm_unreachable("This action is not supported yet!");
518 case TargetLowering::Legal: {
519 // If this is an unaligned store and the target doesn't support it,
520 // expand it.
521 EVT MemVT = ST->getMemoryVT();
522 const DataLayout &DL = DAG.getDataLayout();
523 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
524 *ST->getMemOperand())) {
525 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
526 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
527 ReplaceNode(SDValue(ST, 0), Result);
528 } else
529 LLVM_DEBUG(dbgs() << "Legal store\n");
530 break;
531 }
532 case TargetLowering::Custom: {
533 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
534 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
535 if (Res && Res != SDValue(Node, 0))
536 ReplaceNode(SDValue(Node, 0), Res);
537 return;
538 }
539 case TargetLowering::Promote: {
540 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
541 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
542 "Can only promote stores to same size type");
543 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
544 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
545 ST->getOriginalAlign(), MMOFlags, AAInfo);
546 ReplaceNode(SDValue(Node, 0), Result);
547 break;
548 }
549 }
550 return;
551 }
552
553 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
554 SDValue Value = ST->getValue();
555 EVT StVT = ST->getMemoryVT();
556 TypeSize StWidth = StVT.getSizeInBits();
557 TypeSize StSize = StVT.getStoreSizeInBits();
558 auto &DL = DAG.getDataLayout();
559
560 if (StWidth != StSize) {
561 // Promote to a byte-sized store with upper bits zero if not
562 // storing an integral number of bytes. For example, promote
563 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
564 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue());
565 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
566 SDValue Result =
567 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
568 ST->getOriginalAlign(), MMOFlags, AAInfo);
569 ReplaceNode(SDValue(Node, 0), Result);
570 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedValue())) {
571 // If not storing a power-of-2 number of bits, expand as two stores.
572 assert(!StVT.isVector() && "Unsupported truncstore!");
573 unsigned StWidthBits = StWidth.getFixedValue();
574 unsigned LogStWidth = Log2_32(StWidthBits);
575 assert(LogStWidth < 32);
576 unsigned RoundWidth = 1 << LogStWidth;
577 assert(RoundWidth < StWidthBits);
578 unsigned ExtraWidth = StWidthBits - RoundWidth;
579 assert(ExtraWidth < RoundWidth);
580 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
581 "Store size not an integral number of bytes!");
582 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
583 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
584 SDValue Lo, Hi;
585 unsigned IncrementSize;
586
587 if (DL.isLittleEndian()) {
588 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
589 // Store the bottom RoundWidth bits.
590 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
591 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
592
593 // Store the remaining ExtraWidth bits.
594 IncrementSize = RoundWidth / 8;
595 Ptr =
596 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
597 Hi = DAG.getNode(
598 ISD::SRL, dl, Value.getValueType(), Value,
599 DAG.getConstant(RoundWidth, dl,
600 TLI.getShiftAmountTy(Value.getValueType(), DL)));
601 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
602 ST->getPointerInfo().getWithOffset(IncrementSize),
603 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
604 } else {
605 // Big endian - avoid unaligned stores.
606 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
607 // Store the top RoundWidth bits.
608 Hi = DAG.getNode(
609 ISD::SRL, dl, Value.getValueType(), Value,
610 DAG.getConstant(ExtraWidth, dl,
611 TLI.getShiftAmountTy(Value.getValueType(), DL)));
612 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
613 ST->getOriginalAlign(), MMOFlags, AAInfo);
614
615 // Store the remaining ExtraWidth bits.
616 IncrementSize = RoundWidth / 8;
617 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
618 DAG.getConstant(IncrementSize, dl,
619 Ptr.getValueType()));
620 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
621 ST->getPointerInfo().getWithOffset(IncrementSize),
622 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
623 }
624
625 // The order of the stores doesn't matter.
626 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
627 ReplaceNode(SDValue(Node, 0), Result);
628 } else {
629 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
630 default: llvm_unreachable("This action is not supported yet!");
631 case TargetLowering::Legal: {
632 EVT MemVT = ST->getMemoryVT();
633 // If this is an unaligned store and the target doesn't support it,
634 // expand it.
635 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
636 *ST->getMemOperand())) {
637 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
638 ReplaceNode(SDValue(ST, 0), Result);
639 }
640 break;
641 }
642 case TargetLowering::Custom: {
643 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
644 if (Res && Res != SDValue(Node, 0))
645 ReplaceNode(SDValue(Node, 0), Res);
646 return;
647 }
648 case TargetLowering::Expand:
649 assert(!StVT.isVector() &&
650 "Vector Stores are handled in LegalizeVectorOps");
651
652 SDValue Result;
653
654 // TRUNCSTORE:i16 i32 -> STORE i16
655 if (TLI.isTypeLegal(StVT)) {
656 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
657 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
658 ST->getOriginalAlign(), MMOFlags, AAInfo);
659 } else {
660 // The in-memory type isn't legal. Truncate to the type it would promote
661 // to, and then do a truncstore.
662 Value = DAG.getNode(ISD::TRUNCATE, dl,
663 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
664 Value);
665 Result =
666 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
667 ST->getOriginalAlign(), MMOFlags, AAInfo);
668 }
669
670 ReplaceNode(SDValue(Node, 0), Result);
671 break;
672 }
673 }
674 }
675
LegalizeLoadOps(SDNode * Node)676 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
677 LoadSDNode *LD = cast<LoadSDNode>(Node);
678 SDValue Chain = LD->getChain(); // The chain.
679 SDValue Ptr = LD->getBasePtr(); // The base pointer.
680 SDValue Value; // The value returned by the load op.
681 SDLoc dl(Node);
682
683 ISD::LoadExtType ExtType = LD->getExtensionType();
684 if (ExtType == ISD::NON_EXTLOAD) {
685 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
686 MVT VT = Node->getSimpleValueType(0);
687 SDValue RVal = SDValue(Node, 0);
688 SDValue RChain = SDValue(Node, 1);
689
690 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
691 default: llvm_unreachable("This action is not supported yet!");
692 case TargetLowering::Legal: {
693 EVT MemVT = LD->getMemoryVT();
694 const DataLayout &DL = DAG.getDataLayout();
695 // If this is an unaligned load and the target doesn't support it,
696 // expand it.
697 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
698 *LD->getMemOperand())) {
699 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
700 }
701 break;
702 }
703 case TargetLowering::Custom:
704 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
705 RVal = Res;
706 RChain = Res.getValue(1);
707 }
708 break;
709
710 case TargetLowering::Promote: {
711 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
712 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
713 "Can only promote loads to same size type");
714
715 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
716 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
717 RChain = Res.getValue(1);
718 break;
719 }
720 }
721 if (RChain.getNode() != Node) {
722 assert(RVal.getNode() != Node && "Load must be completely replaced");
723 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
724 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
725 if (UpdatedNodes) {
726 UpdatedNodes->insert(RVal.getNode());
727 UpdatedNodes->insert(RChain.getNode());
728 }
729 ReplacedNode(Node);
730 }
731 return;
732 }
733
734 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
735 EVT SrcVT = LD->getMemoryVT();
736 TypeSize SrcWidth = SrcVT.getSizeInBits();
737 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
738 AAMDNodes AAInfo = LD->getAAInfo();
739
740 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
741 // Some targets pretend to have an i1 loading operation, and actually
742 // load an i8. This trick is correct for ZEXTLOAD because the top 7
743 // bits are guaranteed to be zero; it helps the optimizers understand
744 // that these bits are zero. It is also useful for EXTLOAD, since it
745 // tells the optimizers that those bits are undefined. It would be
746 // nice to have an effective generic way of getting these benefits...
747 // Until such a way is found, don't insist on promoting i1 here.
748 (SrcVT != MVT::i1 ||
749 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
750 TargetLowering::Promote)) {
751 // Promote to a byte-sized load if not loading an integral number of
752 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
753 unsigned NewWidth = SrcVT.getStoreSizeInBits();
754 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
755 SDValue Ch;
756
757 // The extra bits are guaranteed to be zero, since we stored them that
758 // way. A zext load from NVT thus automatically gives zext from SrcVT.
759
760 ISD::LoadExtType NewExtType =
761 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
762
763 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
764 Chain, Ptr, LD->getPointerInfo(), NVT,
765 LD->getOriginalAlign(), MMOFlags, AAInfo);
766
767 Ch = Result.getValue(1); // The chain.
768
769 if (ExtType == ISD::SEXTLOAD)
770 // Having the top bits zero doesn't help when sign extending.
771 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
772 Result.getValueType(),
773 Result, DAG.getValueType(SrcVT));
774 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
775 // All the top bits are guaranteed to be zero - inform the optimizers.
776 Result = DAG.getNode(ISD::AssertZext, dl,
777 Result.getValueType(), Result,
778 DAG.getValueType(SrcVT));
779
780 Value = Result;
781 Chain = Ch;
782 } else if (!isPowerOf2_64(SrcWidth.getKnownMinValue())) {
783 // If not loading a power-of-2 number of bits, expand as two loads.
784 assert(!SrcVT.isVector() && "Unsupported extload!");
785 unsigned SrcWidthBits = SrcWidth.getFixedValue();
786 unsigned LogSrcWidth = Log2_32(SrcWidthBits);
787 assert(LogSrcWidth < 32);
788 unsigned RoundWidth = 1 << LogSrcWidth;
789 assert(RoundWidth < SrcWidthBits);
790 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
791 assert(ExtraWidth < RoundWidth);
792 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
793 "Load size not an integral number of bytes!");
794 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
795 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
796 SDValue Lo, Hi, Ch;
797 unsigned IncrementSize;
798 auto &DL = DAG.getDataLayout();
799
800 if (DL.isLittleEndian()) {
801 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
802 // Load the bottom RoundWidth bits.
803 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
804 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
805 MMOFlags, AAInfo);
806
807 // Load the remaining ExtraWidth bits.
808 IncrementSize = RoundWidth / 8;
809 Ptr =
810 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
811 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
812 LD->getPointerInfo().getWithOffset(IncrementSize),
813 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
814
815 // Build a factor node to remember that this load is independent of
816 // the other one.
817 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
818 Hi.getValue(1));
819
820 // Move the top bits to the right place.
821 Hi = DAG.getNode(
822 ISD::SHL, dl, Hi.getValueType(), Hi,
823 DAG.getConstant(RoundWidth, dl,
824 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
825
826 // Join the hi and lo parts.
827 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
828 } else {
829 // Big endian - avoid unaligned loads.
830 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
831 // Load the top RoundWidth bits.
832 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
833 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
834 MMOFlags, AAInfo);
835
836 // Load the remaining ExtraWidth bits.
837 IncrementSize = RoundWidth / 8;
838 Ptr =
839 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
840 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
841 LD->getPointerInfo().getWithOffset(IncrementSize),
842 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
843
844 // Build a factor node to remember that this load is independent of
845 // the other one.
846 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
847 Hi.getValue(1));
848
849 // Move the top bits to the right place.
850 Hi = DAG.getNode(
851 ISD::SHL, dl, Hi.getValueType(), Hi,
852 DAG.getConstant(ExtraWidth, dl,
853 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
854
855 // Join the hi and lo parts.
856 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
857 }
858
859 Chain = Ch;
860 } else {
861 bool isCustom = false;
862 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
863 SrcVT.getSimpleVT())) {
864 default: llvm_unreachable("This action is not supported yet!");
865 case TargetLowering::Custom:
866 isCustom = true;
867 [[fallthrough]];
868 case TargetLowering::Legal:
869 Value = SDValue(Node, 0);
870 Chain = SDValue(Node, 1);
871
872 if (isCustom) {
873 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
874 Value = Res;
875 Chain = Res.getValue(1);
876 }
877 } else {
878 // If this is an unaligned load and the target doesn't support it,
879 // expand it.
880 EVT MemVT = LD->getMemoryVT();
881 const DataLayout &DL = DAG.getDataLayout();
882 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
883 *LD->getMemOperand())) {
884 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
885 }
886 }
887 break;
888
889 case TargetLowering::Expand: {
890 EVT DestVT = Node->getValueType(0);
891 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
892 // If the source type is not legal, see if there is a legal extload to
893 // an intermediate type that we can then extend further.
894 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
895 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
896 (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
897 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
898 // If we are loading a legal type, this is a non-extload followed by a
899 // full extend.
900 ISD::LoadExtType MidExtType =
901 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
902
903 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
904 SrcVT, LD->getMemOperand());
905 unsigned ExtendOp =
906 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
907 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
908 Chain = Load.getValue(1);
909 break;
910 }
911
912 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
913 // normal undefined upper bits behavior to allow using an in-reg extend
914 // with the illegal FP type, so load as an integer and do the
915 // from-integer conversion.
916 EVT SVT = SrcVT.getScalarType();
917 if (SVT == MVT::f16 || SVT == MVT::bf16) {
918 EVT ISrcVT = SrcVT.changeTypeToInteger();
919 EVT IDestVT = DestVT.changeTypeToInteger();
920 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
921
922 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
923 Ptr, ISrcVT, LD->getMemOperand());
924 Value =
925 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
926 dl, DestVT, Result);
927 Chain = Result.getValue(1);
928 break;
929 }
930 }
931
932 assert(!SrcVT.isVector() &&
933 "Vector Loads are handled in LegalizeVectorOps");
934
935 // FIXME: This does not work for vectors on most targets. Sign-
936 // and zero-extend operations are currently folded into extending
937 // loads, whether they are legal or not, and then we end up here
938 // without any support for legalizing them.
939 assert(ExtType != ISD::EXTLOAD &&
940 "EXTLOAD should always be supported!");
941 // Turn the unsupported load into an EXTLOAD followed by an
942 // explicit zero/sign extend inreg.
943 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
944 Node->getValueType(0),
945 Chain, Ptr, SrcVT,
946 LD->getMemOperand());
947 SDValue ValRes;
948 if (ExtType == ISD::SEXTLOAD)
949 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
950 Result.getValueType(),
951 Result, DAG.getValueType(SrcVT));
952 else
953 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
954 Value = ValRes;
955 Chain = Result.getValue(1);
956 break;
957 }
958 }
959 }
960
961 // Since loads produce two values, make sure to remember that we legalized
962 // both of them.
963 if (Chain.getNode() != Node) {
964 assert(Value.getNode() != Node && "Load must be completely replaced");
965 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
966 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
967 if (UpdatedNodes) {
968 UpdatedNodes->insert(Value.getNode());
969 UpdatedNodes->insert(Chain.getNode());
970 }
971 ReplacedNode(Node);
972 }
973 }
974
975 /// Return a legal replacement for the given operation, with all legal operands.
LegalizeOp(SDNode * Node)976 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
977 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
978
979 // Allow illegal target nodes and illegal registers.
980 if (Node->getOpcode() == ISD::TargetConstant ||
981 Node->getOpcode() == ISD::Register)
982 return;
983
984 #ifndef NDEBUG
985 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
986 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
987 TargetLowering::TypeLegal &&
988 "Unexpected illegal type!");
989
990 for (const SDValue &Op : Node->op_values())
991 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
992 TargetLowering::TypeLegal ||
993 Op.getOpcode() == ISD::TargetConstant ||
994 Op.getOpcode() == ISD::Register) &&
995 "Unexpected illegal type!");
996 #endif
997
998 // Figure out the correct action; the way to query this varies by opcode
999 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1000 bool SimpleFinishLegalizing = true;
1001 switch (Node->getOpcode()) {
1002 case ISD::INTRINSIC_W_CHAIN:
1003 case ISD::INTRINSIC_WO_CHAIN:
1004 case ISD::INTRINSIC_VOID:
1005 case ISD::STACKSAVE:
1006 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1007 break;
1008 case ISD::GET_DYNAMIC_AREA_OFFSET:
1009 Action = TLI.getOperationAction(Node->getOpcode(),
1010 Node->getValueType(0));
1011 break;
1012 case ISD::VAARG:
1013 Action = TLI.getOperationAction(Node->getOpcode(),
1014 Node->getValueType(0));
1015 if (Action != TargetLowering::Promote)
1016 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1017 break;
1018 case ISD::SET_FPENV:
1019 case ISD::SET_FPMODE:
1020 Action = TLI.getOperationAction(Node->getOpcode(),
1021 Node->getOperand(1).getValueType());
1022 break;
1023 case ISD::FP_TO_FP16:
1024 case ISD::FP_TO_BF16:
1025 case ISD::SINT_TO_FP:
1026 case ISD::UINT_TO_FP:
1027 case ISD::EXTRACT_VECTOR_ELT:
1028 case ISD::LROUND:
1029 case ISD::LLROUND:
1030 case ISD::LRINT:
1031 case ISD::LLRINT:
1032 Action = TLI.getOperationAction(Node->getOpcode(),
1033 Node->getOperand(0).getValueType());
1034 break;
1035 case ISD::STRICT_FP_TO_FP16:
1036 case ISD::STRICT_SINT_TO_FP:
1037 case ISD::STRICT_UINT_TO_FP:
1038 case ISD::STRICT_LRINT:
1039 case ISD::STRICT_LLRINT:
1040 case ISD::STRICT_LROUND:
1041 case ISD::STRICT_LLROUND:
1042 // These pseudo-ops are the same as the other STRICT_ ops except
1043 // they are registered with setOperationAction() using the input type
1044 // instead of the output type.
1045 Action = TLI.getOperationAction(Node->getOpcode(),
1046 Node->getOperand(1).getValueType());
1047 break;
1048 case ISD::SIGN_EXTEND_INREG: {
1049 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1050 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1051 break;
1052 }
1053 case ISD::ATOMIC_STORE:
1054 Action = TLI.getOperationAction(Node->getOpcode(),
1055 Node->getOperand(1).getValueType());
1056 break;
1057 case ISD::SELECT_CC:
1058 case ISD::STRICT_FSETCC:
1059 case ISD::STRICT_FSETCCS:
1060 case ISD::SETCC:
1061 case ISD::SETCCCARRY:
1062 case ISD::VP_SETCC:
1063 case ISD::BR_CC: {
1064 unsigned Opc = Node->getOpcode();
1065 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1066 : Opc == ISD::STRICT_FSETCC ? 3
1067 : Opc == ISD::STRICT_FSETCCS ? 3
1068 : Opc == ISD::SETCCCARRY ? 3
1069 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1070 : 1;
1071 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1072 : Opc == ISD::STRICT_FSETCC ? 1
1073 : Opc == ISD::STRICT_FSETCCS ? 1
1074 : 0;
1075 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1076 ISD::CondCode CCCode =
1077 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1078 Action = TLI.getCondCodeAction(CCCode, OpVT);
1079 if (Action == TargetLowering::Legal) {
1080 if (Node->getOpcode() == ISD::SELECT_CC)
1081 Action = TLI.getOperationAction(Node->getOpcode(),
1082 Node->getValueType(0));
1083 else
1084 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1085 }
1086 break;
1087 }
1088 case ISD::LOAD:
1089 case ISD::STORE:
1090 // FIXME: Model these properly. LOAD and STORE are complicated, and
1091 // STORE expects the unlegalized operand in some cases.
1092 SimpleFinishLegalizing = false;
1093 break;
1094 case ISD::CALLSEQ_START:
1095 case ISD::CALLSEQ_END:
1096 // FIXME: This shouldn't be necessary. These nodes have special properties
1097 // dealing with the recursive nature of legalization. Removing this
1098 // special case should be done as part of making LegalizeDAG non-recursive.
1099 SimpleFinishLegalizing = false;
1100 break;
1101 case ISD::EXTRACT_ELEMENT:
1102 case ISD::GET_ROUNDING:
1103 case ISD::MERGE_VALUES:
1104 case ISD::EH_RETURN:
1105 case ISD::FRAME_TO_ARGS_OFFSET:
1106 case ISD::EH_DWARF_CFA:
1107 case ISD::EH_SJLJ_SETJMP:
1108 case ISD::EH_SJLJ_LONGJMP:
1109 case ISD::EH_SJLJ_SETUP_DISPATCH:
1110 // These operations lie about being legal: when they claim to be legal,
1111 // they should actually be expanded.
1112 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1113 if (Action == TargetLowering::Legal)
1114 Action = TargetLowering::Expand;
1115 break;
1116 case ISD::INIT_TRAMPOLINE:
1117 case ISD::ADJUST_TRAMPOLINE:
1118 case ISD::FRAMEADDR:
1119 case ISD::RETURNADDR:
1120 case ISD::ADDROFRETURNADDR:
1121 case ISD::SPONENTRY:
1122 // These operations lie about being legal: when they claim to be legal,
1123 // they should actually be custom-lowered.
1124 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1125 if (Action == TargetLowering::Legal)
1126 Action = TargetLowering::Custom;
1127 break;
1128 case ISD::READCYCLECOUNTER:
1129 // READCYCLECOUNTER returns an i64, even if type legalization might have
1130 // expanded that to several smaller types.
1131 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1132 break;
1133 case ISD::READ_REGISTER:
1134 case ISD::WRITE_REGISTER:
1135 // Named register is legal in the DAG, but blocked by register name
1136 // selection if not implemented by target (to chose the correct register)
1137 // They'll be converted to Copy(To/From)Reg.
1138 Action = TargetLowering::Legal;
1139 break;
1140 case ISD::UBSANTRAP:
1141 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1142 if (Action == TargetLowering::Expand) {
1143 // replace ISD::UBSANTRAP with ISD::TRAP
1144 SDValue NewVal;
1145 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1146 Node->getOperand(0));
1147 ReplaceNode(Node, NewVal.getNode());
1148 LegalizeOp(NewVal.getNode());
1149 return;
1150 }
1151 break;
1152 case ISD::DEBUGTRAP:
1153 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1154 if (Action == TargetLowering::Expand) {
1155 // replace ISD::DEBUGTRAP with ISD::TRAP
1156 SDValue NewVal;
1157 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1158 Node->getOperand(0));
1159 ReplaceNode(Node, NewVal.getNode());
1160 LegalizeOp(NewVal.getNode());
1161 return;
1162 }
1163 break;
1164 case ISD::SADDSAT:
1165 case ISD::UADDSAT:
1166 case ISD::SSUBSAT:
1167 case ISD::USUBSAT:
1168 case ISD::SSHLSAT:
1169 case ISD::USHLSAT:
1170 case ISD::FP_TO_SINT_SAT:
1171 case ISD::FP_TO_UINT_SAT:
1172 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1173 break;
1174 case ISD::SMULFIX:
1175 case ISD::SMULFIXSAT:
1176 case ISD::UMULFIX:
1177 case ISD::UMULFIXSAT:
1178 case ISD::SDIVFIX:
1179 case ISD::SDIVFIXSAT:
1180 case ISD::UDIVFIX:
1181 case ISD::UDIVFIXSAT: {
1182 unsigned Scale = Node->getConstantOperandVal(2);
1183 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1184 Node->getValueType(0), Scale);
1185 break;
1186 }
1187 case ISD::MSCATTER:
1188 Action = TLI.getOperationAction(Node->getOpcode(),
1189 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1190 break;
1191 case ISD::MSTORE:
1192 Action = TLI.getOperationAction(Node->getOpcode(),
1193 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1194 break;
1195 case ISD::VP_SCATTER:
1196 Action = TLI.getOperationAction(
1197 Node->getOpcode(),
1198 cast<VPScatterSDNode>(Node)->getValue().getValueType());
1199 break;
1200 case ISD::VP_STORE:
1201 Action = TLI.getOperationAction(
1202 Node->getOpcode(),
1203 cast<VPStoreSDNode>(Node)->getValue().getValueType());
1204 break;
1205 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1206 Action = TLI.getOperationAction(
1207 Node->getOpcode(),
1208 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1209 break;
1210 case ISD::VECREDUCE_FADD:
1211 case ISD::VECREDUCE_FMUL:
1212 case ISD::VECREDUCE_ADD:
1213 case ISD::VECREDUCE_MUL:
1214 case ISD::VECREDUCE_AND:
1215 case ISD::VECREDUCE_OR:
1216 case ISD::VECREDUCE_XOR:
1217 case ISD::VECREDUCE_SMAX:
1218 case ISD::VECREDUCE_SMIN:
1219 case ISD::VECREDUCE_UMAX:
1220 case ISD::VECREDUCE_UMIN:
1221 case ISD::VECREDUCE_FMAX:
1222 case ISD::VECREDUCE_FMIN:
1223 case ISD::VECREDUCE_FMAXIMUM:
1224 case ISD::VECREDUCE_FMINIMUM:
1225 case ISD::IS_FPCLASS:
1226 Action = TLI.getOperationAction(
1227 Node->getOpcode(), Node->getOperand(0).getValueType());
1228 break;
1229 case ISD::VECREDUCE_SEQ_FADD:
1230 case ISD::VECREDUCE_SEQ_FMUL:
1231 case ISD::VP_REDUCE_FADD:
1232 case ISD::VP_REDUCE_FMUL:
1233 case ISD::VP_REDUCE_ADD:
1234 case ISD::VP_REDUCE_MUL:
1235 case ISD::VP_REDUCE_AND:
1236 case ISD::VP_REDUCE_OR:
1237 case ISD::VP_REDUCE_XOR:
1238 case ISD::VP_REDUCE_SMAX:
1239 case ISD::VP_REDUCE_SMIN:
1240 case ISD::VP_REDUCE_UMAX:
1241 case ISD::VP_REDUCE_UMIN:
1242 case ISD::VP_REDUCE_FMAX:
1243 case ISD::VP_REDUCE_FMIN:
1244 case ISD::VP_REDUCE_SEQ_FADD:
1245 case ISD::VP_REDUCE_SEQ_FMUL:
1246 Action = TLI.getOperationAction(
1247 Node->getOpcode(), Node->getOperand(1).getValueType());
1248 break;
1249 default:
1250 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1251 Action = TLI.getCustomOperationAction(*Node);
1252 } else {
1253 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1254 }
1255 break;
1256 }
1257
1258 if (SimpleFinishLegalizing) {
1259 SDNode *NewNode = Node;
1260 switch (Node->getOpcode()) {
1261 default: break;
1262 case ISD::SHL:
1263 case ISD::SRL:
1264 case ISD::SRA:
1265 case ISD::ROTL:
1266 case ISD::ROTR: {
1267 // Legalizing shifts/rotates requires adjusting the shift amount
1268 // to the appropriate width.
1269 SDValue Op0 = Node->getOperand(0);
1270 SDValue Op1 = Node->getOperand(1);
1271 if (!Op1.getValueType().isVector()) {
1272 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1273 // The getShiftAmountOperand() may create a new operand node or
1274 // return the existing one. If new operand is created we need
1275 // to update the parent node.
1276 // Do not try to legalize SAO here! It will be automatically legalized
1277 // in the next round.
1278 if (SAO != Op1)
1279 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1280 }
1281 }
1282 break;
1283 case ISD::FSHL:
1284 case ISD::FSHR:
1285 case ISD::SRL_PARTS:
1286 case ISD::SRA_PARTS:
1287 case ISD::SHL_PARTS: {
1288 // Legalizing shifts/rotates requires adjusting the shift amount
1289 // to the appropriate width.
1290 SDValue Op0 = Node->getOperand(0);
1291 SDValue Op1 = Node->getOperand(1);
1292 SDValue Op2 = Node->getOperand(2);
1293 if (!Op2.getValueType().isVector()) {
1294 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1295 // The getShiftAmountOperand() may create a new operand node or
1296 // return the existing one. If new operand is created we need
1297 // to update the parent node.
1298 if (SAO != Op2)
1299 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1300 }
1301 break;
1302 }
1303 }
1304
1305 if (NewNode != Node) {
1306 ReplaceNode(Node, NewNode);
1307 Node = NewNode;
1308 }
1309 switch (Action) {
1310 case TargetLowering::Legal:
1311 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1312 return;
1313 case TargetLowering::Custom:
1314 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1315 // FIXME: The handling for custom lowering with multiple results is
1316 // a complete mess.
1317 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1318 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1319 return;
1320
1321 if (Node->getNumValues() == 1) {
1322 // Verify the new types match the original. Glue is waived because
1323 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1324 assert((Res.getValueType() == Node->getValueType(0) ||
1325 Node->getValueType(0) == MVT::Glue) &&
1326 "Type mismatch for custom legalized operation");
1327 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1328 // We can just directly replace this node with the lowered value.
1329 ReplaceNode(SDValue(Node, 0), Res);
1330 return;
1331 }
1332
1333 SmallVector<SDValue, 8> ResultVals;
1334 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1335 // Verify the new types match the original. Glue is waived because
1336 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1337 assert((Res->getValueType(i) == Node->getValueType(i) ||
1338 Node->getValueType(i) == MVT::Glue) &&
1339 "Type mismatch for custom legalized operation");
1340 ResultVals.push_back(Res.getValue(i));
1341 }
1342 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1343 ReplaceNode(Node, ResultVals.data());
1344 return;
1345 }
1346 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1347 [[fallthrough]];
1348 case TargetLowering::Expand:
1349 if (ExpandNode(Node))
1350 return;
1351 [[fallthrough]];
1352 case TargetLowering::LibCall:
1353 ConvertNodeToLibcall(Node);
1354 return;
1355 case TargetLowering::Promote:
1356 PromoteNode(Node);
1357 return;
1358 }
1359 }
1360
1361 switch (Node->getOpcode()) {
1362 default:
1363 #ifndef NDEBUG
1364 dbgs() << "NODE: ";
1365 Node->dump( &DAG);
1366 dbgs() << "\n";
1367 #endif
1368 llvm_unreachable("Do not know how to legalize this operator!");
1369
1370 case ISD::CALLSEQ_START:
1371 case ISD::CALLSEQ_END:
1372 break;
1373 case ISD::LOAD:
1374 return LegalizeLoadOps(Node);
1375 case ISD::STORE:
1376 return LegalizeStoreOps(Node);
1377 }
1378 }
1379
ExpandExtractFromVectorThroughStack(SDValue Op)1380 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1381 SDValue Vec = Op.getOperand(0);
1382 SDValue Idx = Op.getOperand(1);
1383 SDLoc dl(Op);
1384
1385 // Before we generate a new store to a temporary stack slot, see if there is
1386 // already one that we can use. There often is because when we scalarize
1387 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1388 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1389 // the vector. If all are expanded here, we don't want one store per vector
1390 // element.
1391
1392 // Caches for hasPredecessorHelper
1393 SmallPtrSet<const SDNode *, 32> Visited;
1394 SmallVector<const SDNode *, 16> Worklist;
1395 Visited.insert(Op.getNode());
1396 Worklist.push_back(Idx.getNode());
1397 SDValue StackPtr, Ch;
1398 for (SDNode *User : Vec.getNode()->uses()) {
1399 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1400 if (ST->isIndexed() || ST->isTruncatingStore() ||
1401 ST->getValue() != Vec)
1402 continue;
1403
1404 // Make sure that nothing else could have stored into the destination of
1405 // this store.
1406 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1407 continue;
1408
1409 // If the index is dependent on the store we will introduce a cycle when
1410 // creating the load (the load uses the index, and by replacing the chain
1411 // we will make the index dependent on the load). Also, the store might be
1412 // dependent on the extractelement and introduce a cycle when creating
1413 // the load.
1414 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1415 ST->hasPredecessor(Op.getNode()))
1416 continue;
1417
1418 StackPtr = ST->getBasePtr();
1419 Ch = SDValue(ST, 0);
1420 break;
1421 }
1422 }
1423
1424 EVT VecVT = Vec.getValueType();
1425
1426 if (!Ch.getNode()) {
1427 // Store the value to a temporary stack slot, then LOAD the returned part.
1428 StackPtr = DAG.CreateStackTemporary(VecVT);
1429 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1430 MachinePointerInfo());
1431 }
1432
1433 SDValue NewLoad;
1434 Align ElementAlignment =
1435 std::min(cast<StoreSDNode>(Ch)->getAlign(),
1436 DAG.getDataLayout().getPrefTypeAlign(
1437 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1438
1439 if (Op.getValueType().isVector()) {
1440 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1441 Op.getValueType(), Idx);
1442 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1443 MachinePointerInfo(), ElementAlignment);
1444 } else {
1445 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1446 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1447 MachinePointerInfo(), VecVT.getVectorElementType(),
1448 ElementAlignment);
1449 }
1450
1451 // Replace the chain going out of the store, by the one out of the load.
1452 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1453
1454 // We introduced a cycle though, so update the loads operands, making sure
1455 // to use the original store's chain as an incoming chain.
1456 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1457 NewLoad->op_end());
1458 NewLoadOperands[0] = Ch;
1459 NewLoad =
1460 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1461 return NewLoad;
1462 }
1463
ExpandInsertToVectorThroughStack(SDValue Op)1464 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1465 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1466
1467 SDValue Vec = Op.getOperand(0);
1468 SDValue Part = Op.getOperand(1);
1469 SDValue Idx = Op.getOperand(2);
1470 SDLoc dl(Op);
1471
1472 // Store the value to a temporary stack slot, then LOAD the returned part.
1473 EVT VecVT = Vec.getValueType();
1474 EVT SubVecVT = Part.getValueType();
1475 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1476 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1477 MachinePointerInfo PtrInfo =
1478 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1479
1480 // First store the whole vector.
1481 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1482
1483 // Then store the inserted part.
1484 SDValue SubStackPtr =
1485 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1486
1487 // Store the subvector.
1488 Ch = DAG.getStore(
1489 Ch, dl, Part, SubStackPtr,
1490 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1491
1492 // Finally, load the updated vector.
1493 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1494 }
1495
ExpandVectorBuildThroughStack(SDNode * Node)1496 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1497 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1498 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1499 "Unexpected opcode!");
1500
1501 // We can't handle this case efficiently. Allocate a sufficiently
1502 // aligned object on the stack, store each operand into it, then load
1503 // the result as a vector.
1504 // Create the stack frame object.
1505 EVT VT = Node->getValueType(0);
1506 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1507 : Node->getOperand(0).getValueType();
1508 SDLoc dl(Node);
1509 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1510 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1511 MachinePointerInfo PtrInfo =
1512 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1513
1514 // Emit a store of each element to the stack slot.
1515 SmallVector<SDValue, 8> Stores;
1516 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1517 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1518
1519 // If the destination vector element type of a BUILD_VECTOR is narrower than
1520 // the source element type, only store the bits necessary.
1521 bool Truncate = isa<BuildVectorSDNode>(Node) &&
1522 MemVT.bitsLT(Node->getOperand(0).getValueType());
1523
1524 // Store (in the right endianness) the elements to memory.
1525 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1526 // Ignore undef elements.
1527 if (Node->getOperand(i).isUndef()) continue;
1528
1529 unsigned Offset = TypeByteSize*i;
1530
1531 SDValue Idx =
1532 DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl);
1533
1534 if (Truncate)
1535 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1536 Node->getOperand(i), Idx,
1537 PtrInfo.getWithOffset(Offset), MemVT));
1538 else
1539 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1540 Idx, PtrInfo.getWithOffset(Offset)));
1541 }
1542
1543 SDValue StoreChain;
1544 if (!Stores.empty()) // Not all undef elements?
1545 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1546 else
1547 StoreChain = DAG.getEntryNode();
1548
1549 // Result is a load from the stack slot.
1550 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1551 }
1552
1553 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1554 /// containing the sign bit if the target has no integer value capable of
1555 /// holding all bits of the floating-point value.
getSignAsIntValue(FloatSignAsInt & State,const SDLoc & DL,SDValue Value) const1556 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1557 const SDLoc &DL,
1558 SDValue Value) const {
1559 EVT FloatVT = Value.getValueType();
1560 unsigned NumBits = FloatVT.getScalarSizeInBits();
1561 State.FloatVT = FloatVT;
1562 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1563 // Convert to an integer of the same size.
1564 if (TLI.isTypeLegal(IVT)) {
1565 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1566 State.SignMask = APInt::getSignMask(NumBits);
1567 State.SignBit = NumBits - 1;
1568 return;
1569 }
1570
1571 auto &DataLayout = DAG.getDataLayout();
1572 // Store the float to memory, then load the sign part out as an integer.
1573 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1574 // First create a temporary that is aligned for both the load and store.
1575 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1576 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1577 // Then store the float to it.
1578 State.FloatPtr = StackPtr;
1579 MachineFunction &MF = DAG.getMachineFunction();
1580 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1581 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1582 State.FloatPointerInfo);
1583
1584 SDValue IntPtr;
1585 if (DataLayout.isBigEndian()) {
1586 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1587 // Load out a legal integer with the same sign bit as the float.
1588 IntPtr = StackPtr;
1589 State.IntPointerInfo = State.FloatPointerInfo;
1590 } else {
1591 // Advance the pointer so that the loaded byte will contain the sign bit.
1592 unsigned ByteOffset = (NumBits / 8) - 1;
1593 IntPtr =
1594 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
1595 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1596 ByteOffset);
1597 }
1598
1599 State.IntPtr = IntPtr;
1600 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1601 State.IntPointerInfo, MVT::i8);
1602 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1603 State.SignBit = 7;
1604 }
1605
1606 /// Replace the integer value produced by getSignAsIntValue() with a new value
1607 /// and cast the result back to a floating-point type.
modifySignAsInt(const FloatSignAsInt & State,const SDLoc & DL,SDValue NewIntValue) const1608 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1609 const SDLoc &DL,
1610 SDValue NewIntValue) const {
1611 if (!State.Chain)
1612 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1613
1614 // Override the part containing the sign bit in the value stored on the stack.
1615 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1616 State.IntPointerInfo, MVT::i8);
1617 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1618 State.FloatPointerInfo);
1619 }
1620
ExpandFCOPYSIGN(SDNode * Node) const1621 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1622 SDLoc DL(Node);
1623 SDValue Mag = Node->getOperand(0);
1624 SDValue Sign = Node->getOperand(1);
1625
1626 // Get sign bit into an integer value.
1627 FloatSignAsInt SignAsInt;
1628 getSignAsIntValue(SignAsInt, DL, Sign);
1629
1630 EVT IntVT = SignAsInt.IntValue.getValueType();
1631 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1632 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1633 SignMask);
1634
1635 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1636 EVT FloatVT = Mag.getValueType();
1637 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1638 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1639 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1640 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1641 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1642 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1643 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1644 }
1645
1646 // Transform Mag value to integer, and clear the sign bit.
1647 FloatSignAsInt MagAsInt;
1648 getSignAsIntValue(MagAsInt, DL, Mag);
1649 EVT MagVT = MagAsInt.IntValue.getValueType();
1650 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1651 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1652 ClearSignMask);
1653
1654 // Get the signbit at the right position for MagAsInt.
1655 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1656 EVT ShiftVT = IntVT;
1657 if (SignBit.getScalarValueSizeInBits() <
1658 ClearedSign.getScalarValueSizeInBits()) {
1659 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1660 ShiftVT = MagVT;
1661 }
1662 if (ShiftAmount > 0) {
1663 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1664 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1665 } else if (ShiftAmount < 0) {
1666 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1667 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1668 }
1669 if (SignBit.getScalarValueSizeInBits() >
1670 ClearedSign.getScalarValueSizeInBits()) {
1671 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1672 }
1673
1674 // Store the part with the modified sign and convert back to float.
1675 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1676 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1677 }
1678
ExpandFNEG(SDNode * Node) const1679 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1680 // Get the sign bit as an integer.
1681 SDLoc DL(Node);
1682 FloatSignAsInt SignAsInt;
1683 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1684 EVT IntVT = SignAsInt.IntValue.getValueType();
1685
1686 // Flip the sign.
1687 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1688 SDValue SignFlip =
1689 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1690
1691 // Convert back to float.
1692 return modifySignAsInt(SignAsInt, DL, SignFlip);
1693 }
1694
ExpandFABS(SDNode * Node) const1695 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1696 SDLoc DL(Node);
1697 SDValue Value = Node->getOperand(0);
1698
1699 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1700 EVT FloatVT = Value.getValueType();
1701 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1702 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1703 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1704 }
1705
1706 // Transform value to integer, clear the sign bit and transform back.
1707 FloatSignAsInt ValueAsInt;
1708 getSignAsIntValue(ValueAsInt, DL, Value);
1709 EVT IntVT = ValueAsInt.IntValue.getValueType();
1710 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1711 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1712 ClearSignMask);
1713 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1714 }
1715
ExpandDYNAMIC_STACKALLOC(SDNode * Node,SmallVectorImpl<SDValue> & Results)1716 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1717 SmallVectorImpl<SDValue> &Results) {
1718 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1719 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1720 " not tell us which reg is the stack pointer!");
1721 SDLoc dl(Node);
1722 EVT VT = Node->getValueType(0);
1723 SDValue Tmp1 = SDValue(Node, 0);
1724 SDValue Tmp2 = SDValue(Node, 1);
1725 SDValue Tmp3 = Node->getOperand(2);
1726 SDValue Chain = Tmp1.getOperand(0);
1727
1728 // Chain the dynamic stack allocation so that it doesn't modify the stack
1729 // pointer when other instructions are using the stack.
1730 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1731
1732 SDValue Size = Tmp2.getOperand(1);
1733 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1734 Chain = SP.getValue(1);
1735 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1736 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1737 unsigned Opc =
1738 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1739 ISD::ADD : ISD::SUB;
1740
1741 Align StackAlign = TFL->getStackAlign();
1742 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1743 if (Alignment > StackAlign)
1744 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1745 DAG.getConstant(-Alignment.value(), dl, VT));
1746 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1747
1748 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
1749
1750 Results.push_back(Tmp1);
1751 Results.push_back(Tmp2);
1752 }
1753
1754 /// Emit a store/load combination to the stack. This stores
1755 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1756 /// a load from the stack slot to DestVT, extending it if needed.
1757 /// The resultant code need not be legal.
EmitStackConvert(SDValue SrcOp,EVT SlotVT,EVT DestVT,const SDLoc & dl)1758 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1759 EVT DestVT, const SDLoc &dl) {
1760 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1761 }
1762
EmitStackConvert(SDValue SrcOp,EVT SlotVT,EVT DestVT,const SDLoc & dl,SDValue Chain)1763 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1764 EVT DestVT, const SDLoc &dl,
1765 SDValue Chain) {
1766 EVT SrcVT = SrcOp.getValueType();
1767 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1768 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1769
1770 // Don't convert with stack if the load/store is expensive.
1771 if ((SrcVT.bitsGT(SlotVT) &&
1772 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1773 (SlotVT.bitsLT(DestVT) &&
1774 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1775 return SDValue();
1776
1777 // Create the stack frame object.
1778 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1779 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1780 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1781
1782 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1783 int SPFI = StackPtrFI->getIndex();
1784 MachinePointerInfo PtrInfo =
1785 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1786
1787 // Emit a store to the stack slot. Use a truncstore if the input value is
1788 // later than DestVT.
1789 SDValue Store;
1790
1791 if (SrcVT.bitsGT(SlotVT))
1792 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1793 SlotVT, SrcAlign);
1794 else {
1795 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1796 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1797 }
1798
1799 // Result is a load from the stack slot.
1800 if (SlotVT.bitsEq(DestVT))
1801 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1802
1803 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1804 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1805 DestAlign);
1806 }
1807
ExpandSCALAR_TO_VECTOR(SDNode * Node)1808 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1809 SDLoc dl(Node);
1810 // Create a vector sized/aligned stack slot, store the value to element #0,
1811 // then load the whole vector back out.
1812 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1813
1814 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1815 int SPFI = StackPtrFI->getIndex();
1816
1817 SDValue Ch = DAG.getTruncStore(
1818 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1819 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1820 Node->getValueType(0).getVectorElementType());
1821 return DAG.getLoad(
1822 Node->getValueType(0), dl, Ch, StackPtr,
1823 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1824 }
1825
1826 static bool
ExpandBVWithShuffles(SDNode * Node,SelectionDAG & DAG,const TargetLowering & TLI,SDValue & Res)1827 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1828 const TargetLowering &TLI, SDValue &Res) {
1829 unsigned NumElems = Node->getNumOperands();
1830 SDLoc dl(Node);
1831 EVT VT = Node->getValueType(0);
1832
1833 // Try to group the scalars into pairs, shuffle the pairs together, then
1834 // shuffle the pairs of pairs together, etc. until the vector has
1835 // been built. This will work only if all of the necessary shuffle masks
1836 // are legal.
1837
1838 // We do this in two phases; first to check the legality of the shuffles,
1839 // and next, assuming that all shuffles are legal, to create the new nodes.
1840 for (int Phase = 0; Phase < 2; ++Phase) {
1841 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1842 NewIntermedVals;
1843 for (unsigned i = 0; i < NumElems; ++i) {
1844 SDValue V = Node->getOperand(i);
1845 if (V.isUndef())
1846 continue;
1847
1848 SDValue Vec;
1849 if (Phase)
1850 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1851 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1852 }
1853
1854 while (IntermedVals.size() > 2) {
1855 NewIntermedVals.clear();
1856 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1857 // This vector and the next vector are shuffled together (simply to
1858 // append the one to the other).
1859 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1860
1861 SmallVector<int, 16> FinalIndices;
1862 FinalIndices.reserve(IntermedVals[i].second.size() +
1863 IntermedVals[i+1].second.size());
1864
1865 int k = 0;
1866 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1867 ++j, ++k) {
1868 ShuffleVec[k] = j;
1869 FinalIndices.push_back(IntermedVals[i].second[j]);
1870 }
1871 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1872 ++j, ++k) {
1873 ShuffleVec[k] = NumElems + j;
1874 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1875 }
1876
1877 SDValue Shuffle;
1878 if (Phase)
1879 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1880 IntermedVals[i+1].first,
1881 ShuffleVec);
1882 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1883 return false;
1884 NewIntermedVals.push_back(
1885 std::make_pair(Shuffle, std::move(FinalIndices)));
1886 }
1887
1888 // If we had an odd number of defined values, then append the last
1889 // element to the array of new vectors.
1890 if ((IntermedVals.size() & 1) != 0)
1891 NewIntermedVals.push_back(IntermedVals.back());
1892
1893 IntermedVals.swap(NewIntermedVals);
1894 }
1895
1896 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1897 "Invalid number of intermediate vectors");
1898 SDValue Vec1 = IntermedVals[0].first;
1899 SDValue Vec2;
1900 if (IntermedVals.size() > 1)
1901 Vec2 = IntermedVals[1].first;
1902 else if (Phase)
1903 Vec2 = DAG.getUNDEF(VT);
1904
1905 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1906 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1907 ShuffleVec[IntermedVals[0].second[i]] = i;
1908 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1909 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1910
1911 if (Phase)
1912 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1913 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1914 return false;
1915 }
1916
1917 return true;
1918 }
1919
1920 /// Expand a BUILD_VECTOR node on targets that don't
1921 /// support the operation, but do support the resultant vector type.
ExpandBUILD_VECTOR(SDNode * Node)1922 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1923 unsigned NumElems = Node->getNumOperands();
1924 SDValue Value1, Value2;
1925 SDLoc dl(Node);
1926 EVT VT = Node->getValueType(0);
1927 EVT OpVT = Node->getOperand(0).getValueType();
1928 EVT EltVT = VT.getVectorElementType();
1929
1930 // If the only non-undef value is the low element, turn this into a
1931 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1932 bool isOnlyLowElement = true;
1933 bool MoreThanTwoValues = false;
1934 bool isConstant = true;
1935 for (unsigned i = 0; i < NumElems; ++i) {
1936 SDValue V = Node->getOperand(i);
1937 if (V.isUndef())
1938 continue;
1939 if (i > 0)
1940 isOnlyLowElement = false;
1941 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1942 isConstant = false;
1943
1944 if (!Value1.getNode()) {
1945 Value1 = V;
1946 } else if (!Value2.getNode()) {
1947 if (V != Value1)
1948 Value2 = V;
1949 } else if (V != Value1 && V != Value2) {
1950 MoreThanTwoValues = true;
1951 }
1952 }
1953
1954 if (!Value1.getNode())
1955 return DAG.getUNDEF(VT);
1956
1957 if (isOnlyLowElement)
1958 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1959
1960 // If all elements are constants, create a load from the constant pool.
1961 if (isConstant) {
1962 SmallVector<Constant*, 16> CV;
1963 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1964 if (ConstantFPSDNode *V =
1965 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1966 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1967 } else if (ConstantSDNode *V =
1968 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1969 if (OpVT==EltVT)
1970 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1971 else {
1972 // If OpVT and EltVT don't match, EltVT is not legal and the
1973 // element values have been promoted/truncated earlier. Undo this;
1974 // we don't want a v16i8 to become a v16i32 for example.
1975 const ConstantInt *CI = V->getConstantIntValue();
1976 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1977 CI->getZExtValue()));
1978 }
1979 } else {
1980 assert(Node->getOperand(i).isUndef());
1981 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1982 CV.push_back(UndefValue::get(OpNTy));
1983 }
1984 }
1985 Constant *CP = ConstantVector::get(CV);
1986 SDValue CPIdx =
1987 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1988 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1989 return DAG.getLoad(
1990 VT, dl, DAG.getEntryNode(), CPIdx,
1991 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1992 Alignment);
1993 }
1994
1995 SmallSet<SDValue, 16> DefinedValues;
1996 for (unsigned i = 0; i < NumElems; ++i) {
1997 if (Node->getOperand(i).isUndef())
1998 continue;
1999 DefinedValues.insert(Node->getOperand(i));
2000 }
2001
2002 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
2003 if (!MoreThanTwoValues) {
2004 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2005 for (unsigned i = 0; i < NumElems; ++i) {
2006 SDValue V = Node->getOperand(i);
2007 if (V.isUndef())
2008 continue;
2009 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2010 }
2011 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2012 // Get the splatted value into the low element of a vector register.
2013 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2014 SDValue Vec2;
2015 if (Value2.getNode())
2016 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2017 else
2018 Vec2 = DAG.getUNDEF(VT);
2019
2020 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2021 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2022 }
2023 } else {
2024 SDValue Res;
2025 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2026 return Res;
2027 }
2028 }
2029
2030 // Otherwise, we can't handle this case efficiently.
2031 return ExpandVectorBuildThroughStack(Node);
2032 }
2033
ExpandSPLAT_VECTOR(SDNode * Node)2034 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2035 SDLoc DL(Node);
2036 EVT VT = Node->getValueType(0);
2037 SDValue SplatVal = Node->getOperand(0);
2038
2039 return DAG.getSplatBuildVector(VT, DL, SplatVal);
2040 }
2041
2042 // Expand a node into a call to a libcall, returning the value as the first
2043 // result and the chain as the second. If the result value does not fit into a
2044 // register, return the lo part and set the hi part to the by-reg argument in
2045 // the first. If it does fit into a single register, return the result and
2046 // leave the Hi part unset.
ExpandLibCall(RTLIB::Libcall LC,SDNode * Node,TargetLowering::ArgListTy && Args,bool isSigned)2047 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2048 TargetLowering::ArgListTy &&Args,
2049 bool isSigned) {
2050 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2051 TLI.getPointerTy(DAG.getDataLayout()));
2052
2053 EVT RetVT = Node->getValueType(0);
2054 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2055
2056 // By default, the input chain to this libcall is the entry node of the
2057 // function. If the libcall is going to be emitted as a tail call then
2058 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2059 // node which is being folded has a non-entry input chain.
2060 SDValue InChain = DAG.getEntryNode();
2061
2062 // isTailCall may be true since the callee does not reference caller stack
2063 // frame. Check if it's in the right position and that the return types match.
2064 SDValue TCChain = InChain;
2065 const Function &F = DAG.getMachineFunction().getFunction();
2066 bool isTailCall =
2067 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2068 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2069 if (isTailCall)
2070 InChain = TCChain;
2071
2072 TargetLowering::CallLoweringInfo CLI(DAG);
2073 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2074 CLI.setDebugLoc(SDLoc(Node))
2075 .setChain(InChain)
2076 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2077 std::move(Args))
2078 .setTailCall(isTailCall)
2079 .setSExtResult(signExtend)
2080 .setZExtResult(!signExtend)
2081 .setIsPostTypeLegalization(true);
2082
2083 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2084
2085 if (!CallInfo.second.getNode()) {
2086 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2087 // It's a tailcall, return the chain (which is the DAG root).
2088 return {DAG.getRoot(), DAG.getRoot()};
2089 }
2090
2091 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2092 return CallInfo;
2093 }
2094
ExpandLibCall(RTLIB::Libcall LC,SDNode * Node,bool isSigned)2095 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2096 bool isSigned) {
2097 TargetLowering::ArgListTy Args;
2098 TargetLowering::ArgListEntry Entry;
2099 for (const SDValue &Op : Node->op_values()) {
2100 EVT ArgVT = Op.getValueType();
2101 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2102 Entry.Node = Op;
2103 Entry.Ty = ArgTy;
2104 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2105 Entry.IsZExt = !Entry.IsSExt;
2106 Args.push_back(Entry);
2107 }
2108
2109 return ExpandLibCall(LC, Node, std::move(Args), isSigned);
2110 }
2111
ExpandFrexpLibCall(SDNode * Node,SmallVectorImpl<SDValue> & Results)2112 void SelectionDAGLegalize::ExpandFrexpLibCall(
2113 SDNode *Node, SmallVectorImpl<SDValue> &Results) {
2114 SDLoc dl(Node);
2115 EVT VT = Node->getValueType(0);
2116 EVT ExpVT = Node->getValueType(1);
2117
2118 SDValue FPOp = Node->getOperand(0);
2119
2120 EVT ArgVT = FPOp.getValueType();
2121 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2122
2123 TargetLowering::ArgListEntry FPArgEntry;
2124 FPArgEntry.Node = FPOp;
2125 FPArgEntry.Ty = ArgTy;
2126
2127 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2128 TargetLowering::ArgListEntry PtrArgEntry;
2129 PtrArgEntry.Node = StackSlot;
2130 PtrArgEntry.Ty = PointerType::get(*DAG.getContext(),
2131 DAG.getDataLayout().getAllocaAddrSpace());
2132
2133 TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2134
2135 RTLIB::Libcall LC = RTLIB::getFREXP(VT);
2136 auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false);
2137
2138 // FIXME: Get type of int for libcall declaration and cast
2139
2140 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2141 auto PtrInfo =
2142 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
2143
2144 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2145 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2146 LoadExp.getValue(1), DAG.getRoot());
2147 DAG.setRoot(OutputChain);
2148
2149 Results.push_back(Call);
2150 Results.push_back(LoadExp);
2151 }
2152
ExpandFPLibCall(SDNode * Node,RTLIB::Libcall LC,SmallVectorImpl<SDValue> & Results)2153 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2154 RTLIB::Libcall LC,
2155 SmallVectorImpl<SDValue> &Results) {
2156 if (LC == RTLIB::UNKNOWN_LIBCALL)
2157 llvm_unreachable("Can't create an unknown libcall!");
2158
2159 if (Node->isStrictFPOpcode()) {
2160 EVT RetVT = Node->getValueType(0);
2161 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));
2162 TargetLowering::MakeLibCallOptions CallOptions;
2163 // FIXME: This doesn't support tail calls.
2164 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2165 Ops, CallOptions,
2166 SDLoc(Node),
2167 Node->getOperand(0));
2168 Results.push_back(Tmp.first);
2169 Results.push_back(Tmp.second);
2170 } else {
2171 SDValue Tmp = ExpandLibCall(LC, Node, false).first;
2172 Results.push_back(Tmp);
2173 }
2174 }
2175
2176 /// Expand the node to a libcall based on the result type.
ExpandFPLibCall(SDNode * Node,RTLIB::Libcall Call_F32,RTLIB::Libcall Call_F64,RTLIB::Libcall Call_F80,RTLIB::Libcall Call_F128,RTLIB::Libcall Call_PPCF128,SmallVectorImpl<SDValue> & Results)2177 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2178 RTLIB::Libcall Call_F32,
2179 RTLIB::Libcall Call_F64,
2180 RTLIB::Libcall Call_F80,
2181 RTLIB::Libcall Call_F128,
2182 RTLIB::Libcall Call_PPCF128,
2183 SmallVectorImpl<SDValue> &Results) {
2184 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2185 Call_F32, Call_F64, Call_F80,
2186 Call_F128, Call_PPCF128);
2187 ExpandFPLibCall(Node, LC, Results);
2188 }
2189
ExpandIntLibCall(SDNode * Node,bool isSigned,RTLIB::Libcall Call_I8,RTLIB::Libcall Call_I16,RTLIB::Libcall Call_I32,RTLIB::Libcall Call_I64,RTLIB::Libcall Call_I128)2190 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2191 RTLIB::Libcall Call_I8,
2192 RTLIB::Libcall Call_I16,
2193 RTLIB::Libcall Call_I32,
2194 RTLIB::Libcall Call_I64,
2195 RTLIB::Libcall Call_I128) {
2196 RTLIB::Libcall LC;
2197 switch (Node->getSimpleValueType(0).SimpleTy) {
2198 default: llvm_unreachable("Unexpected request for libcall!");
2199 case MVT::i8: LC = Call_I8; break;
2200 case MVT::i16: LC = Call_I16; break;
2201 case MVT::i32: LC = Call_I32; break;
2202 case MVT::i64: LC = Call_I64; break;
2203 case MVT::i128: LC = Call_I128; break;
2204 }
2205 return ExpandLibCall(LC, Node, isSigned).first;
2206 }
2207
2208 /// Expand the node to a libcall based on first argument type (for instance
2209 /// lround and its variant).
ExpandArgFPLibCall(SDNode * Node,RTLIB::Libcall Call_F32,RTLIB::Libcall Call_F64,RTLIB::Libcall Call_F80,RTLIB::Libcall Call_F128,RTLIB::Libcall Call_PPCF128,SmallVectorImpl<SDValue> & Results)2210 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2211 RTLIB::Libcall Call_F32,
2212 RTLIB::Libcall Call_F64,
2213 RTLIB::Libcall Call_F80,
2214 RTLIB::Libcall Call_F128,
2215 RTLIB::Libcall Call_PPCF128,
2216 SmallVectorImpl<SDValue> &Results) {
2217 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2218 RTLIB::Libcall LC = RTLIB::getFPLibCall(InVT.getSimpleVT(),
2219 Call_F32, Call_F64, Call_F80,
2220 Call_F128, Call_PPCF128);
2221 ExpandFPLibCall(Node, LC, Results);
2222 }
2223
2224 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2225 void
ExpandDivRemLibCall(SDNode * Node,SmallVectorImpl<SDValue> & Results)2226 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2227 SmallVectorImpl<SDValue> &Results) {
2228 unsigned Opcode = Node->getOpcode();
2229 bool isSigned = Opcode == ISD::SDIVREM;
2230
2231 RTLIB::Libcall LC;
2232 switch (Node->getSimpleValueType(0).SimpleTy) {
2233 default: llvm_unreachable("Unexpected request for libcall!");
2234 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2235 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2236 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2237 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2238 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2239 }
2240
2241 // The input chain to this libcall is the entry node of the function.
2242 // Legalizing the call will automatically add the previous call to the
2243 // dependence.
2244 SDValue InChain = DAG.getEntryNode();
2245
2246 EVT RetVT = Node->getValueType(0);
2247 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2248
2249 TargetLowering::ArgListTy Args;
2250 TargetLowering::ArgListEntry Entry;
2251 for (const SDValue &Op : Node->op_values()) {
2252 EVT ArgVT = Op.getValueType();
2253 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2254 Entry.Node = Op;
2255 Entry.Ty = ArgTy;
2256 Entry.IsSExt = isSigned;
2257 Entry.IsZExt = !isSigned;
2258 Args.push_back(Entry);
2259 }
2260
2261 // Also pass the return address of the remainder.
2262 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2263 Entry.Node = FIPtr;
2264 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2265 Entry.IsSExt = isSigned;
2266 Entry.IsZExt = !isSigned;
2267 Args.push_back(Entry);
2268
2269 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2270 TLI.getPointerTy(DAG.getDataLayout()));
2271
2272 SDLoc dl(Node);
2273 TargetLowering::CallLoweringInfo CLI(DAG);
2274 CLI.setDebugLoc(dl)
2275 .setChain(InChain)
2276 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2277 std::move(Args))
2278 .setSExtResult(isSigned)
2279 .setZExtResult(!isSigned);
2280
2281 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2282
2283 // Remainder is loaded back from the stack frame.
2284 SDValue Rem =
2285 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2286 Results.push_back(CallInfo.first);
2287 Results.push_back(Rem);
2288 }
2289
2290 /// Return true if sincos libcall is available.
isSinCosLibcallAvailable(SDNode * Node,const TargetLowering & TLI)2291 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2292 RTLIB::Libcall LC;
2293 switch (Node->getSimpleValueType(0).SimpleTy) {
2294 default: llvm_unreachable("Unexpected request for libcall!");
2295 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2296 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2297 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2298 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2299 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2300 }
2301 return TLI.getLibcallName(LC) != nullptr;
2302 }
2303
2304 /// Only issue sincos libcall if both sin and cos are needed.
useSinCos(SDNode * Node)2305 static bool useSinCos(SDNode *Node) {
2306 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2307 ? ISD::FCOS : ISD::FSIN;
2308
2309 SDValue Op0 = Node->getOperand(0);
2310 for (const SDNode *User : Op0.getNode()->uses()) {
2311 if (User == Node)
2312 continue;
2313 // The other user might have been turned into sincos already.
2314 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2315 return true;
2316 }
2317 return false;
2318 }
2319
2320 /// Issue libcalls to sincos to compute sin / cos pairs.
2321 void
ExpandSinCosLibCall(SDNode * Node,SmallVectorImpl<SDValue> & Results)2322 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2323 SmallVectorImpl<SDValue> &Results) {
2324 RTLIB::Libcall LC;
2325 switch (Node->getSimpleValueType(0).SimpleTy) {
2326 default: llvm_unreachable("Unexpected request for libcall!");
2327 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2328 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2329 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2330 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2331 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2332 }
2333
2334 // The input chain to this libcall is the entry node of the function.
2335 // Legalizing the call will automatically add the previous call to the
2336 // dependence.
2337 SDValue InChain = DAG.getEntryNode();
2338
2339 EVT RetVT = Node->getValueType(0);
2340 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2341
2342 TargetLowering::ArgListTy Args;
2343 TargetLowering::ArgListEntry Entry;
2344
2345 // Pass the argument.
2346 Entry.Node = Node->getOperand(0);
2347 Entry.Ty = RetTy;
2348 Entry.IsSExt = false;
2349 Entry.IsZExt = false;
2350 Args.push_back(Entry);
2351
2352 // Pass the return address of sin.
2353 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2354 Entry.Node = SinPtr;
2355 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2356 Entry.IsSExt = false;
2357 Entry.IsZExt = false;
2358 Args.push_back(Entry);
2359
2360 // Also pass the return address of the cos.
2361 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2362 Entry.Node = CosPtr;
2363 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2364 Entry.IsSExt = false;
2365 Entry.IsZExt = false;
2366 Args.push_back(Entry);
2367
2368 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2369 TLI.getPointerTy(DAG.getDataLayout()));
2370
2371 SDLoc dl(Node);
2372 TargetLowering::CallLoweringInfo CLI(DAG);
2373 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2374 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2375 std::move(Args));
2376
2377 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2378
2379 Results.push_back(
2380 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2381 Results.push_back(
2382 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2383 }
2384
expandLdexp(SDNode * Node) const2385 SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2386 SDLoc dl(Node);
2387 EVT VT = Node->getValueType(0);
2388 SDValue X = Node->getOperand(0);
2389 SDValue N = Node->getOperand(1);
2390 EVT ExpVT = N.getValueType();
2391 EVT AsIntVT = VT.changeTypeToInteger();
2392 if (AsIntVT == EVT()) // TODO: How to handle f80?
2393 return SDValue();
2394
2395 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2396 return SDValue();
2397
2398 SDNodeFlags NSW;
2399 NSW.setNoSignedWrap(true);
2400 SDNodeFlags NUW_NSW;
2401 NUW_NSW.setNoUnsignedWrap(true);
2402 NUW_NSW.setNoSignedWrap(true);
2403
2404 EVT SetCCVT =
2405 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2406 const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2407
2408 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2409 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2410 const int Precision = APFloat::semanticsPrecision(FltSem);
2411
2412 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2413 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2414
2415 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2416
2417 const APFloat One(FltSem, "1.0");
2418 APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2419
2420 // Offset by precision to avoid denormal range.
2421 APFloat ScaleDownK =
2422 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2423
2424 // TODO: Should really introduce control flow and use a block for the >
2425 // MaxExp, < MinExp cases
2426
2427 // First, handle exponents Exp > MaxExp and scale down.
2428 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT);
2429
2430 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW);
2431 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2432 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal);
2433 SDValue DecN1 =
2434 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);
2435
2436 SDValue ScaleUpTwice =
2437 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT);
2438
2439 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2440 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal);
2441 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal);
2442
2443 SDValue SelectN_Big =
2444 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0);
2445 SDValue SelectX_Big =
2446 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0);
2447
2448 // Now handle exponents Exp < MinExp
2449 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT);
2450
2451 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2452 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2453
2454 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW);
2455
2456 SDValue ClampMinVal =
2457 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2458 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal);
2459 SDValue IncN1 =
2460 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);
2461
2462 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2463 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal);
2464 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal);
2465
2466 SDValue ScaleDownTwice = DAG.getSetCC(
2467 dl, SetCCVT, N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2468 ISD::SETULT);
2469
2470 SDValue SelectN_Small =
2471 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0);
2472 SDValue SelectX_Small =
2473 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);
2474
2475 // Now combine the two out of range exponent handling cases with the base
2476 // case.
2477 SDValue NewX = DAG.getNode(
2478 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,
2479 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X));
2480
2481 SDValue NewN = DAG.getNode(
2482 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,
2483 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N));
2484
2485 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW);
2486
2487 SDValue ExponentShiftAmt =
2488 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2489 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2490
2491 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy,
2492 ExponentShiftAmt, NUW_NSW);
2493 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);
2494 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP);
2495 }
2496
expandFrexp(SDNode * Node) const2497 SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2498 SDLoc dl(Node);
2499 SDValue Val = Node->getOperand(0);
2500 EVT VT = Val.getValueType();
2501 EVT ExpVT = Node->getValueType(1);
2502 EVT AsIntVT = VT.changeTypeToInteger();
2503 if (AsIntVT == EVT()) // TODO: How to handle f80?
2504 return SDValue();
2505
2506 const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2507 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2508 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2509 const unsigned BitSize = VT.getScalarSizeInBits();
2510
2511 // TODO: Could introduce control flow and skip over the denormal handling.
2512
2513 // scale_up = fmul value, scalbn(1.0, precision + 1)
2514 // extracted_exp = (bitcast value to uint) >> precision - 1
2515 // biased_exp = extracted_exp + min_exp
2516 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2517 //
2518 // is_denormal = val < smallest_normalized
2519 // computed_fract = is_denormal ? scale_up : extracted_fract
2520 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2521 //
2522 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2523 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2524
2525 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2526 APFloat::getSmallestNormalized(FltSem, true).bitcastToAPInt(), dl,
2527 AsIntVT);
2528
2529 SDValue SmallestNormalizedInt = DAG.getConstant(
2530 APFloat::getSmallestNormalized(FltSem, false).bitcastToAPInt(), dl,
2531 AsIntVT);
2532
2533 // Masks out the exponent bits.
2534 SDValue ExpMask =
2535 DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2536
2537 // Mask out the exponent part of the value.
2538 //
2539 // e.g, for f32 FractSignMaskVal = 0x807fffff
2540 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);
2541 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2542
2543 APInt SignMaskVal = APInt::getSignedMaxValue(BitSize);
2544 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2545
2546 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2547
2548 const APFloat One(FltSem, "1.0");
2549 // Scale a possible denormal input.
2550 // e.g., for f64, 0x1p+54
2551 APFloat ScaleUpKVal =
2552 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2553
2554 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2555 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK);
2556
2557 EVT SetCCVT =
2558 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2559
2560 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);
2561
2562 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask);
2563
2564 SDValue AddNegSmallestNormal =
2565 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);
2566 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2567 NegSmallestNormalizedInt, ISD::SETULE);
2568
2569 SDValue IsDenormal =
2570 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
2571
2572 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2573 SDValue Zero = DAG.getConstant(0, dl, ExpVT);
2574
2575 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
2576 SDValue ScaledSelect =
2577 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);
2578
2579 SDValue ExpMaskScaled =
2580 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);
2581
2582 SDValue ScaledValue =
2583 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);
2584
2585 // Extract the exponent bits.
2586 SDValue ExponentShiftAmt =
2587 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2588 SDValue ShiftedExp =
2589 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);
2590 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2591
2592 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp);
2593 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2594 SDValue DenormalExpBias =
2595 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);
2596
2597 SDValue MaskedFractAsInt =
2598 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);
2599 const APFloat Half(FltSem, "0.5");
2600 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2601 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf);
2602 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);
2603
2604 SDValue ComputedExp =
2605 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);
2606
2607 SDValue Result0 =
2608 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract);
2609
2610 SDValue Result1 =
2611 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);
2612
2613 return DAG.getMergeValues({Result0, Result1}, dl);
2614 }
2615
2616 /// This function is responsible for legalizing a
2617 /// INT_TO_FP operation of the specified operand when the target requests that
2618 /// we expand it. At this point, we know that the result and operand types are
2619 /// legal for the target.
ExpandLegalINT_TO_FP(SDNode * Node,SDValue & Chain)2620 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2621 SDValue &Chain) {
2622 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2623 Node->getOpcode() == ISD::SINT_TO_FP);
2624 EVT DestVT = Node->getValueType(0);
2625 SDLoc dl(Node);
2626 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2627 SDValue Op0 = Node->getOperand(OpNo);
2628 EVT SrcVT = Op0.getValueType();
2629
2630 // TODO: Should any fast-math-flags be set for the created nodes?
2631 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2632 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2633 (DestVT.bitsLE(MVT::f64) ||
2634 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2635 : ISD::FP_EXTEND,
2636 DestVT))) {
2637 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2638 "expansion\n");
2639
2640 // Get the stack frame index of a 8 byte buffer.
2641 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2642
2643 SDValue Lo = Op0;
2644 // if signed map to unsigned space
2645 if (isSigned) {
2646 // Invert sign bit (signed to unsigned mapping).
2647 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2648 DAG.getConstant(0x80000000u, dl, MVT::i32));
2649 }
2650 // Initial hi portion of constructed double.
2651 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2652
2653 // If this a big endian target, swap the lo and high data.
2654 if (DAG.getDataLayout().isBigEndian())
2655 std::swap(Lo, Hi);
2656
2657 SDValue MemChain = DAG.getEntryNode();
2658
2659 // Store the lo of the constructed double.
2660 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2661 MachinePointerInfo());
2662 // Store the hi of the constructed double.
2663 SDValue HiPtr =
2664 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
2665 SDValue Store2 =
2666 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2667 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2668
2669 // load the constructed double
2670 SDValue Load =
2671 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2672 // FP constant to bias correct the final result
2673 SDValue Bias = DAG.getConstantFP(
2674 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2675 : llvm::bit_cast<double>(0x4330000000000000ULL),
2676 dl, MVT::f64);
2677 // Subtract the bias and get the final result.
2678 SDValue Sub;
2679 SDValue Result;
2680 if (Node->isStrictFPOpcode()) {
2681 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2682 {Node->getOperand(0), Load, Bias});
2683 Chain = Sub.getValue(1);
2684 if (DestVT != Sub.getValueType()) {
2685 std::pair<SDValue, SDValue> ResultPair;
2686 ResultPair =
2687 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2688 Result = ResultPair.first;
2689 Chain = ResultPair.second;
2690 }
2691 else
2692 Result = Sub;
2693 } else {
2694 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2695 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2696 }
2697 return Result;
2698 }
2699
2700 if (isSigned)
2701 return SDValue();
2702
2703 // TODO: Generalize this for use with other types.
2704 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2705 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2706 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2707 // For unsigned conversions, convert them to signed conversions using the
2708 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2709 // should be valid for i32->f32 as well.
2710
2711 // More generally this transform should be valid if there are 3 more bits
2712 // in the integer type than the significand. Rounding uses the first bit
2713 // after the width of the significand and the OR of all bits after that. So
2714 // we need to be able to OR the shifted out bit into one of the bits that
2715 // participate in the OR.
2716
2717 // TODO: This really should be implemented using a branch rather than a
2718 // select. We happen to get lucky and machinesink does the right
2719 // thing most of the time. This would be a good candidate for a
2720 // pseudo-op, or, even better, for whole-function isel.
2721 EVT SetCCVT = getSetCCResultType(SrcVT);
2722
2723 SDValue SignBitTest = DAG.getSetCC(
2724 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2725
2726 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2727 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2728 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2729 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2730 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2731 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2732
2733 SDValue Slow, Fast;
2734 if (Node->isStrictFPOpcode()) {
2735 // In strict mode, we must avoid spurious exceptions, and therefore
2736 // must make sure to only emit a single STRICT_SINT_TO_FP.
2737 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2738 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2739 { Node->getOperand(0), InCvt });
2740 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2741 { Fast.getValue(1), Fast, Fast });
2742 Chain = Slow.getValue(1);
2743 // The STRICT_SINT_TO_FP inherits the exception mode from the
2744 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2745 // never raise any exception.
2746 SDNodeFlags Flags;
2747 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2748 Fast->setFlags(Flags);
2749 Flags.setNoFPExcept(true);
2750 Slow->setFlags(Flags);
2751 } else {
2752 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2753 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2754 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2755 }
2756
2757 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2758 }
2759
2760 // Don't expand it if there isn't cheap fadd.
2761 if (!TLI.isOperationLegalOrCustom(
2762 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2763 return SDValue();
2764
2765 // The following optimization is valid only if every value in SrcVT (when
2766 // treated as signed) is representable in DestVT. Check that the mantissa
2767 // size of DestVT is >= than the number of bits in SrcVT -1.
2768 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2769 SrcVT.getSizeInBits() - 1 &&
2770 "Cannot perform lossless SINT_TO_FP!");
2771
2772 SDValue Tmp1;
2773 if (Node->isStrictFPOpcode()) {
2774 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2775 { Node->getOperand(0), Op0 });
2776 } else
2777 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2778
2779 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2780 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2781 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2782 Four = DAG.getIntPtrConstant(4, dl);
2783 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2784 SignSet, Four, Zero);
2785
2786 // If the sign bit of the integer is set, the large number will be treated
2787 // as a negative number. To counteract this, the dynamic code adds an
2788 // offset depending on the data type.
2789 uint64_t FF;
2790 switch (SrcVT.getSimpleVT().SimpleTy) {
2791 default:
2792 return SDValue();
2793 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2794 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2795 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2796 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2797 }
2798 if (DAG.getDataLayout().isLittleEndian())
2799 FF <<= 32;
2800 Constant *FudgeFactor = ConstantInt::get(
2801 Type::getInt64Ty(*DAG.getContext()), FF);
2802
2803 SDValue CPIdx =
2804 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2805 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2806 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2807 Alignment = commonAlignment(Alignment, 4);
2808 SDValue FudgeInReg;
2809 if (DestVT == MVT::f32)
2810 FudgeInReg = DAG.getLoad(
2811 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2812 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2813 Alignment);
2814 else {
2815 SDValue Load = DAG.getExtLoad(
2816 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2817 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2818 Alignment);
2819 HandleSDNode Handle(Load);
2820 LegalizeOp(Load.getNode());
2821 FudgeInReg = Handle.getValue();
2822 }
2823
2824 if (Node->isStrictFPOpcode()) {
2825 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2826 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2827 Chain = Result.getValue(1);
2828 return Result;
2829 }
2830
2831 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2832 }
2833
2834 /// This function is responsible for legalizing a
2835 /// *INT_TO_FP operation of the specified operand when the target requests that
2836 /// we promote it. At this point, we know that the result and operand types are
2837 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2838 /// operation that takes a larger input.
PromoteLegalINT_TO_FP(SDNode * N,const SDLoc & dl,SmallVectorImpl<SDValue> & Results)2839 void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2840 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2841 bool IsStrict = N->isStrictFPOpcode();
2842 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2843 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2844 EVT DestVT = N->getValueType(0);
2845 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2846 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2847 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2848
2849 // First step, figure out the appropriate *INT_TO_FP operation to use.
2850 EVT NewInTy = LegalOp.getValueType();
2851
2852 unsigned OpToUse = 0;
2853
2854 // Scan for the appropriate larger type to use.
2855 while (true) {
2856 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2857 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2858
2859 // If the target supports SINT_TO_FP of this type, use it.
2860 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2861 OpToUse = SIntOp;
2862 break;
2863 }
2864 if (IsSigned)
2865 continue;
2866
2867 // If the target supports UINT_TO_FP of this type, use it.
2868 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2869 OpToUse = UIntOp;
2870 break;
2871 }
2872
2873 // Otherwise, try a larger type.
2874 }
2875
2876 // Okay, we found the operation and type to use. Zero extend our input to the
2877 // desired type then run the operation on it.
2878 if (IsStrict) {
2879 SDValue Res =
2880 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2881 {N->getOperand(0),
2882 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2883 dl, NewInTy, LegalOp)});
2884 Results.push_back(Res);
2885 Results.push_back(Res.getValue(1));
2886 return;
2887 }
2888
2889 Results.push_back(
2890 DAG.getNode(OpToUse, dl, DestVT,
2891 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2892 dl, NewInTy, LegalOp)));
2893 }
2894
2895 /// This function is responsible for legalizing a
2896 /// FP_TO_*INT operation of the specified operand when the target requests that
2897 /// we promote it. At this point, we know that the result and operand types are
2898 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2899 /// operation that returns a larger result.
PromoteLegalFP_TO_INT(SDNode * N,const SDLoc & dl,SmallVectorImpl<SDValue> & Results)2900 void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2901 SmallVectorImpl<SDValue> &Results) {
2902 bool IsStrict = N->isStrictFPOpcode();
2903 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2904 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2905 EVT DestVT = N->getValueType(0);
2906 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2907 // First step, figure out the appropriate FP_TO*INT operation to use.
2908 EVT NewOutTy = DestVT;
2909
2910 unsigned OpToUse = 0;
2911
2912 // Scan for the appropriate larger type to use.
2913 while (true) {
2914 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2915 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2916
2917 // A larger signed type can hold all unsigned values of the requested type,
2918 // so using FP_TO_SINT is valid
2919 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2920 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2921 break;
2922
2923 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2924 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2925 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2926 break;
2927
2928 // Otherwise, try a larger type.
2929 }
2930
2931 // Okay, we found the operation and type to use.
2932 SDValue Operation;
2933 if (IsStrict) {
2934 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2935 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2936 } else
2937 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2938
2939 // Truncate the result of the extended FP_TO_*INT operation to the desired
2940 // size.
2941 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2942 Results.push_back(Trunc);
2943 if (IsStrict)
2944 Results.push_back(Operation.getValue(1));
2945 }
2946
2947 /// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2948 /// the result and operand types are legal and there must be a legal
2949 /// FP_TO_*INT_SAT operation for a larger result type.
PromoteLegalFP_TO_INT_SAT(SDNode * Node,const SDLoc & dl)2950 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2951 const SDLoc &dl) {
2952 unsigned Opcode = Node->getOpcode();
2953
2954 // Scan for the appropriate larger type to use.
2955 EVT NewOutTy = Node->getValueType(0);
2956 while (true) {
2957 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2958 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2959
2960 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2961 break;
2962 }
2963
2964 // Saturation width is determined by second operand, so we don't have to
2965 // perform any fixup and can directly truncate the result.
2966 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2967 Node->getOperand(1));
2968 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2969 }
2970
2971 /// Open code the operations for PARITY of the specified operation.
ExpandPARITY(SDValue Op,const SDLoc & dl)2972 SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2973 EVT VT = Op.getValueType();
2974 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2975 unsigned Sz = VT.getScalarSizeInBits();
2976
2977 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2978 SDValue Result;
2979 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2980 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2981 } else {
2982 Result = Op;
2983 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2984 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2985 DAG.getConstant(1ULL << (--i), dl, ShVT));
2986 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2987 }
2988 }
2989
2990 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2991 }
2992
ExpandNode(SDNode * Node)2993 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2994 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2995 SmallVector<SDValue, 8> Results;
2996 SDLoc dl(Node);
2997 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2998 bool NeedInvert;
2999 switch (Node->getOpcode()) {
3000 case ISD::ABS:
3001 if ((Tmp1 = TLI.expandABS(Node, DAG)))
3002 Results.push_back(Tmp1);
3003 break;
3004 case ISD::ABDS:
3005 case ISD::ABDU:
3006 if ((Tmp1 = TLI.expandABD(Node, DAG)))
3007 Results.push_back(Tmp1);
3008 break;
3009 case ISD::CTPOP:
3010 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
3011 Results.push_back(Tmp1);
3012 break;
3013 case ISD::CTLZ:
3014 case ISD::CTLZ_ZERO_UNDEF:
3015 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
3016 Results.push_back(Tmp1);
3017 break;
3018 case ISD::CTTZ:
3019 case ISD::CTTZ_ZERO_UNDEF:
3020 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
3021 Results.push_back(Tmp1);
3022 break;
3023 case ISD::BITREVERSE:
3024 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
3025 Results.push_back(Tmp1);
3026 break;
3027 case ISD::BSWAP:
3028 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
3029 Results.push_back(Tmp1);
3030 break;
3031 case ISD::PARITY:
3032 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
3033 break;
3034 case ISD::FRAMEADDR:
3035 case ISD::RETURNADDR:
3036 case ISD::FRAME_TO_ARGS_OFFSET:
3037 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3038 break;
3039 case ISD::EH_DWARF_CFA: {
3040 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
3041 TLI.getPointerTy(DAG.getDataLayout()));
3042 SDValue Offset = DAG.getNode(ISD::ADD, dl,
3043 CfaArg.getValueType(),
3044 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
3045 CfaArg.getValueType()),
3046 CfaArg);
3047 SDValue FA = DAG.getNode(
3048 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
3049 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3050 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
3051 FA, Offset));
3052 break;
3053 }
3054 case ISD::GET_ROUNDING:
3055 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
3056 Results.push_back(Node->getOperand(0));
3057 break;
3058 case ISD::EH_RETURN:
3059 case ISD::EH_LABEL:
3060 case ISD::PREFETCH:
3061 case ISD::VAEND:
3062 case ISD::EH_SJLJ_LONGJMP:
3063 // If the target didn't expand these, there's nothing to do, so just
3064 // preserve the chain and be done.
3065 Results.push_back(Node->getOperand(0));
3066 break;
3067 case ISD::READCYCLECOUNTER:
3068 // If the target didn't expand this, just return 'zero' and preserve the
3069 // chain.
3070 Results.append(Node->getNumValues() - 1,
3071 DAG.getConstant(0, dl, Node->getValueType(0)));
3072 Results.push_back(Node->getOperand(0));
3073 break;
3074 case ISD::EH_SJLJ_SETJMP:
3075 // If the target didn't expand this, just return 'zero' and preserve the
3076 // chain.
3077 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3078 Results.push_back(Node->getOperand(0));
3079 break;
3080 case ISD::ATOMIC_LOAD: {
3081 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3082 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
3083 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3084 SDValue Swap = DAG.getAtomicCmpSwap(
3085 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3086 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3087 cast<AtomicSDNode>(Node)->getMemOperand());
3088 Results.push_back(Swap.getValue(0));
3089 Results.push_back(Swap.getValue(1));
3090 break;
3091 }
3092 case ISD::ATOMIC_STORE: {
3093 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3094 SDValue Swap = DAG.getAtomic(
3095 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
3096 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3097 cast<AtomicSDNode>(Node)->getMemOperand());
3098 Results.push_back(Swap.getValue(1));
3099 break;
3100 }
3101 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3102 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3103 // splits out the success value as a comparison. Expanding the resulting
3104 // ATOMIC_CMP_SWAP will produce a libcall.
3105 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3106 SDValue Res = DAG.getAtomicCmpSwap(
3107 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3108 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3109 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
3110
3111 SDValue ExtRes = Res;
3112 SDValue LHS = Res;
3113 SDValue RHS = Node->getOperand(1);
3114
3115 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
3116 EVT OuterType = Node->getValueType(0);
3117 switch (TLI.getExtendForAtomicOps()) {
3118 case ISD::SIGN_EXTEND:
3119 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
3120 DAG.getValueType(AtomicType));
3121 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
3122 Node->getOperand(2), DAG.getValueType(AtomicType));
3123 ExtRes = LHS;
3124 break;
3125 case ISD::ZERO_EXTEND:
3126 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
3127 DAG.getValueType(AtomicType));
3128 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3129 ExtRes = LHS;
3130 break;
3131 case ISD::ANY_EXTEND:
3132 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3133 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3134 break;
3135 default:
3136 llvm_unreachable("Invalid atomic op extension");
3137 }
3138
3139 SDValue Success =
3140 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
3141
3142 Results.push_back(ExtRes.getValue(0));
3143 Results.push_back(Success);
3144 Results.push_back(Res.getValue(1));
3145 break;
3146 }
3147 case ISD::ATOMIC_LOAD_SUB: {
3148 SDLoc DL(Node);
3149 EVT VT = Node->getValueType(0);
3150 SDValue RHS = Node->getOperand(2);
3151 AtomicSDNode *AN = cast<AtomicSDNode>(Node);
3152 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3153 cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT())
3154 RHS = RHS->getOperand(0);
3155 SDValue NewRHS =
3156 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
3157 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(),
3158 Node->getOperand(0), Node->getOperand(1),
3159 NewRHS, AN->getMemOperand());
3160 Results.push_back(Res);
3161 Results.push_back(Res.getValue(1));
3162 break;
3163 }
3164 case ISD::DYNAMIC_STACKALLOC:
3165 ExpandDYNAMIC_STACKALLOC(Node, Results);
3166 break;
3167 case ISD::MERGE_VALUES:
3168 for (unsigned i = 0; i < Node->getNumValues(); i++)
3169 Results.push_back(Node->getOperand(i));
3170 break;
3171 case ISD::UNDEF: {
3172 EVT VT = Node->getValueType(0);
3173 if (VT.isInteger())
3174 Results.push_back(DAG.getConstant(0, dl, VT));
3175 else {
3176 assert(VT.isFloatingPoint() && "Unknown value type!");
3177 Results.push_back(DAG.getConstantFP(0, dl, VT));
3178 }
3179 break;
3180 }
3181 case ISD::STRICT_FP_ROUND:
3182 // When strict mode is enforced we can't do expansion because it
3183 // does not honor the "strict" properties. Only libcall is allowed.
3184 if (TLI.isStrictFPEnabled())
3185 break;
3186 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3187 // since this operation is more efficient than stack operation.
3188 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3189 Node->getValueType(0))
3190 == TargetLowering::Legal)
3191 break;
3192 // We fall back to use stack operation when the FP_ROUND operation
3193 // isn't available.
3194 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
3195 Node->getValueType(0), dl,
3196 Node->getOperand(0)))) {
3197 ReplaceNode(Node, Tmp1.getNode());
3198 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3199 return true;
3200 }
3201 break;
3202 case ISD::FP_ROUND: {
3203 EVT VT = Node->getValueType(0);
3204 if (VT.getScalarType() == MVT::bf16) {
3205 Results.push_back(
3206 DAG.getNode(ISD::FP_TO_BF16, SDLoc(Node), VT, Node->getOperand(0)));
3207 break;
3208 }
3209
3210 LLVM_FALLTHROUGH;
3211 }
3212 case ISD::BITCAST:
3213 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3214 Node->getValueType(0), dl)))
3215 Results.push_back(Tmp1);
3216 break;
3217 case ISD::STRICT_FP_EXTEND:
3218 // When strict mode is enforced we can't do expansion because it
3219 // does not honor the "strict" properties. Only libcall is allowed.
3220 if (TLI.isStrictFPEnabled())
3221 break;
3222 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3223 // since this operation is more efficient than stack operation.
3224 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3225 Node->getValueType(0))
3226 == TargetLowering::Legal)
3227 break;
3228 // We fall back to use stack operation when the FP_EXTEND operation
3229 // isn't available.
3230 if ((Tmp1 = EmitStackConvert(
3231 Node->getOperand(1), Node->getOperand(1).getValueType(),
3232 Node->getValueType(0), dl, Node->getOperand(0)))) {
3233 ReplaceNode(Node, Tmp1.getNode());
3234 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3235 return true;
3236 }
3237 break;
3238 case ISD::FP_EXTEND: {
3239 SDValue Op = Node->getOperand(0);
3240 EVT SrcVT = Op.getValueType();
3241 EVT DstVT = Node->getValueType(0);
3242 if (SrcVT.getScalarType() == MVT::bf16) {
3243 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op));
3244 break;
3245 }
3246
3247 if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl)))
3248 Results.push_back(Tmp1);
3249 break;
3250 }
3251 case ISD::BF16_TO_FP: {
3252 // Always expand bf16 to f32 casts, they lower to ext + shift.
3253 //
3254 // Note that the operand of this code can be bf16 or an integer type in case
3255 // bf16 is not supported on the target and was softened.
3256 SDValue Op = Node->getOperand(0);
3257 if (Op.getValueType() == MVT::bf16) {
3258 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3259 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3260 } else {
3261 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3262 }
3263 Op = DAG.getNode(
3264 ISD::SHL, dl, MVT::i32, Op,
3265 DAG.getConstant(16, dl,
3266 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3267 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3268 // Add fp_extend in case the output is bigger than f32.
3269 if (Node->getValueType(0) != MVT::f32)
3270 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
3271 Results.push_back(Op);
3272 break;
3273 }
3274 case ISD::FP_TO_BF16: {
3275 SDValue Op = Node->getOperand(0);
3276 if (Op.getValueType() != MVT::f32)
3277 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3278 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3279 Op = DAG.getNode(
3280 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3281 DAG.getConstant(16, dl,
3282 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3283 // The result of this node can be bf16 or an integer type in case bf16 is
3284 // not supported on the target and was softened to i16 for storage.
3285 if (Node->getValueType(0) == MVT::bf16) {
3286 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3287 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3288 } else {
3289 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0));
3290 }
3291 Results.push_back(Op);
3292 break;
3293 }
3294 case ISD::SIGN_EXTEND_INREG: {
3295 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3296 EVT VT = Node->getValueType(0);
3297
3298 // An in-register sign-extend of a boolean is a negation:
3299 // 'true' (1) sign-extended is -1.
3300 // 'false' (0) sign-extended is 0.
3301 // However, we must mask the high bits of the source operand because the
3302 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3303
3304 // TODO: Do this for vectors too?
3305 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3306 SDValue One = DAG.getConstant(1, dl, VT);
3307 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3308 SDValue Zero = DAG.getConstant(0, dl, VT);
3309 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3310 Results.push_back(Neg);
3311 break;
3312 }
3313
3314 // NOTE: we could fall back on load/store here too for targets without
3315 // SRA. However, it is doubtful that any exist.
3316 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3317 unsigned BitsDiff = VT.getScalarSizeInBits() -
3318 ExtraVT.getScalarSizeInBits();
3319 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3320 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3321 Node->getOperand(0), ShiftCst);
3322 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3323 Results.push_back(Tmp1);
3324 break;
3325 }
3326 case ISD::UINT_TO_FP:
3327 case ISD::STRICT_UINT_TO_FP:
3328 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
3329 Results.push_back(Tmp1);
3330 if (Node->isStrictFPOpcode())
3331 Results.push_back(Tmp2);
3332 break;
3333 }
3334 [[fallthrough]];
3335 case ISD::SINT_TO_FP:
3336 case ISD::STRICT_SINT_TO_FP:
3337 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3338 Results.push_back(Tmp1);
3339 if (Node->isStrictFPOpcode())
3340 Results.push_back(Tmp2);
3341 }
3342 break;
3343 case ISD::FP_TO_SINT:
3344 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3345 Results.push_back(Tmp1);
3346 break;
3347 case ISD::STRICT_FP_TO_SINT:
3348 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
3349 ReplaceNode(Node, Tmp1.getNode());
3350 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3351 return true;
3352 }
3353 break;
3354 case ISD::FP_TO_UINT:
3355 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
3356 Results.push_back(Tmp1);
3357 break;
3358 case ISD::STRICT_FP_TO_UINT:
3359 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
3360 // Relink the chain.
3361 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
3362 // Replace the new UINT result.
3363 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
3364 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3365 return true;
3366 }
3367 break;
3368 case ISD::FP_TO_SINT_SAT:
3369 case ISD::FP_TO_UINT_SAT:
3370 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
3371 break;
3372 case ISD::VAARG:
3373 Results.push_back(DAG.expandVAArg(Node));
3374 Results.push_back(Results[0].getValue(1));
3375 break;
3376 case ISD::VACOPY:
3377 Results.push_back(DAG.expandVACopy(Node));
3378 break;
3379 case ISD::EXTRACT_VECTOR_ELT:
3380 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3381 // This must be an access of the only element. Return it.
3382 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3383 Node->getOperand(0));
3384 else
3385 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3386 Results.push_back(Tmp1);
3387 break;
3388 case ISD::EXTRACT_SUBVECTOR:
3389 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3390 break;
3391 case ISD::INSERT_SUBVECTOR:
3392 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3393 break;
3394 case ISD::CONCAT_VECTORS:
3395 Results.push_back(ExpandVectorBuildThroughStack(Node));
3396 break;
3397 case ISD::SCALAR_TO_VECTOR:
3398 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3399 break;
3400 case ISD::INSERT_VECTOR_ELT:
3401 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3402 Node->getOperand(1),
3403 Node->getOperand(2), dl));
3404 break;
3405 case ISD::VECTOR_SHUFFLE: {
3406 SmallVector<int, 32> NewMask;
3407 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3408
3409 EVT VT = Node->getValueType(0);
3410 EVT EltVT = VT.getVectorElementType();
3411 SDValue Op0 = Node->getOperand(0);
3412 SDValue Op1 = Node->getOperand(1);
3413 if (!TLI.isTypeLegal(EltVT)) {
3414 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3415
3416 // BUILD_VECTOR operands are allowed to be wider than the element type.
3417 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3418 // it.
3419 if (NewEltVT.bitsLT(EltVT)) {
3420 // Convert shuffle node.
3421 // If original node was v4i64 and the new EltVT is i32,
3422 // cast operands to v8i32 and re-build the mask.
3423
3424 // Calculate new VT, the size of the new VT should be equal to original.
3425 EVT NewVT =
3426 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3427 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3428 assert(NewVT.bitsEq(VT));
3429
3430 // cast operands to new VT
3431 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3432 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3433
3434 // Convert the shuffle mask
3435 unsigned int factor =
3436 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3437
3438 // EltVT gets smaller
3439 assert(factor > 0);
3440
3441 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3442 if (Mask[i] < 0) {
3443 for (unsigned fi = 0; fi < factor; ++fi)
3444 NewMask.push_back(Mask[i]);
3445 }
3446 else {
3447 for (unsigned fi = 0; fi < factor; ++fi)
3448 NewMask.push_back(Mask[i]*factor+fi);
3449 }
3450 }
3451 Mask = NewMask;
3452 VT = NewVT;
3453 }
3454 EltVT = NewEltVT;
3455 }
3456 unsigned NumElems = VT.getVectorNumElements();
3457 SmallVector<SDValue, 16> Ops;
3458 for (unsigned i = 0; i != NumElems; ++i) {
3459 if (Mask[i] < 0) {
3460 Ops.push_back(DAG.getUNDEF(EltVT));
3461 continue;
3462 }
3463 unsigned Idx = Mask[i];
3464 if (Idx < NumElems)
3465 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3466 DAG.getVectorIdxConstant(Idx, dl)));
3467 else
3468 Ops.push_back(
3469 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3470 DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3471 }
3472
3473 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3474 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3475 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3476 Results.push_back(Tmp1);
3477 break;
3478 }
3479 case ISD::VECTOR_SPLICE: {
3480 Results.push_back(TLI.expandVectorSplice(Node, DAG));
3481 break;
3482 }
3483 case ISD::EXTRACT_ELEMENT: {
3484 EVT OpTy = Node->getOperand(0).getValueType();
3485 if (Node->getConstantOperandVal(1)) {
3486 // 1 -> Hi
3487 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3488 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3489 TLI.getShiftAmountTy(
3490 Node->getOperand(0).getValueType(),
3491 DAG.getDataLayout())));
3492 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3493 } else {
3494 // 0 -> Lo
3495 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3496 Node->getOperand(0));
3497 }
3498 Results.push_back(Tmp1);
3499 break;
3500 }
3501 case ISD::STACKSAVE:
3502 // Expand to CopyFromReg if the target set
3503 // StackPointerRegisterToSaveRestore.
3504 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3505 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3506 Node->getValueType(0)));
3507 Results.push_back(Results[0].getValue(1));
3508 } else {
3509 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3510 Results.push_back(Node->getOperand(0));
3511 }
3512 break;
3513 case ISD::STACKRESTORE:
3514 // Expand to CopyToReg if the target set
3515 // StackPointerRegisterToSaveRestore.
3516 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3517 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3518 Node->getOperand(1)));
3519 } else {
3520 Results.push_back(Node->getOperand(0));
3521 }
3522 break;
3523 case ISD::GET_DYNAMIC_AREA_OFFSET:
3524 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3525 Results.push_back(Results[0].getValue(0));
3526 break;
3527 case ISD::FCOPYSIGN:
3528 Results.push_back(ExpandFCOPYSIGN(Node));
3529 break;
3530 case ISD::FNEG:
3531 Results.push_back(ExpandFNEG(Node));
3532 break;
3533 case ISD::FABS:
3534 Results.push_back(ExpandFABS(Node));
3535 break;
3536 case ISD::IS_FPCLASS: {
3537 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1));
3538 if (SDValue Expanded =
3539 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3540 Test, Node->getFlags(), SDLoc(Node), DAG))
3541 Results.push_back(Expanded);
3542 break;
3543 }
3544 case ISD::SMIN:
3545 case ISD::SMAX:
3546 case ISD::UMIN:
3547 case ISD::UMAX: {
3548 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3549 ISD::CondCode Pred;
3550 switch (Node->getOpcode()) {
3551 default: llvm_unreachable("How did we get here?");
3552 case ISD::SMAX: Pred = ISD::SETGT; break;
3553 case ISD::SMIN: Pred = ISD::SETLT; break;
3554 case ISD::UMAX: Pred = ISD::SETUGT; break;
3555 case ISD::UMIN: Pred = ISD::SETULT; break;
3556 }
3557 Tmp1 = Node->getOperand(0);
3558 Tmp2 = Node->getOperand(1);
3559 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3560 Results.push_back(Tmp1);
3561 break;
3562 }
3563 case ISD::FMINNUM:
3564 case ISD::FMAXNUM: {
3565 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3566 Results.push_back(Expanded);
3567 break;
3568 }
3569 case ISD::FSIN:
3570 case ISD::FCOS: {
3571 EVT VT = Node->getValueType(0);
3572 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3573 // fcos which share the same operand and both are used.
3574 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3575 isSinCosLibcallAvailable(Node, TLI))
3576 && useSinCos(Node)) {
3577 SDVTList VTs = DAG.getVTList(VT, VT);
3578 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3579 if (Node->getOpcode() == ISD::FCOS)
3580 Tmp1 = Tmp1.getValue(1);
3581 Results.push_back(Tmp1);
3582 }
3583 break;
3584 }
3585 case ISD::FLDEXP:
3586 case ISD::STRICT_FLDEXP: {
3587 EVT VT = Node->getValueType(0);
3588 RTLIB::Libcall LC = RTLIB::getLDEXP(VT);
3589 // Use the LibCall instead, it is very likely faster
3590 // FIXME: Use separate LibCall action.
3591 if (TLI.getLibcallName(LC))
3592 break;
3593
3594 if (SDValue Expanded = expandLdexp(Node)) {
3595 Results.push_back(Expanded);
3596 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3597 Results.push_back(Expanded.getValue(1));
3598 }
3599
3600 break;
3601 }
3602 case ISD::FFREXP: {
3603 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));
3604 // Use the LibCall instead, it is very likely faster
3605 // FIXME: Use separate LibCall action.
3606 if (TLI.getLibcallName(LC))
3607 break;
3608
3609 if (SDValue Expanded = expandFrexp(Node)) {
3610 Results.push_back(Expanded);
3611 Results.push_back(Expanded.getValue(1));
3612 }
3613 break;
3614 }
3615 case ISD::FMAD:
3616 llvm_unreachable("Illegal fmad should never be formed");
3617
3618 case ISD::FP16_TO_FP:
3619 if (Node->getValueType(0) != MVT::f32) {
3620 // We can extend to types bigger than f32 in two steps without changing
3621 // the result. Since "f16 -> f32" is much more commonly available, give
3622 // CodeGen the option of emitting that before resorting to a libcall.
3623 SDValue Res =
3624 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3625 Results.push_back(
3626 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3627 }
3628 break;
3629 case ISD::STRICT_FP16_TO_FP:
3630 if (Node->getValueType(0) != MVT::f32) {
3631 // We can extend to types bigger than f32 in two steps without changing
3632 // the result. Since "f16 -> f32" is much more commonly available, give
3633 // CodeGen the option of emitting that before resorting to a libcall.
3634 SDValue Res =
3635 DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other},
3636 {Node->getOperand(0), Node->getOperand(1)});
3637 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3638 {Node->getValueType(0), MVT::Other},
3639 {Res.getValue(1), Res});
3640 Results.push_back(Res);
3641 Results.push_back(Res.getValue(1));
3642 }
3643 break;
3644 case ISD::FP_TO_FP16:
3645 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3646 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3647 SDValue Op = Node->getOperand(0);
3648 MVT SVT = Op.getSimpleValueType();
3649 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3650 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3651 // Under fastmath, we can expand this node into a fround followed by
3652 // a float-half conversion.
3653 SDValue FloatVal =
3654 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3655 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3656 Results.push_back(
3657 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3658 }
3659 }
3660 break;
3661 case ISD::ConstantFP: {
3662 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3663 // Check to see if this FP immediate is already legal.
3664 // If this is a legal constant, turn it into a TargetConstantFP node.
3665 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3666 DAG.shouldOptForSize()))
3667 Results.push_back(ExpandConstantFP(CFP, true));
3668 break;
3669 }
3670 case ISD::Constant: {
3671 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3672 Results.push_back(ExpandConstant(CP));
3673 break;
3674 }
3675 case ISD::FSUB: {
3676 EVT VT = Node->getValueType(0);
3677 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3678 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3679 const SDNodeFlags Flags = Node->getFlags();
3680 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3681 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3682 Results.push_back(Tmp1);
3683 }
3684 break;
3685 }
3686 case ISD::SUB: {
3687 EVT VT = Node->getValueType(0);
3688 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3689 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3690 "Don't know how to expand this subtraction!");
3691 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3692 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3693 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3694 break;
3695 }
3696 case ISD::UREM:
3697 case ISD::SREM:
3698 if (TLI.expandREM(Node, Tmp1, DAG))
3699 Results.push_back(Tmp1);
3700 break;
3701 case ISD::UDIV:
3702 case ISD::SDIV: {
3703 bool isSigned = Node->getOpcode() == ISD::SDIV;
3704 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3705 EVT VT = Node->getValueType(0);
3706 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3707 SDVTList VTs = DAG.getVTList(VT, VT);
3708 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3709 Node->getOperand(1));
3710 Results.push_back(Tmp1);
3711 }
3712 break;
3713 }
3714 case ISD::MULHU:
3715 case ISD::MULHS: {
3716 unsigned ExpandOpcode =
3717 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3718 EVT VT = Node->getValueType(0);
3719 SDVTList VTs = DAG.getVTList(VT, VT);
3720
3721 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3722 Node->getOperand(1));
3723 Results.push_back(Tmp1.getValue(1));
3724 break;
3725 }
3726 case ISD::UMUL_LOHI:
3727 case ISD::SMUL_LOHI: {
3728 SDValue LHS = Node->getOperand(0);
3729 SDValue RHS = Node->getOperand(1);
3730 MVT VT = LHS.getSimpleValueType();
3731 unsigned MULHOpcode =
3732 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3733
3734 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3735 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3736 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3737 break;
3738 }
3739
3740 SmallVector<SDValue, 4> Halves;
3741 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3742 assert(TLI.isTypeLegal(HalfType));
3743 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3744 HalfType, DAG,
3745 TargetLowering::MulExpansionKind::Always)) {
3746 for (unsigned i = 0; i < 2; ++i) {
3747 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3748 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3749 SDValue Shift = DAG.getConstant(
3750 HalfType.getScalarSizeInBits(), dl,
3751 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3752 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3753 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3754 }
3755 break;
3756 }
3757 break;
3758 }
3759 case ISD::MUL: {
3760 EVT VT = Node->getValueType(0);
3761 SDVTList VTs = DAG.getVTList(VT, VT);
3762 // See if multiply or divide can be lowered using two-result operations.
3763 // We just need the low half of the multiply; try both the signed
3764 // and unsigned forms. If the target supports both SMUL_LOHI and
3765 // UMUL_LOHI, form a preference by checking which forms of plain
3766 // MULH it supports.
3767 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3768 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3769 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3770 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3771 unsigned OpToUse = 0;
3772 if (HasSMUL_LOHI && !HasMULHS) {
3773 OpToUse = ISD::SMUL_LOHI;
3774 } else if (HasUMUL_LOHI && !HasMULHU) {
3775 OpToUse = ISD::UMUL_LOHI;
3776 } else if (HasSMUL_LOHI) {
3777 OpToUse = ISD::SMUL_LOHI;
3778 } else if (HasUMUL_LOHI) {
3779 OpToUse = ISD::UMUL_LOHI;
3780 }
3781 if (OpToUse) {
3782 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3783 Node->getOperand(1)));
3784 break;
3785 }
3786
3787 SDValue Lo, Hi;
3788 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3789 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3790 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3791 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3792 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3793 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3794 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3795 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3796 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3797 SDValue Shift =
3798 DAG.getConstant(HalfType.getSizeInBits(), dl,
3799 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3800 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3801 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3802 }
3803 break;
3804 }
3805 case ISD::FSHL:
3806 case ISD::FSHR:
3807 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3808 Results.push_back(Expanded);
3809 break;
3810 case ISD::ROTL:
3811 case ISD::ROTR:
3812 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3813 Results.push_back(Expanded);
3814 break;
3815 case ISD::SADDSAT:
3816 case ISD::UADDSAT:
3817 case ISD::SSUBSAT:
3818 case ISD::USUBSAT:
3819 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3820 break;
3821 case ISD::SSHLSAT:
3822 case ISD::USHLSAT:
3823 Results.push_back(TLI.expandShlSat(Node, DAG));
3824 break;
3825 case ISD::SMULFIX:
3826 case ISD::SMULFIXSAT:
3827 case ISD::UMULFIX:
3828 case ISD::UMULFIXSAT:
3829 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3830 break;
3831 case ISD::SDIVFIX:
3832 case ISD::SDIVFIXSAT:
3833 case ISD::UDIVFIX:
3834 case ISD::UDIVFIXSAT:
3835 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3836 Node->getOperand(0),
3837 Node->getOperand(1),
3838 Node->getConstantOperandVal(2),
3839 DAG)) {
3840 Results.push_back(V);
3841 break;
3842 }
3843 // FIXME: We might want to retry here with a wider type if we fail, if that
3844 // type is legal.
3845 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3846 // <= 128 (which is the case for all of the default Embedded-C types),
3847 // we will only get here with types and scales that we could always expand
3848 // if we were allowed to generate libcalls to division functions of illegal
3849 // type. But we cannot do that.
3850 llvm_unreachable("Cannot expand DIVFIX!");
3851 case ISD::UADDO_CARRY:
3852 case ISD::USUBO_CARRY: {
3853 SDValue LHS = Node->getOperand(0);
3854 SDValue RHS = Node->getOperand(1);
3855 SDValue Carry = Node->getOperand(2);
3856
3857 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3858
3859 // Initial add of the 2 operands.
3860 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3861 EVT VT = LHS.getValueType();
3862 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3863
3864 // Initial check for overflow.
3865 EVT CarryType = Node->getValueType(1);
3866 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3867 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3868 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3869
3870 // Add of the sum and the carry.
3871 SDValue One = DAG.getConstant(1, dl, VT);
3872 SDValue CarryExt =
3873 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3874 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3875
3876 // Second check for overflow. If we are adding, we can only overflow if the
3877 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3878 // If we are subtracting, we can only overflow if the initial sum is 0 and
3879 // the carry is set, resulting in a new sum of all 1s.
3880 SDValue Zero = DAG.getConstant(0, dl, VT);
3881 SDValue Overflow2 =
3882 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3883 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3884 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3885 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3886
3887 SDValue ResultCarry =
3888 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3889
3890 Results.push_back(Sum2);
3891 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3892 break;
3893 }
3894 case ISD::SADDO:
3895 case ISD::SSUBO: {
3896 SDValue Result, Overflow;
3897 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3898 Results.push_back(Result);
3899 Results.push_back(Overflow);
3900 break;
3901 }
3902 case ISD::UADDO:
3903 case ISD::USUBO: {
3904 SDValue Result, Overflow;
3905 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3906 Results.push_back(Result);
3907 Results.push_back(Overflow);
3908 break;
3909 }
3910 case ISD::UMULO:
3911 case ISD::SMULO: {
3912 SDValue Result, Overflow;
3913 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3914 Results.push_back(Result);
3915 Results.push_back(Overflow);
3916 }
3917 break;
3918 }
3919 case ISD::BUILD_PAIR: {
3920 EVT PairTy = Node->getValueType(0);
3921 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3922 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3923 Tmp2 = DAG.getNode(
3924 ISD::SHL, dl, PairTy, Tmp2,
3925 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3926 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3927 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3928 break;
3929 }
3930 case ISD::SELECT:
3931 Tmp1 = Node->getOperand(0);
3932 Tmp2 = Node->getOperand(1);
3933 Tmp3 = Node->getOperand(2);
3934 if (Tmp1.getOpcode() == ISD::SETCC) {
3935 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3936 Tmp2, Tmp3,
3937 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3938 } else {
3939 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3940 DAG.getConstant(0, dl, Tmp1.getValueType()),
3941 Tmp2, Tmp3, ISD::SETNE);
3942 }
3943 Tmp1->setFlags(Node->getFlags());
3944 Results.push_back(Tmp1);
3945 break;
3946 case ISD::BR_JT: {
3947 SDValue Chain = Node->getOperand(0);
3948 SDValue Table = Node->getOperand(1);
3949 SDValue Index = Node->getOperand(2);
3950 int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex();
3951
3952 const DataLayout &TD = DAG.getDataLayout();
3953 EVT PTy = TLI.getPointerTy(TD);
3954
3955 unsigned EntrySize =
3956 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3957
3958 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3959 // This transformation needs to be done here since otherwise the MIPS
3960 // backend will end up emitting a three instruction multiply sequence
3961 // instead of a single shift and MSP430 will call a runtime function.
3962 if (llvm::isPowerOf2_32(EntrySize))
3963 Index = DAG.getNode(
3964 ISD::SHL, dl, Index.getValueType(), Index,
3965 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3966 else
3967 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3968 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3969 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3970 Index, Table);
3971
3972 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3973 SDValue LD = DAG.getExtLoad(
3974 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3975 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3976 Addr = LD;
3977 if (TLI.isJumpTableRelative()) {
3978 // For PIC, the sequence is:
3979 // BRIND(load(Jumptable + index) + RelocBase)
3980 // RelocBase can be JumpTable, GOT or some sort of global base.
3981 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3982 TLI.getPICJumpTableRelocBase(Table, DAG));
3983 }
3984
3985 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);
3986 Results.push_back(Tmp1);
3987 break;
3988 }
3989 case ISD::BRCOND:
3990 // Expand brcond's setcc into its constituent parts and create a BR_CC
3991 // Node.
3992 Tmp1 = Node->getOperand(0);
3993 Tmp2 = Node->getOperand(1);
3994 if (Tmp2.getOpcode() == ISD::SETCC &&
3995 TLI.isOperationLegalOrCustom(ISD::BR_CC,
3996 Tmp2.getOperand(0).getValueType())) {
3997 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
3998 Tmp2.getOperand(0), Tmp2.getOperand(1),
3999 Node->getOperand(2));
4000 } else {
4001 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4002 if (Tmp2.isUndef() ||
4003 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1))))
4004 Tmp3 = Tmp2;
4005 else
4006 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
4007 DAG.getConstant(1, dl, Tmp2.getValueType()));
4008 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
4009 DAG.getCondCode(ISD::SETNE), Tmp3,
4010 DAG.getConstant(0, dl, Tmp3.getValueType()),
4011 Node->getOperand(2));
4012 }
4013 Results.push_back(Tmp1);
4014 break;
4015 case ISD::SETCC:
4016 case ISD::VP_SETCC:
4017 case ISD::STRICT_FSETCC:
4018 case ISD::STRICT_FSETCCS: {
4019 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4020 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4021 Node->getOpcode() == ISD::STRICT_FSETCCS;
4022 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4023 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4024 unsigned Offset = IsStrict ? 1 : 0;
4025 Tmp1 = Node->getOperand(0 + Offset);
4026 Tmp2 = Node->getOperand(1 + Offset);
4027 Tmp3 = Node->getOperand(2 + Offset);
4028 SDValue Mask, EVL;
4029 if (IsVP) {
4030 Mask = Node->getOperand(3 + Offset);
4031 EVL = Node->getOperand(4 + Offset);
4032 }
4033 bool Legalized = TLI.LegalizeSetCCCondCode(
4034 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4035 Chain, IsSignaling);
4036
4037 if (Legalized) {
4038 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4039 // condition code, create a new SETCC node.
4040 if (Tmp3.getNode()) {
4041 if (IsStrict) {
4042 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
4043 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
4044 Chain = Tmp1.getValue(1);
4045 } else if (IsVP) {
4046 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
4047 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
4048 } else {
4049 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
4050 Tmp2, Tmp3, Node->getFlags());
4051 }
4052 }
4053
4054 // If we expanded the SETCC by inverting the condition code, then wrap
4055 // the existing SETCC in a NOT to restore the intended condition.
4056 if (NeedInvert) {
4057 if (!IsVP)
4058 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
4059 else
4060 Tmp1 =
4061 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
4062 }
4063
4064 Results.push_back(Tmp1);
4065 if (IsStrict)
4066 Results.push_back(Chain);
4067
4068 break;
4069 }
4070
4071 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4072 // understand if this code is useful for strict nodes.
4073 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4074
4075 // Otherwise, SETCC for the given comparison type must be completely
4076 // illegal; expand it into a SELECT_CC.
4077 // FIXME: This drops the mask/evl for VP_SETCC.
4078 EVT VT = Node->getValueType(0);
4079 EVT Tmp1VT = Tmp1.getValueType();
4080 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
4081 DAG.getBoolConstant(true, dl, VT, Tmp1VT),
4082 DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
4083 Tmp1->setFlags(Node->getFlags());
4084 Results.push_back(Tmp1);
4085 break;
4086 }
4087 case ISD::SELECT_CC: {
4088 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4089 Tmp1 = Node->getOperand(0); // LHS
4090 Tmp2 = Node->getOperand(1); // RHS
4091 Tmp3 = Node->getOperand(2); // True
4092 Tmp4 = Node->getOperand(3); // False
4093 EVT VT = Node->getValueType(0);
4094 SDValue Chain;
4095 SDValue CC = Node->getOperand(4);
4096 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
4097
4098 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
4099 // If the condition code is legal, then we need to expand this
4100 // node using SETCC and SELECT.
4101 EVT CmpVT = Tmp1.getValueType();
4102 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4103 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4104 "expanded.");
4105 EVT CCVT = getSetCCResultType(CmpVT);
4106 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
4107 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
4108 break;
4109 }
4110
4111 // SELECT_CC is legal, so the condition code must not be.
4112 bool Legalized = false;
4113 // Try to legalize by inverting the condition. This is for targets that
4114 // might support an ordered version of a condition, but not the unordered
4115 // version (or vice versa).
4116 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
4117 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
4118 // Use the new condition code and swap true and false
4119 Legalized = true;
4120 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4121 Tmp1->setFlags(Node->getFlags());
4122 } else {
4123 // If The inverse is not legal, then try to swap the arguments using
4124 // the inverse condition code.
4125 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
4126 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
4127 // The swapped inverse condition is legal, so swap true and false,
4128 // lhs and rhs.
4129 Legalized = true;
4130 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4131 Tmp1->setFlags(Node->getFlags());
4132 }
4133 }
4134
4135 if (!Legalized) {
4136 Legalized = TLI.LegalizeSetCCCondCode(
4137 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
4138 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4139
4140 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4141
4142 // If we expanded the SETCC by inverting the condition code, then swap
4143 // the True/False operands to match.
4144 if (NeedInvert)
4145 std::swap(Tmp3, Tmp4);
4146
4147 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4148 // condition code, create a new SELECT_CC node.
4149 if (CC.getNode()) {
4150 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
4151 Tmp1, Tmp2, Tmp3, Tmp4, CC);
4152 } else {
4153 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
4154 CC = DAG.getCondCode(ISD::SETNE);
4155 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
4156 Tmp2, Tmp3, Tmp4, CC);
4157 }
4158 Tmp1->setFlags(Node->getFlags());
4159 }
4160 Results.push_back(Tmp1);
4161 break;
4162 }
4163 case ISD::BR_CC: {
4164 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4165 SDValue Chain;
4166 Tmp1 = Node->getOperand(0); // Chain
4167 Tmp2 = Node->getOperand(2); // LHS
4168 Tmp3 = Node->getOperand(3); // RHS
4169 Tmp4 = Node->getOperand(1); // CC
4170
4171 bool Legalized = TLI.LegalizeSetCCCondCode(
4172 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
4173 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4174 (void)Legalized;
4175 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4176
4177 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4178 // node.
4179 if (Tmp4.getNode()) {
4180 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4181
4182 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4183 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4184 } else {
4185 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
4186 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
4187 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4188 Tmp2, Tmp3, Node->getOperand(4));
4189 }
4190 Results.push_back(Tmp1);
4191 break;
4192 }
4193 case ISD::BUILD_VECTOR:
4194 Results.push_back(ExpandBUILD_VECTOR(Node));
4195 break;
4196 case ISD::SPLAT_VECTOR:
4197 Results.push_back(ExpandSPLAT_VECTOR(Node));
4198 break;
4199 case ISD::SRA:
4200 case ISD::SRL:
4201 case ISD::SHL: {
4202 // Scalarize vector SRA/SRL/SHL.
4203 EVT VT = Node->getValueType(0);
4204 assert(VT.isVector() && "Unable to legalize non-vector shift");
4205 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4206 unsigned NumElem = VT.getVectorNumElements();
4207
4208 SmallVector<SDValue, 8> Scalars;
4209 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4210 SDValue Ex =
4211 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(),
4212 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
4213 SDValue Sh =
4214 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(),
4215 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
4216 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
4217 VT.getScalarType(), Ex, Sh));
4218 }
4219
4220 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
4221 Results.push_back(Result);
4222 break;
4223 }
4224 case ISD::VECREDUCE_FADD:
4225 case ISD::VECREDUCE_FMUL:
4226 case ISD::VECREDUCE_ADD:
4227 case ISD::VECREDUCE_MUL:
4228 case ISD::VECREDUCE_AND:
4229 case ISD::VECREDUCE_OR:
4230 case ISD::VECREDUCE_XOR:
4231 case ISD::VECREDUCE_SMAX:
4232 case ISD::VECREDUCE_SMIN:
4233 case ISD::VECREDUCE_UMAX:
4234 case ISD::VECREDUCE_UMIN:
4235 case ISD::VECREDUCE_FMAX:
4236 case ISD::VECREDUCE_FMIN:
4237 case ISD::VECREDUCE_FMAXIMUM:
4238 case ISD::VECREDUCE_FMINIMUM:
4239 Results.push_back(TLI.expandVecReduce(Node, DAG));
4240 break;
4241 case ISD::GLOBAL_OFFSET_TABLE:
4242 case ISD::GlobalAddress:
4243 case ISD::GlobalTLSAddress:
4244 case ISD::ExternalSymbol:
4245 case ISD::ConstantPool:
4246 case ISD::JumpTable:
4247 case ISD::INTRINSIC_W_CHAIN:
4248 case ISD::INTRINSIC_WO_CHAIN:
4249 case ISD::INTRINSIC_VOID:
4250 // FIXME: Custom lowering for these operations shouldn't return null!
4251 // Return true so that we don't call ConvertNodeToLibcall which also won't
4252 // do anything.
4253 return true;
4254 }
4255
4256 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4257 // FIXME: We were asked to expand a strict floating-point operation,
4258 // but there is currently no expansion implemented that would preserve
4259 // the "strict" properties. For now, we just fall back to the non-strict
4260 // version if that is legal on the target. The actual mutation of the
4261 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4262 switch (Node->getOpcode()) {
4263 default:
4264 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4265 Node->getValueType(0))
4266 == TargetLowering::Legal)
4267 return true;
4268 break;
4269 case ISD::STRICT_FSUB: {
4270 if (TLI.getStrictFPOperationAction(
4271 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
4272 return true;
4273 if (TLI.getStrictFPOperationAction(
4274 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
4275 break;
4276
4277 EVT VT = Node->getValueType(0);
4278 const SDNodeFlags Flags = Node->getFlags();
4279 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
4280 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
4281 {Node->getOperand(0), Node->getOperand(1), Neg},
4282 Flags);
4283
4284 Results.push_back(Fadd);
4285 Results.push_back(Fadd.getValue(1));
4286 break;
4287 }
4288 case ISD::STRICT_SINT_TO_FP:
4289 case ISD::STRICT_UINT_TO_FP:
4290 case ISD::STRICT_LRINT:
4291 case ISD::STRICT_LLRINT:
4292 case ISD::STRICT_LROUND:
4293 case ISD::STRICT_LLROUND:
4294 // These are registered by the operand type instead of the value
4295 // type. Reflect that here.
4296 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4297 Node->getOperand(1).getValueType())
4298 == TargetLowering::Legal)
4299 return true;
4300 break;
4301 }
4302 }
4303
4304 // Replace the original node with the legalized result.
4305 if (Results.empty()) {
4306 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4307 return false;
4308 }
4309
4310 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4311 ReplaceNode(Node, Results.data());
4312 return true;
4313 }
4314
ConvertNodeToLibcall(SDNode * Node)4315 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4316 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4317 SmallVector<SDValue, 8> Results;
4318 SDLoc dl(Node);
4319 // FIXME: Check flags on the node to see if we can use a finite call.
4320 unsigned Opc = Node->getOpcode();
4321 switch (Opc) {
4322 case ISD::ATOMIC_FENCE: {
4323 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4324 // FIXME: handle "fence singlethread" more efficiently.
4325 TargetLowering::ArgListTy Args;
4326
4327 TargetLowering::CallLoweringInfo CLI(DAG);
4328 CLI.setDebugLoc(dl)
4329 .setChain(Node->getOperand(0))
4330 .setLibCallee(
4331 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4332 DAG.getExternalSymbol("__sync_synchronize",
4333 TLI.getPointerTy(DAG.getDataLayout())),
4334 std::move(Args));
4335
4336 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4337
4338 Results.push_back(CallResult.second);
4339 break;
4340 }
4341 // By default, atomic intrinsics are marked Legal and lowered. Targets
4342 // which don't support them directly, however, may want libcalls, in which
4343 // case they mark them Expand, and we get here.
4344 case ISD::ATOMIC_SWAP:
4345 case ISD::ATOMIC_LOAD_ADD:
4346 case ISD::ATOMIC_LOAD_SUB:
4347 case ISD::ATOMIC_LOAD_AND:
4348 case ISD::ATOMIC_LOAD_CLR:
4349 case ISD::ATOMIC_LOAD_OR:
4350 case ISD::ATOMIC_LOAD_XOR:
4351 case ISD::ATOMIC_LOAD_NAND:
4352 case ISD::ATOMIC_LOAD_MIN:
4353 case ISD::ATOMIC_LOAD_MAX:
4354 case ISD::ATOMIC_LOAD_UMIN:
4355 case ISD::ATOMIC_LOAD_UMAX:
4356 case ISD::ATOMIC_CMP_SWAP: {
4357 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4358 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
4359 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4360 EVT RetVT = Node->getValueType(0);
4361 TargetLowering::MakeLibCallOptions CallOptions;
4362 SmallVector<SDValue, 4> Ops;
4363 if (TLI.getLibcallName(LC)) {
4364 // If outline atomic available, prepare its arguments and expand.
4365 Ops.append(Node->op_begin() + 2, Node->op_end());
4366 Ops.push_back(Node->getOperand(1));
4367
4368 } else {
4369 LC = RTLIB::getSYNC(Opc, VT);
4370 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4371 "Unexpected atomic op or value type!");
4372 // Arguments for expansion to sync libcall
4373 Ops.append(Node->op_begin() + 1, Node->op_end());
4374 }
4375 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4376 Ops, CallOptions,
4377 SDLoc(Node),
4378 Node->getOperand(0));
4379 Results.push_back(Tmp.first);
4380 Results.push_back(Tmp.second);
4381 break;
4382 }
4383 case ISD::TRAP: {
4384 // If this operation is not supported, lower it to 'abort()' call
4385 TargetLowering::ArgListTy Args;
4386 TargetLowering::CallLoweringInfo CLI(DAG);
4387 CLI.setDebugLoc(dl)
4388 .setChain(Node->getOperand(0))
4389 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4390 DAG.getExternalSymbol(
4391 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4392 std::move(Args));
4393 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4394
4395 Results.push_back(CallResult.second);
4396 break;
4397 }
4398 case ISD::FMINNUM:
4399 case ISD::STRICT_FMINNUM:
4400 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4401 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4402 RTLIB::FMIN_PPCF128, Results);
4403 break;
4404 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4405 // libcall legalization for these nodes, but there is no default expasion for
4406 // these nodes either (see PR63267 for example).
4407 case ISD::FMAXNUM:
4408 case ISD::STRICT_FMAXNUM:
4409 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4410 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4411 RTLIB::FMAX_PPCF128, Results);
4412 break;
4413 case ISD::FSQRT:
4414 case ISD::STRICT_FSQRT:
4415 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4416 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4417 RTLIB::SQRT_PPCF128, Results);
4418 break;
4419 case ISD::FCBRT:
4420 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4421 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4422 RTLIB::CBRT_PPCF128, Results);
4423 break;
4424 case ISD::FSIN:
4425 case ISD::STRICT_FSIN:
4426 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4427 RTLIB::SIN_F80, RTLIB::SIN_F128,
4428 RTLIB::SIN_PPCF128, Results);
4429 break;
4430 case ISD::FCOS:
4431 case ISD::STRICT_FCOS:
4432 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4433 RTLIB::COS_F80, RTLIB::COS_F128,
4434 RTLIB::COS_PPCF128, Results);
4435 break;
4436 case ISD::FSINCOS:
4437 // Expand into sincos libcall.
4438 ExpandSinCosLibCall(Node, Results);
4439 break;
4440 case ISD::FLOG:
4441 case ISD::STRICT_FLOG:
4442 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4443 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4444 break;
4445 case ISD::FLOG2:
4446 case ISD::STRICT_FLOG2:
4447 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4448 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4449 break;
4450 case ISD::FLOG10:
4451 case ISD::STRICT_FLOG10:
4452 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4453 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4454 break;
4455 case ISD::FEXP:
4456 case ISD::STRICT_FEXP:
4457 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4458 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4459 break;
4460 case ISD::FEXP2:
4461 case ISD::STRICT_FEXP2:
4462 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4463 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4464 break;
4465 case ISD::FEXP10:
4466 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4467 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);
4468 break;
4469 case ISD::FTRUNC:
4470 case ISD::STRICT_FTRUNC:
4471 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4472 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4473 RTLIB::TRUNC_PPCF128, Results);
4474 break;
4475 case ISD::FFLOOR:
4476 case ISD::STRICT_FFLOOR:
4477 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4478 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4479 RTLIB::FLOOR_PPCF128, Results);
4480 break;
4481 case ISD::FCEIL:
4482 case ISD::STRICT_FCEIL:
4483 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4484 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4485 RTLIB::CEIL_PPCF128, Results);
4486 break;
4487 case ISD::FRINT:
4488 case ISD::STRICT_FRINT:
4489 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4490 RTLIB::RINT_F80, RTLIB::RINT_F128,
4491 RTLIB::RINT_PPCF128, Results);
4492 break;
4493 case ISD::FNEARBYINT:
4494 case ISD::STRICT_FNEARBYINT:
4495 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4496 RTLIB::NEARBYINT_F64,
4497 RTLIB::NEARBYINT_F80,
4498 RTLIB::NEARBYINT_F128,
4499 RTLIB::NEARBYINT_PPCF128, Results);
4500 break;
4501 case ISD::FROUND:
4502 case ISD::STRICT_FROUND:
4503 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4504 RTLIB::ROUND_F64,
4505 RTLIB::ROUND_F80,
4506 RTLIB::ROUND_F128,
4507 RTLIB::ROUND_PPCF128, Results);
4508 break;
4509 case ISD::FROUNDEVEN:
4510 case ISD::STRICT_FROUNDEVEN:
4511 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4512 RTLIB::ROUNDEVEN_F64,
4513 RTLIB::ROUNDEVEN_F80,
4514 RTLIB::ROUNDEVEN_F128,
4515 RTLIB::ROUNDEVEN_PPCF128, Results);
4516 break;
4517 case ISD::FLDEXP:
4518 case ISD::STRICT_FLDEXP:
4519 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4520 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
4521 break;
4522 case ISD::FFREXP: {
4523 ExpandFrexpLibCall(Node, Results);
4524 break;
4525 }
4526 case ISD::FPOWI:
4527 case ISD::STRICT_FPOWI: {
4528 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4529 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4530 if (!TLI.getLibcallName(LC)) {
4531 // Some targets don't have a powi libcall; use pow instead.
4532 if (Node->isStrictFPOpcode()) {
4533 SDValue Exponent =
4534 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4535 {Node->getValueType(0), Node->getValueType(1)},
4536 {Node->getOperand(0), Node->getOperand(2)});
4537 SDValue FPOW =
4538 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4539 {Node->getValueType(0), Node->getValueType(1)},
4540 {Exponent.getValue(1), Node->getOperand(1), Exponent});
4541 Results.push_back(FPOW);
4542 Results.push_back(FPOW.getValue(1));
4543 } else {
4544 SDValue Exponent =
4545 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4546 Node->getOperand(1));
4547 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4548 Node->getValueType(0),
4549 Node->getOperand(0), Exponent));
4550 }
4551 break;
4552 }
4553 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4554 bool ExponentHasSizeOfInt =
4555 DAG.getLibInfo().getIntSize() ==
4556 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4557 if (!ExponentHasSizeOfInt) {
4558 // If the exponent does not match with sizeof(int) a libcall to
4559 // RTLIB::POWI would use the wrong type for the argument.
4560 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4561 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4562 break;
4563 }
4564 ExpandFPLibCall(Node, LC, Results);
4565 break;
4566 }
4567 case ISD::FPOW:
4568 case ISD::STRICT_FPOW:
4569 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4570 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4571 break;
4572 case ISD::LROUND:
4573 case ISD::STRICT_LROUND:
4574 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4575 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4576 RTLIB::LROUND_F128,
4577 RTLIB::LROUND_PPCF128, Results);
4578 break;
4579 case ISD::LLROUND:
4580 case ISD::STRICT_LLROUND:
4581 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4582 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4583 RTLIB::LLROUND_F128,
4584 RTLIB::LLROUND_PPCF128, Results);
4585 break;
4586 case ISD::LRINT:
4587 case ISD::STRICT_LRINT:
4588 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4589 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4590 RTLIB::LRINT_F128,
4591 RTLIB::LRINT_PPCF128, Results);
4592 break;
4593 case ISD::LLRINT:
4594 case ISD::STRICT_LLRINT:
4595 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4596 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4597 RTLIB::LLRINT_F128,
4598 RTLIB::LLRINT_PPCF128, Results);
4599 break;
4600 case ISD::FDIV:
4601 case ISD::STRICT_FDIV:
4602 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4603 RTLIB::DIV_F80, RTLIB::DIV_F128,
4604 RTLIB::DIV_PPCF128, Results);
4605 break;
4606 case ISD::FREM:
4607 case ISD::STRICT_FREM:
4608 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4609 RTLIB::REM_F80, RTLIB::REM_F128,
4610 RTLIB::REM_PPCF128, Results);
4611 break;
4612 case ISD::FMA:
4613 case ISD::STRICT_FMA:
4614 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4615 RTLIB::FMA_F80, RTLIB::FMA_F128,
4616 RTLIB::FMA_PPCF128, Results);
4617 break;
4618 case ISD::FADD:
4619 case ISD::STRICT_FADD:
4620 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4621 RTLIB::ADD_F80, RTLIB::ADD_F128,
4622 RTLIB::ADD_PPCF128, Results);
4623 break;
4624 case ISD::FMUL:
4625 case ISD::STRICT_FMUL:
4626 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4627 RTLIB::MUL_F80, RTLIB::MUL_F128,
4628 RTLIB::MUL_PPCF128, Results);
4629 break;
4630 case ISD::FP16_TO_FP:
4631 if (Node->getValueType(0) == MVT::f32) {
4632 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);
4633 }
4634 break;
4635 case ISD::STRICT_FP16_TO_FP: {
4636 if (Node->getValueType(0) == MVT::f32) {
4637 TargetLowering::MakeLibCallOptions CallOptions;
4638 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4639 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4640 SDLoc(Node), Node->getOperand(0));
4641 Results.push_back(Tmp.first);
4642 Results.push_back(Tmp.second);
4643 }
4644 break;
4645 }
4646 case ISD::FP_TO_FP16: {
4647 RTLIB::Libcall LC =
4648 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4649 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4650 Results.push_back(ExpandLibCall(LC, Node, false).first);
4651 break;
4652 }
4653 case ISD::FP_TO_BF16: {
4654 RTLIB::Libcall LC =
4655 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);
4656 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4657 Results.push_back(ExpandLibCall(LC, Node, false).first);
4658 break;
4659 }
4660 case ISD::STRICT_SINT_TO_FP:
4661 case ISD::STRICT_UINT_TO_FP:
4662 case ISD::SINT_TO_FP:
4663 case ISD::UINT_TO_FP: {
4664 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4665 bool IsStrict = Node->isStrictFPOpcode();
4666 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4667 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4668 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4669 EVT RVT = Node->getValueType(0);
4670 EVT NVT = EVT();
4671 SDLoc dl(Node);
4672
4673 // Even if the input is legal, no libcall may exactly match, eg. we don't
4674 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4675 // eg: i13 -> fp. Then, look for an appropriate libcall.
4676 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4677 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4678 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4679 ++t) {
4680 NVT = (MVT::SimpleValueType)t;
4681 // The source needs to big enough to hold the operand.
4682 if (NVT.bitsGE(SVT))
4683 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4684 : RTLIB::getUINTTOFP(NVT, RVT);
4685 }
4686 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4687
4688 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4689 // Sign/zero extend the argument if the libcall takes a larger type.
4690 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4691 NVT, Node->getOperand(IsStrict ? 1 : 0));
4692 TargetLowering::MakeLibCallOptions CallOptions;
4693 CallOptions.setSExt(Signed);
4694 std::pair<SDValue, SDValue> Tmp =
4695 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4696 Results.push_back(Tmp.first);
4697 if (IsStrict)
4698 Results.push_back(Tmp.second);
4699 break;
4700 }
4701 case ISD::FP_TO_SINT:
4702 case ISD::FP_TO_UINT:
4703 case ISD::STRICT_FP_TO_SINT:
4704 case ISD::STRICT_FP_TO_UINT: {
4705 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4706 bool IsStrict = Node->isStrictFPOpcode();
4707 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4708 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4709
4710 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4711 EVT SVT = Op.getValueType();
4712 EVT RVT = Node->getValueType(0);
4713 EVT NVT = EVT();
4714 SDLoc dl(Node);
4715
4716 // Even if the result is legal, no libcall may exactly match, eg. we don't
4717 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4718 // eg: fp -> i32. Then, look for an appropriate libcall.
4719 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4720 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4721 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4722 ++IntVT) {
4723 NVT = (MVT::SimpleValueType)IntVT;
4724 // The type needs to big enough to hold the result.
4725 if (NVT.bitsGE(RVT))
4726 LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4727 : RTLIB::getFPTOUINT(SVT, NVT);
4728 }
4729 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4730
4731 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4732 TargetLowering::MakeLibCallOptions CallOptions;
4733 std::pair<SDValue, SDValue> Tmp =
4734 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4735
4736 // Truncate the result if the libcall returns a larger type.
4737 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4738 if (IsStrict)
4739 Results.push_back(Tmp.second);
4740 break;
4741 }
4742
4743 case ISD::FP_ROUND:
4744 case ISD::STRICT_FP_ROUND: {
4745 // X = FP_ROUND(Y, TRUNC)
4746 // TRUNC is a flag, which is always an integer that is zero or one.
4747 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4748 // is known to not change the value of Y.
4749 // We can only expand it into libcall if the TRUNC is 0.
4750 bool IsStrict = Node->isStrictFPOpcode();
4751 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4752 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4753 EVT VT = Node->getValueType(0);
4754 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4755 "Unable to expand as libcall if it is not normal rounding");
4756
4757 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4758 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4759
4760 TargetLowering::MakeLibCallOptions CallOptions;
4761 std::pair<SDValue, SDValue> Tmp =
4762 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4763 Results.push_back(Tmp.first);
4764 if (IsStrict)
4765 Results.push_back(Tmp.second);
4766 break;
4767 }
4768 case ISD::FP_EXTEND: {
4769 Results.push_back(
4770 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4771 Node->getValueType(0)),
4772 Node, false).first);
4773 break;
4774 }
4775 case ISD::STRICT_FP_EXTEND:
4776 case ISD::STRICT_FP_TO_FP16: {
4777 RTLIB::Libcall LC =
4778 Node->getOpcode() == ISD::STRICT_FP_TO_FP16
4779 ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16)
4780 : RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4781 Node->getValueType(0));
4782 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4783
4784 TargetLowering::MakeLibCallOptions CallOptions;
4785 std::pair<SDValue, SDValue> Tmp =
4786 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4787 CallOptions, SDLoc(Node), Node->getOperand(0));
4788 Results.push_back(Tmp.first);
4789 Results.push_back(Tmp.second);
4790 break;
4791 }
4792 case ISD::FSUB:
4793 case ISD::STRICT_FSUB:
4794 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4795 RTLIB::SUB_F80, RTLIB::SUB_F128,
4796 RTLIB::SUB_PPCF128, Results);
4797 break;
4798 case ISD::SREM:
4799 Results.push_back(ExpandIntLibCall(Node, true,
4800 RTLIB::SREM_I8,
4801 RTLIB::SREM_I16, RTLIB::SREM_I32,
4802 RTLIB::SREM_I64, RTLIB::SREM_I128));
4803 break;
4804 case ISD::UREM:
4805 Results.push_back(ExpandIntLibCall(Node, false,
4806 RTLIB::UREM_I8,
4807 RTLIB::UREM_I16, RTLIB::UREM_I32,
4808 RTLIB::UREM_I64, RTLIB::UREM_I128));
4809 break;
4810 case ISD::SDIV:
4811 Results.push_back(ExpandIntLibCall(Node, true,
4812 RTLIB::SDIV_I8,
4813 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4814 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4815 break;
4816 case ISD::UDIV:
4817 Results.push_back(ExpandIntLibCall(Node, false,
4818 RTLIB::UDIV_I8,
4819 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4820 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4821 break;
4822 case ISD::SDIVREM:
4823 case ISD::UDIVREM:
4824 // Expand into divrem libcall
4825 ExpandDivRemLibCall(Node, Results);
4826 break;
4827 case ISD::MUL:
4828 Results.push_back(ExpandIntLibCall(Node, false,
4829 RTLIB::MUL_I8,
4830 RTLIB::MUL_I16, RTLIB::MUL_I32,
4831 RTLIB::MUL_I64, RTLIB::MUL_I128));
4832 break;
4833 case ISD::CTLZ_ZERO_UNDEF:
4834 switch (Node->getSimpleValueType(0).SimpleTy) {
4835 default:
4836 llvm_unreachable("LibCall explicitly requested, but not available");
4837 case MVT::i32:
4838 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false).first);
4839 break;
4840 case MVT::i64:
4841 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false).first);
4842 break;
4843 case MVT::i128:
4844 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false).first);
4845 break;
4846 }
4847 break;
4848 case ISD::RESET_FPENV: {
4849 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
4850 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
4851 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4852 SDValue Chain = Node->getOperand(0);
4853 Results.push_back(
4854 DAG.makeStateFunctionCall(RTLIB::FESETENV, Ptr, Chain, dl));
4855 break;
4856 }
4857 case ISD::GET_FPENV_MEM: {
4858 SDValue Chain = Node->getOperand(0);
4859 SDValue EnvPtr = Node->getOperand(1);
4860 Results.push_back(
4861 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4862 break;
4863 }
4864 case ISD::SET_FPENV_MEM: {
4865 SDValue Chain = Node->getOperand(0);
4866 SDValue EnvPtr = Node->getOperand(1);
4867 Results.push_back(
4868 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4869 break;
4870 }
4871 case ISD::GET_FPMODE: {
4872 // Call fegetmode, which saves control modes into a stack slot. Then load
4873 // the value to return from the stack.
4874 EVT ModeVT = Node->getValueType(0);
4875 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);
4876 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4877 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4878 Node->getOperand(0), dl);
4879 SDValue LdInst = DAG.getLoad(
4880 ModeVT, dl, Chain, StackPtr,
4881 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
4882 Results.push_back(LdInst);
4883 Results.push_back(LdInst.getValue(1));
4884 break;
4885 }
4886 case ISD::SET_FPMODE: {
4887 // Move control modes to stack slot and then call fesetmode with the pointer
4888 // to the slot as argument.
4889 SDValue Mode = Node->getOperand(1);
4890 EVT ModeVT = Mode.getValueType();
4891 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);
4892 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4893 SDValue StInst = DAG.getStore(
4894 Node->getOperand(0), dl, Mode, StackPtr,
4895 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
4896 Results.push_back(
4897 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4898 break;
4899 }
4900 case ISD::RESET_FPMODE: {
4901 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
4902 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
4903 // target must provide custom lowering.
4904 const DataLayout &DL = DAG.getDataLayout();
4905 EVT PtrTy = TLI.getPointerTy(DL);
4906 SDValue Mode = DAG.getConstant(-1LL, dl, PtrTy);
4907 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4908 Node->getOperand(0), dl));
4909 break;
4910 }
4911 }
4912
4913 // Replace the original node with the legalized result.
4914 if (!Results.empty()) {
4915 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4916 ReplaceNode(Node, Results.data());
4917 } else
4918 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4919 }
4920
4921 // Determine the vector type to use in place of an original scalar element when
4922 // promoting equally sized vectors.
getPromotedVectorElementType(const TargetLowering & TLI,MVT EltVT,MVT NewEltVT)4923 static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4924 MVT EltVT, MVT NewEltVT) {
4925 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4926 MVT MidVT = OldEltsPerNewElt == 1
4927 ? NewEltVT
4928 : MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4929 assert(TLI.isTypeLegal(MidVT) && "unexpected");
4930 return MidVT;
4931 }
4932
PromoteNode(SDNode * Node)4933 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4934 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4935 SmallVector<SDValue, 8> Results;
4936 MVT OVT = Node->getSimpleValueType(0);
4937 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4938 Node->getOpcode() == ISD::SINT_TO_FP ||
4939 Node->getOpcode() == ISD::SETCC ||
4940 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4941 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4942 OVT = Node->getOperand(0).getSimpleValueType();
4943 }
4944 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4945 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4946 Node->getOpcode() == ISD::STRICT_FSETCC ||
4947 Node->getOpcode() == ISD::STRICT_FSETCCS)
4948 OVT = Node->getOperand(1).getSimpleValueType();
4949 if (Node->getOpcode() == ISD::BR_CC ||
4950 Node->getOpcode() == ISD::SELECT_CC)
4951 OVT = Node->getOperand(2).getSimpleValueType();
4952 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4953 SDLoc dl(Node);
4954 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4955 switch (Node->getOpcode()) {
4956 case ISD::CTTZ:
4957 case ISD::CTTZ_ZERO_UNDEF:
4958 case ISD::CTLZ:
4959 case ISD::CTLZ_ZERO_UNDEF:
4960 case ISD::CTPOP:
4961 // Zero extend the argument unless its cttz, then use any_extend.
4962 if (Node->getOpcode() == ISD::CTTZ ||
4963 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4964 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4965 else
4966 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4967
4968 if (Node->getOpcode() == ISD::CTTZ) {
4969 // The count is the same in the promoted type except if the original
4970 // value was zero. This can be handled by setting the bit just off
4971 // the top of the original type.
4972 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4973 OVT.getSizeInBits());
4974 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4975 DAG.getConstant(TopBit, dl, NVT));
4976 }
4977 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4978 // already the correct result.
4979 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4980 if (Node->getOpcode() == ISD::CTLZ ||
4981 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4982 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4983 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4984 DAG.getConstant(NVT.getSizeInBits() -
4985 OVT.getSizeInBits(), dl, NVT));
4986 }
4987 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4988 break;
4989 case ISD::BITREVERSE:
4990 case ISD::BSWAP: {
4991 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4992 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4993 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4994 Tmp1 = DAG.getNode(
4995 ISD::SRL, dl, NVT, Tmp1,
4996 DAG.getConstant(DiffBits, dl,
4997 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4998
4999 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5000 break;
5001 }
5002 case ISD::FP_TO_UINT:
5003 case ISD::STRICT_FP_TO_UINT:
5004 case ISD::FP_TO_SINT:
5005 case ISD::STRICT_FP_TO_SINT:
5006 PromoteLegalFP_TO_INT(Node, dl, Results);
5007 break;
5008 case ISD::FP_TO_UINT_SAT:
5009 case ISD::FP_TO_SINT_SAT:
5010 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
5011 break;
5012 case ISD::UINT_TO_FP:
5013 case ISD::STRICT_UINT_TO_FP:
5014 case ISD::SINT_TO_FP:
5015 case ISD::STRICT_SINT_TO_FP:
5016 PromoteLegalINT_TO_FP(Node, dl, Results);
5017 break;
5018 case ISD::VAARG: {
5019 SDValue Chain = Node->getOperand(0); // Get the chain.
5020 SDValue Ptr = Node->getOperand(1); // Get the pointer.
5021
5022 unsigned TruncOp;
5023 if (OVT.isVector()) {
5024 TruncOp = ISD::BITCAST;
5025 } else {
5026 assert(OVT.isInteger()
5027 && "VAARG promotion is supported only for vectors or integer types");
5028 TruncOp = ISD::TRUNCATE;
5029 }
5030
5031 // Perform the larger operation, then convert back
5032 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
5033 Node->getConstantOperandVal(3));
5034 Chain = Tmp1.getValue(1);
5035
5036 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
5037
5038 // Modified the chain result - switch anything that used the old chain to
5039 // use the new one.
5040 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
5041 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
5042 if (UpdatedNodes) {
5043 UpdatedNodes->insert(Tmp2.getNode());
5044 UpdatedNodes->insert(Chain.getNode());
5045 }
5046 ReplacedNode(Node);
5047 break;
5048 }
5049 case ISD::MUL:
5050 case ISD::SDIV:
5051 case ISD::SREM:
5052 case ISD::UDIV:
5053 case ISD::UREM:
5054 case ISD::SMIN:
5055 case ISD::SMAX:
5056 case ISD::UMIN:
5057 case ISD::UMAX:
5058 case ISD::AND:
5059 case ISD::OR:
5060 case ISD::XOR: {
5061 unsigned ExtOp, TruncOp;
5062 if (OVT.isVector()) {
5063 ExtOp = ISD::BITCAST;
5064 TruncOp = ISD::BITCAST;
5065 } else {
5066 assert(OVT.isInteger() && "Cannot promote logic operation");
5067
5068 switch (Node->getOpcode()) {
5069 default:
5070 ExtOp = ISD::ANY_EXTEND;
5071 break;
5072 case ISD::SDIV:
5073 case ISD::SREM:
5074 case ISD::SMIN:
5075 case ISD::SMAX:
5076 ExtOp = ISD::SIGN_EXTEND;
5077 break;
5078 case ISD::UDIV:
5079 case ISD::UREM:
5080 ExtOp = ISD::ZERO_EXTEND;
5081 break;
5082 case ISD::UMIN:
5083 case ISD::UMAX:
5084 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5085 ExtOp = ISD::SIGN_EXTEND;
5086 else
5087 ExtOp = ISD::ZERO_EXTEND;
5088 break;
5089 }
5090 TruncOp = ISD::TRUNCATE;
5091 }
5092 // Promote each of the values to the new type.
5093 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5094 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5095 // Perform the larger operation, then convert back
5096 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5097 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5098 break;
5099 }
5100 case ISD::UMUL_LOHI:
5101 case ISD::SMUL_LOHI: {
5102 // Promote to a multiply in a wider integer type.
5103 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5104 : ISD::SIGN_EXTEND;
5105 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5106 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5107 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
5108
5109 auto &DL = DAG.getDataLayout();
5110 unsigned OriginalSize = OVT.getScalarSizeInBits();
5111 Tmp2 = DAG.getNode(
5112 ISD::SRL, dl, NVT, Tmp1,
5113 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
5114 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5115 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
5116 break;
5117 }
5118 case ISD::SELECT: {
5119 unsigned ExtOp, TruncOp;
5120 if (Node->getValueType(0).isVector() ||
5121 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
5122 ExtOp = ISD::BITCAST;
5123 TruncOp = ISD::BITCAST;
5124 } else if (Node->getValueType(0).isInteger()) {
5125 ExtOp = ISD::ANY_EXTEND;
5126 TruncOp = ISD::TRUNCATE;
5127 } else {
5128 ExtOp = ISD::FP_EXTEND;
5129 TruncOp = ISD::FP_ROUND;
5130 }
5131 Tmp1 = Node->getOperand(0);
5132 // Promote each of the values to the new type.
5133 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5134 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5135 // Perform the larger operation, then round down.
5136 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5137 Tmp1->setFlags(Node->getFlags());
5138 if (TruncOp != ISD::FP_ROUND)
5139 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
5140 else
5141 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
5142 DAG.getIntPtrConstant(0, dl));
5143 Results.push_back(Tmp1);
5144 break;
5145 }
5146 case ISD::VECTOR_SHUFFLE: {
5147 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
5148
5149 // Cast the two input vectors.
5150 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
5151 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
5152
5153 // Convert the shuffle mask to the right # elements.
5154 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5155 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
5156 Results.push_back(Tmp1);
5157 break;
5158 }
5159 case ISD::VECTOR_SPLICE: {
5160 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
5161 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
5162 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
5163 Node->getOperand(2));
5164 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
5165 break;
5166 }
5167 case ISD::SELECT_CC: {
5168 SDValue Cond = Node->getOperand(4);
5169 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
5170 // Type of the comparison operands.
5171 MVT CVT = Node->getSimpleValueType(0);
5172 assert(CVT == OVT && "not handled");
5173
5174 unsigned ExtOp = ISD::FP_EXTEND;
5175 if (NVT.isInteger()) {
5176 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5177 }
5178
5179 // Promote the comparison operands, if needed.
5180 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5181 Tmp1 = Node->getOperand(0);
5182 Tmp2 = Node->getOperand(1);
5183 } else {
5184 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5185 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5186 }
5187 // Cast the true/false operands.
5188 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5189 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5190
5191 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5192 Node->getFlags());
5193
5194 // Cast the result back to the original type.
5195 if (ExtOp != ISD::FP_EXTEND)
5196 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
5197 else
5198 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
5199 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
5200
5201 Results.push_back(Tmp1);
5202 break;
5203 }
5204 case ISD::SETCC:
5205 case ISD::STRICT_FSETCC:
5206 case ISD::STRICT_FSETCCS: {
5207 unsigned ExtOp = ISD::FP_EXTEND;
5208 if (NVT.isInteger()) {
5209 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
5210 if (isSignedIntSetCC(CCCode) ||
5211 TLI.isSExtCheaperThanZExt(Node->getOperand(0).getValueType(), NVT))
5212 ExtOp = ISD::SIGN_EXTEND;
5213 else
5214 ExtOp = ISD::ZERO_EXTEND;
5215 }
5216 if (Node->isStrictFPOpcode()) {
5217 SDValue InChain = Node->getOperand(0);
5218 std::tie(Tmp1, std::ignore) =
5219 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
5220 std::tie(Tmp2, std::ignore) =
5221 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
5222 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
5223 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5224 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
5225 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
5226 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5227 Node->getFlags()));
5228 Results.push_back(Results.back().getValue(1));
5229 break;
5230 }
5231 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5232 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5233 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
5234 Tmp2, Node->getOperand(2), Node->getFlags()));
5235 break;
5236 }
5237 case ISD::BR_CC: {
5238 unsigned ExtOp = ISD::FP_EXTEND;
5239 if (NVT.isInteger()) {
5240 ISD::CondCode CCCode =
5241 cast<CondCodeSDNode>(Node->getOperand(1))->get();
5242 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5243 }
5244 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5245 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5246 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
5247 Node->getOperand(0), Node->getOperand(1),
5248 Tmp1, Tmp2, Node->getOperand(4)));
5249 break;
5250 }
5251 case ISD::FADD:
5252 case ISD::FSUB:
5253 case ISD::FMUL:
5254 case ISD::FDIV:
5255 case ISD::FREM:
5256 case ISD::FMINNUM:
5257 case ISD::FMAXNUM:
5258 case ISD::FMINIMUM:
5259 case ISD::FMAXIMUM:
5260 case ISD::FPOW:
5261 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5262 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5263 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5264 Node->getFlags());
5265 Results.push_back(
5266 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
5267 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5268 break;
5269 case ISD::STRICT_FADD:
5270 case ISD::STRICT_FSUB:
5271 case ISD::STRICT_FMUL:
5272 case ISD::STRICT_FDIV:
5273 case ISD::STRICT_FMINNUM:
5274 case ISD::STRICT_FMAXNUM:
5275 case ISD::STRICT_FREM:
5276 case ISD::STRICT_FPOW:
5277 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5278 {Node->getOperand(0), Node->getOperand(1)});
5279 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5280 {Node->getOperand(0), Node->getOperand(2)});
5281 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5282 Tmp2.getValue(1));
5283 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5284 {Tmp3, Tmp1, Tmp2});
5285 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5286 {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5287 Results.push_back(Tmp1);
5288 Results.push_back(Tmp1.getValue(1));
5289 break;
5290 case ISD::FMA:
5291 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5292 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5293 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
5294 Results.push_back(
5295 DAG.getNode(ISD::FP_ROUND, dl, OVT,
5296 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5297 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5298 break;
5299 case ISD::STRICT_FMA:
5300 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5301 {Node->getOperand(0), Node->getOperand(1)});
5302 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5303 {Node->getOperand(0), Node->getOperand(2)});
5304 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5305 {Node->getOperand(0), Node->getOperand(3)});
5306 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5307 Tmp2.getValue(1), Tmp3.getValue(1));
5308 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5309 {Tmp4, Tmp1, Tmp2, Tmp3});
5310 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5311 {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5312 Results.push_back(Tmp4);
5313 Results.push_back(Tmp4.getValue(1));
5314 break;
5315 case ISD::FCOPYSIGN:
5316 case ISD::FLDEXP:
5317 case ISD::FPOWI: {
5318 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5319 Tmp2 = Node->getOperand(1);
5320 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5321
5322 // fcopysign doesn't change anything but the sign bit, so
5323 // (fp_round (fcopysign (fpext a), b))
5324 // is as precise as
5325 // (fp_round (fpext a))
5326 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5327 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5328 Results.push_back(
5329 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
5330 DAG.getIntPtrConstant(isTrunc, dl, /*isTarget=*/true)));
5331 break;
5332 }
5333 case ISD::STRICT_FPOWI:
5334 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5335 {Node->getOperand(0), Node->getOperand(1)});
5336 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5337 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
5338 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5339 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5340 Results.push_back(Tmp3);
5341 Results.push_back(Tmp3.getValue(1));
5342 break;
5343 case ISD::FFREXP: {
5344 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5345 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1);
5346
5347 Results.push_back(
5348 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5349 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5350
5351 Results.push_back(Tmp2.getValue(1));
5352 break;
5353 }
5354 case ISD::FFLOOR:
5355 case ISD::FCEIL:
5356 case ISD::FRINT:
5357 case ISD::FNEARBYINT:
5358 case ISD::FROUND:
5359 case ISD::FROUNDEVEN:
5360 case ISD::FTRUNC:
5361 case ISD::FNEG:
5362 case ISD::FSQRT:
5363 case ISD::FSIN:
5364 case ISD::FCOS:
5365 case ISD::FLOG:
5366 case ISD::FLOG2:
5367 case ISD::FLOG10:
5368 case ISD::FABS:
5369 case ISD::FEXP:
5370 case ISD::FEXP2:
5371 case ISD::FEXP10:
5372 case ISD::FCANONICALIZE:
5373 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5374 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5375 Results.push_back(
5376 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5377 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5378 break;
5379 case ISD::STRICT_FFLOOR:
5380 case ISD::STRICT_FCEIL:
5381 case ISD::STRICT_FRINT:
5382 case ISD::STRICT_FNEARBYINT:
5383 case ISD::STRICT_FROUND:
5384 case ISD::STRICT_FROUNDEVEN:
5385 case ISD::STRICT_FTRUNC:
5386 case ISD::STRICT_FSQRT:
5387 case ISD::STRICT_FSIN:
5388 case ISD::STRICT_FCOS:
5389 case ISD::STRICT_FLOG:
5390 case ISD::STRICT_FLOG2:
5391 case ISD::STRICT_FLOG10:
5392 case ISD::STRICT_FEXP:
5393 case ISD::STRICT_FEXP2:
5394 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5395 {Node->getOperand(0), Node->getOperand(1)});
5396 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5397 {Tmp1.getValue(1), Tmp1});
5398 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5399 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5400 Results.push_back(Tmp3);
5401 Results.push_back(Tmp3.getValue(1));
5402 break;
5403 case ISD::BUILD_VECTOR: {
5404 MVT EltVT = OVT.getVectorElementType();
5405 MVT NewEltVT = NVT.getVectorElementType();
5406
5407 // Handle bitcasts to a different vector type with the same total bit size
5408 //
5409 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
5410 // =>
5411 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
5412
5413 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5414 "Invalid promote type for build_vector");
5415 assert(NewEltVT.bitsLE(EltVT) && "not handled");
5416
5417 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5418
5419 SmallVector<SDValue, 8> NewOps;
5420 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
5421 SDValue Op = Node->getOperand(I);
5422 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
5423 }
5424
5425 SDLoc SL(Node);
5426 SDValue Concat =
5427 DAG.getNode(MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
5428 SL, NVT, NewOps);
5429 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5430 Results.push_back(CvtVec);
5431 break;
5432 }
5433 case ISD::EXTRACT_VECTOR_ELT: {
5434 MVT EltVT = OVT.getVectorElementType();
5435 MVT NewEltVT = NVT.getVectorElementType();
5436
5437 // Handle bitcasts to a different vector type with the same total bit size.
5438 //
5439 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
5440 // =>
5441 // v4i32:castx = bitcast x:v2i64
5442 //
5443 // i64 = bitcast
5444 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
5445 // (i32 (extract_vector_elt castx, (2 * y + 1)))
5446 //
5447
5448 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5449 "Invalid promote type for extract_vector_elt");
5450 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5451
5452 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5453 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5454
5455 SDValue Idx = Node->getOperand(1);
5456 EVT IdxVT = Idx.getValueType();
5457 SDLoc SL(Node);
5458 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5459 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
5460
5461 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
5462
5463 SmallVector<SDValue, 8> NewOps;
5464 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5465 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
5466 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
5467
5468 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
5469 CastVec, TmpIdx);
5470 NewOps.push_back(Elt);
5471 }
5472
5473 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5474 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
5475 break;
5476 }
5477 case ISD::INSERT_VECTOR_ELT: {
5478 MVT EltVT = OVT.getVectorElementType();
5479 MVT NewEltVT = NVT.getVectorElementType();
5480
5481 // Handle bitcasts to a different vector type with the same total bit size
5482 //
5483 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
5484 // =>
5485 // v4i32:castx = bitcast x:v2i64
5486 // v2i32:casty = bitcast y:i64
5487 //
5488 // v2i64 = bitcast
5489 // (v4i32 insert_vector_elt
5490 // (v4i32 insert_vector_elt v4i32:castx,
5491 // (extract_vector_elt casty, 0), 2 * z),
5492 // (extract_vector_elt casty, 1), (2 * z + 1))
5493
5494 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5495 "Invalid promote type for insert_vector_elt");
5496 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5497
5498 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5499 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5500
5501 SDValue Val = Node->getOperand(1);
5502 SDValue Idx = Node->getOperand(2);
5503 EVT IdxVT = Idx.getValueType();
5504 SDLoc SL(Node);
5505
5506 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
5507 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
5508
5509 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
5510 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
5511
5512 SDValue NewVec = CastVec;
5513 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5514 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
5515 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
5516
5517 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
5518 CastVal, IdxOffset);
5519
5520 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
5521 NewVec, Elt, InEltIdx);
5522 }
5523
5524 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
5525 break;
5526 }
5527 case ISD::SCALAR_TO_VECTOR: {
5528 MVT EltVT = OVT.getVectorElementType();
5529 MVT NewEltVT = NVT.getVectorElementType();
5530
5531 // Handle bitcasts to different vector type with the same total bit size.
5532 //
5533 // e.g. v2i64 = scalar_to_vector x:i64
5534 // =>
5535 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
5536 //
5537
5538 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5539 SDValue Val = Node->getOperand(0);
5540 SDLoc SL(Node);
5541
5542 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
5543 SDValue Undef = DAG.getUNDEF(MidVT);
5544
5545 SmallVector<SDValue, 8> NewElts;
5546 NewElts.push_back(CastVal);
5547 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5548 NewElts.push_back(Undef);
5549
5550 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
5551 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5552 Results.push_back(CvtVec);
5553 break;
5554 }
5555 case ISD::ATOMIC_SWAP: {
5556 AtomicSDNode *AM = cast<AtomicSDNode>(Node);
5557 SDLoc SL(Node);
5558 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
5559 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5560 "unexpected promotion type");
5561 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5562 "unexpected atomic_swap with illegal type");
5563
5564 SDValue NewAtomic
5565 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
5566 DAG.getVTList(NVT, MVT::Other),
5567 { AM->getChain(), AM->getBasePtr(), CastVal },
5568 AM->getMemOperand());
5569 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
5570 Results.push_back(NewAtomic.getValue(1));
5571 break;
5572 }
5573 case ISD::SPLAT_VECTOR: {
5574 SDValue Scalar = Node->getOperand(0);
5575 MVT ScalarType = Scalar.getSimpleValueType();
5576 MVT NewScalarType = NVT.getVectorElementType();
5577 if (ScalarType.isInteger()) {
5578 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NewScalarType, Scalar);
5579 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5580 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
5581 break;
5582 }
5583 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar);
5584 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5585 Results.push_back(
5586 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5587 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5588 break;
5589 }
5590 }
5591
5592 // Replace the original node with the legalized result.
5593 if (!Results.empty()) {
5594 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
5595 ReplaceNode(Node, Results.data());
5596 } else
5597 LLVM_DEBUG(dbgs() << "Could not promote node\n");
5598 }
5599
5600 /// This is the entry point for the file.
Legalize()5601 void SelectionDAG::Legalize() {
5602 AssignTopologicalOrder();
5603
5604 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5605 // Use a delete listener to remove nodes which were deleted during
5606 // legalization from LegalizeNodes. This is needed to handle the situation
5607 // where a new node is allocated by the object pool to the same address of a
5608 // previously deleted node.
5609 DAGNodeDeletedListener DeleteListener(
5610 *this,
5611 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
5612
5613 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
5614
5615 // Visit all the nodes. We start in topological order, so that we see
5616 // nodes with their original operands intact. Legalization can produce
5617 // new nodes which may themselves need to be legalized. Iterate until all
5618 // nodes have been legalized.
5619 while (true) {
5620 bool AnyLegalized = false;
5621 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
5622 --NI;
5623
5624 SDNode *N = &*NI;
5625 if (N->use_empty() && N != getRoot().getNode()) {
5626 ++NI;
5627 DeleteNode(N);
5628 continue;
5629 }
5630
5631 if (LegalizedNodes.insert(N).second) {
5632 AnyLegalized = true;
5633 Legalizer.LegalizeOp(N);
5634
5635 if (N->use_empty() && N != getRoot().getNode()) {
5636 ++NI;
5637 DeleteNode(N);
5638 }
5639 }
5640 }
5641 if (!AnyLegalized)
5642 break;
5643
5644 }
5645
5646 // Remove dead nodes now.
5647 RemoveDeadNodes();
5648 }
5649
LegalizeOp(SDNode * N,SmallSetVector<SDNode *,16> & UpdatedNodes)5650 bool SelectionDAG::LegalizeOp(SDNode *N,
5651 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
5652 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5653 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
5654
5655 // Directly insert the node in question, and legalize it. This will recurse
5656 // as needed through operands.
5657 LegalizedNodes.insert(N);
5658 Legalizer.LegalizeOp(N);
5659
5660 return LegalizedNodes.count(N);
5661 }
5662