1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include "llvm/Support/KnownBits.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
PromoteIntegerResult(SDNode * N,unsigned ResNo)37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39 dbgs() << "\n");
40 SDValue Res = SDValue();
41
42 // See if the target wants to custom expand this node.
43 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45 return;
46 }
47
48 switch (N->getOpcode()) {
49 default:
50 #ifndef NDEBUG
51 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52 N->dump(&DAG); dbgs() << "\n";
53 #endif
54 llvm_unreachable("Do not know how to promote this operator!");
55 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
63 case ISD::CTLZ_ZERO_UNDEF:
64 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
66 case ISD::CTTZ_ZERO_UNDEF:
67 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
68 case ISD::EXTRACT_VECTOR_ELT:
69 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72 break;
73 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74 break;
75 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78 case ISD::STRICT_FSETCC:
79 case ISD::STRICT_FSETCCS:
80 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
81 case ISD::SMIN:
82 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
83 case ISD::UMIN:
84 case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
85
86 case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
87 case ISD::SIGN_EXTEND_INREG:
88 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
89 case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
90 case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
91 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
92 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
93 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
94
95 case ISD::EXTRACT_SUBVECTOR:
96 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
97 case ISD::VECTOR_SHUFFLE:
98 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
99 case ISD::INSERT_VECTOR_ELT:
100 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
101 case ISD::BUILD_VECTOR:
102 Res = PromoteIntRes_BUILD_VECTOR(N); break;
103 case ISD::SCALAR_TO_VECTOR:
104 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
105 case ISD::SPLAT_VECTOR:
106 Res = PromoteIntRes_SPLAT_VECTOR(N); break;
107 case ISD::CONCAT_VECTORS:
108 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
109
110 case ISD::ANY_EXTEND_VECTOR_INREG:
111 case ISD::SIGN_EXTEND_VECTOR_INREG:
112 case ISD::ZERO_EXTEND_VECTOR_INREG:
113 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
114
115 case ISD::SIGN_EXTEND:
116 case ISD::ZERO_EXTEND:
117 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
118
119 case ISD::STRICT_FP_TO_SINT:
120 case ISD::STRICT_FP_TO_UINT:
121 case ISD::FP_TO_SINT:
122 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
123
124 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
125
126 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
127
128 case ISD::AND:
129 case ISD::OR:
130 case ISD::XOR:
131 case ISD::ADD:
132 case ISD::SUB:
133 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
134
135 case ISD::SDIV:
136 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
137
138 case ISD::UDIV:
139 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
140
141 case ISD::SADDO:
142 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
143 case ISD::UADDO:
144 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
145 case ISD::SMULO:
146 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
147
148 case ISD::ADDE:
149 case ISD::SUBE:
150 case ISD::ADDCARRY:
151 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
152
153 case ISD::SADDSAT:
154 case ISD::UADDSAT:
155 case ISD::SSUBSAT:
156 case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
157
158 case ISD::SMULFIX:
159 case ISD::SMULFIXSAT:
160 case ISD::UMULFIX:
161 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
162
163 case ISD::SDIVFIX:
164 case ISD::UDIVFIX: Res = PromoteIntRes_DIVFIX(N); break;
165
166 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
167
168 case ISD::ATOMIC_LOAD:
169 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
170
171 case ISD::ATOMIC_LOAD_ADD:
172 case ISD::ATOMIC_LOAD_SUB:
173 case ISD::ATOMIC_LOAD_AND:
174 case ISD::ATOMIC_LOAD_CLR:
175 case ISD::ATOMIC_LOAD_OR:
176 case ISD::ATOMIC_LOAD_XOR:
177 case ISD::ATOMIC_LOAD_NAND:
178 case ISD::ATOMIC_LOAD_MIN:
179 case ISD::ATOMIC_LOAD_MAX:
180 case ISD::ATOMIC_LOAD_UMIN:
181 case ISD::ATOMIC_LOAD_UMAX:
182 case ISD::ATOMIC_SWAP:
183 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
184
185 case ISD::ATOMIC_CMP_SWAP:
186 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
187 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
188 break;
189
190 case ISD::VECREDUCE_ADD:
191 case ISD::VECREDUCE_MUL:
192 case ISD::VECREDUCE_AND:
193 case ISD::VECREDUCE_OR:
194 case ISD::VECREDUCE_XOR:
195 case ISD::VECREDUCE_SMAX:
196 case ISD::VECREDUCE_SMIN:
197 case ISD::VECREDUCE_UMAX:
198 case ISD::VECREDUCE_UMIN:
199 Res = PromoteIntRes_VECREDUCE(N);
200 break;
201 }
202
203 // If the result is null then the sub-method took care of registering it.
204 if (Res.getNode())
205 SetPromotedInteger(SDValue(N, ResNo), Res);
206 }
207
PromoteIntRes_MERGE_VALUES(SDNode * N,unsigned ResNo)208 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
209 unsigned ResNo) {
210 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
211 return GetPromotedInteger(Op);
212 }
213
PromoteIntRes_AssertSext(SDNode * N)214 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
215 // Sign-extend the new bits, and continue the assertion.
216 SDValue Op = SExtPromotedInteger(N->getOperand(0));
217 return DAG.getNode(ISD::AssertSext, SDLoc(N),
218 Op.getValueType(), Op, N->getOperand(1));
219 }
220
PromoteIntRes_AssertZext(SDNode * N)221 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
222 // Zero the new bits, and continue the assertion.
223 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
224 return DAG.getNode(ISD::AssertZext, SDLoc(N),
225 Op.getValueType(), Op, N->getOperand(1));
226 }
227
PromoteIntRes_Atomic0(AtomicSDNode * N)228 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
229 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
230 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
231 N->getMemoryVT(), ResVT,
232 N->getChain(), N->getBasePtr(),
233 N->getMemOperand());
234 // Legalize the chain result - switch anything that used the old chain to
235 // use the new one.
236 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
237 return Res;
238 }
239
PromoteIntRes_Atomic1(AtomicSDNode * N)240 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
241 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
242 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
243 N->getMemoryVT(),
244 N->getChain(), N->getBasePtr(),
245 Op2, N->getMemOperand());
246 // Legalize the chain result - switch anything that used the old chain to
247 // use the new one.
248 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
249 return Res;
250 }
251
PromoteIntRes_AtomicCmpSwap(AtomicSDNode * N,unsigned ResNo)252 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
253 unsigned ResNo) {
254 if (ResNo == 1) {
255 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
256 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
257 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
258
259 // Only use the result of getSetCCResultType if it is legal,
260 // otherwise just use the promoted result type (NVT).
261 if (!TLI.isTypeLegal(SVT))
262 SVT = NVT;
263
264 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
265 SDValue Res = DAG.getAtomicCmpSwap(
266 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
267 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
268 N->getMemOperand());
269 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
270 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
271 return Res.getValue(1);
272 }
273
274 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
275 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
276 SDVTList VTs =
277 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
278 SDValue Res = DAG.getAtomicCmpSwap(
279 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
280 N->getBasePtr(), Op2, Op3, N->getMemOperand());
281 // Update the use to N with the newly created Res.
282 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
283 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
284 return Res;
285 }
286
PromoteIntRes_BITCAST(SDNode * N)287 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
288 SDValue InOp = N->getOperand(0);
289 EVT InVT = InOp.getValueType();
290 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
291 EVT OutVT = N->getValueType(0);
292 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
293 SDLoc dl(N);
294
295 switch (getTypeAction(InVT)) {
296 case TargetLowering::TypeLegal:
297 break;
298 case TargetLowering::TypePromoteInteger:
299 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
300 // The input promotes to the same size. Convert the promoted value.
301 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
302 break;
303 case TargetLowering::TypeSoftenFloat:
304 // Promote the integer operand by hand.
305 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
306 case TargetLowering::TypePromoteFloat: {
307 // Convert the promoted float by hand.
308 if (!NOutVT.isVector())
309 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
310 break;
311 }
312 case TargetLowering::TypeExpandInteger:
313 case TargetLowering::TypeExpandFloat:
314 break;
315 case TargetLowering::TypeScalarizeVector:
316 // Convert the element to an integer and promote it by hand.
317 if (!NOutVT.isVector())
318 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
319 BitConvertToInteger(GetScalarizedVector(InOp)));
320 break;
321 case TargetLowering::TypeSplitVector: {
322 if (!NOutVT.isVector()) {
323 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
324 // pieces of the input into integers and reassemble in the final type.
325 SDValue Lo, Hi;
326 GetSplitVector(N->getOperand(0), Lo, Hi);
327 Lo = BitConvertToInteger(Lo);
328 Hi = BitConvertToInteger(Hi);
329
330 if (DAG.getDataLayout().isBigEndian())
331 std::swap(Lo, Hi);
332
333 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
334 EVT::getIntegerVT(*DAG.getContext(),
335 NOutVT.getSizeInBits()),
336 JoinIntegers(Lo, Hi));
337 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
338 }
339 break;
340 }
341 case TargetLowering::TypeWidenVector:
342 // The input is widened to the same size. Convert to the widened value.
343 // Make sure that the outgoing value is not a vector, because this would
344 // make us bitcast between two vectors which are legalized in different ways.
345 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
346 SDValue Res =
347 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
348
349 // For big endian targets we need to shift the casted value or the
350 // interesting bits will end up at the wrong place.
351 if (DAG.getDataLayout().isBigEndian()) {
352 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
353 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout());
354 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
355 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
356 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
357 }
358 return Res;
359 }
360 // If the output type is also a vector and widening it to the same size
361 // as the widened input type would be a legal type, we can widen the bitcast
362 // and handle the promotion after.
363 if (NOutVT.isVector()) {
364 unsigned WidenInSize = NInVT.getSizeInBits();
365 unsigned OutSize = OutVT.getSizeInBits();
366 if (WidenInSize % OutSize == 0) {
367 unsigned Scale = WidenInSize / OutSize;
368 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
369 OutVT.getVectorElementType(),
370 OutVT.getVectorNumElements() * Scale);
371 if (isTypeLegal(WideOutVT)) {
372 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
373 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
374 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
375 DAG.getConstant(0, dl, IdxTy));
376 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
377 }
378 }
379 }
380 }
381
382 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
383 CreateStackStoreLoad(InOp, OutVT));
384 }
385
386 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount
387 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
getShiftAmountTyForConstant(EVT VT,const TargetLowering & TLI,SelectionDAG & DAG)388 static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI,
389 SelectionDAG &DAG) {
390 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
391 // If any possible shift value won't fit in the prefered type, just use
392 // something safe. It will be legalized when the shift is expanded.
393 if (!ShiftVT.isVector() &&
394 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits()))
395 ShiftVT = MVT::i32;
396 return ShiftVT;
397 }
398
PromoteIntRes_BSWAP(SDNode * N)399 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
400 SDValue Op = GetPromotedInteger(N->getOperand(0));
401 EVT OVT = N->getValueType(0);
402 EVT NVT = Op.getValueType();
403 SDLoc dl(N);
404
405 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
406 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
407 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
408 DAG.getConstant(DiffBits, dl, ShiftVT));
409 }
410
PromoteIntRes_BITREVERSE(SDNode * N)411 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
412 SDValue Op = GetPromotedInteger(N->getOperand(0));
413 EVT OVT = N->getValueType(0);
414 EVT NVT = Op.getValueType();
415 SDLoc dl(N);
416
417 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
418 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
419 return DAG.getNode(ISD::SRL, dl, NVT,
420 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
421 DAG.getConstant(DiffBits, dl, ShiftVT));
422 }
423
PromoteIntRes_BUILD_PAIR(SDNode * N)424 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
425 // The pair element type may be legal, or may not promote to the same type as
426 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
427 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
428 TLI.getTypeToTransformTo(*DAG.getContext(),
429 N->getValueType(0)), JoinIntegers(N->getOperand(0),
430 N->getOperand(1)));
431 }
432
PromoteIntRes_Constant(SDNode * N)433 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
434 EVT VT = N->getValueType(0);
435 // FIXME there is no actual debug info here
436 SDLoc dl(N);
437 // Zero extend things like i1, sign extend everything else. It shouldn't
438 // matter in theory which one we pick, but this tends to give better code?
439 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
440 SDValue Result = DAG.getNode(Opc, dl,
441 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
442 SDValue(N, 0));
443 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
444 return Result;
445 }
446
PromoteIntRes_CTLZ(SDNode * N)447 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
448 // Zero extend to the promoted type and do the count there.
449 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
450 SDLoc dl(N);
451 EVT OVT = N->getValueType(0);
452 EVT NVT = Op.getValueType();
453 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
454 // Subtract off the extra leading bits in the bigger type.
455 return DAG.getNode(
456 ISD::SUB, dl, NVT, Op,
457 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
458 NVT));
459 }
460
PromoteIntRes_CTPOP(SDNode * N)461 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
462 // Zero extend to the promoted type and do the count there.
463 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
464 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
465 }
466
PromoteIntRes_CTTZ(SDNode * N)467 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
468 SDValue Op = GetPromotedInteger(N->getOperand(0));
469 EVT OVT = N->getValueType(0);
470 EVT NVT = Op.getValueType();
471 SDLoc dl(N);
472 if (N->getOpcode() == ISD::CTTZ) {
473 // The count is the same in the promoted type except if the original
474 // value was zero. This can be handled by setting the bit just off
475 // the top of the original type.
476 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
477 OVT.getScalarSizeInBits());
478 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
479 }
480 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
481 }
482
PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode * N)483 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
484 SDLoc dl(N);
485 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
486
487 SDValue Op0 = N->getOperand(0);
488 SDValue Op1 = N->getOperand(1);
489
490 // If the input also needs to be promoted, do that first so we can get a
491 // get a good idea for the output type.
492 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
493 == TargetLowering::TypePromoteInteger) {
494 SDValue In = GetPromotedInteger(Op0);
495
496 // If the new type is larger than NVT, use it. We probably won't need to
497 // promote it again.
498 EVT SVT = In.getValueType().getScalarType();
499 if (SVT.bitsGE(NVT)) {
500 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
501 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
502 }
503 }
504
505 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
506 }
507
PromoteIntRes_FP_TO_XINT(SDNode * N)508 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
509 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
510 unsigned NewOpc = N->getOpcode();
511 SDLoc dl(N);
512
513 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
514 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
515 // and SINT conversions are Custom, there is no way to tell which is
516 // preferable. We choose SINT because that's the right thing on PPC.)
517 if (N->getOpcode() == ISD::FP_TO_UINT &&
518 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
519 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
520 NewOpc = ISD::FP_TO_SINT;
521
522 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
523 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
524 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
525 NewOpc = ISD::STRICT_FP_TO_SINT;
526
527 SDValue Res;
528 if (N->isStrictFPOpcode()) {
529 Res = DAG.getNode(NewOpc, dl, { NVT, MVT::Other },
530 { N->getOperand(0), N->getOperand(1) });
531 // Legalize the chain result - switch anything that used the old chain to
532 // use the new one.
533 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
534 } else
535 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
536
537 // Assert that the converted value fits in the original type. If it doesn't
538 // (eg: because the value being converted is too big), then the result of the
539 // original operation was undefined anyway, so the assert is still correct.
540 //
541 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
542 // before legalization: fp-to-uint16, 65534. -> 0xfffe
543 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
544 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
545 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ?
546 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
547 DAG.getValueType(N->getValueType(0).getScalarType()));
548 }
549
PromoteIntRes_FP_TO_FP16(SDNode * N)550 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
552 SDLoc dl(N);
553
554 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
555 }
556
PromoteIntRes_FLT_ROUNDS(SDNode * N)557 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
558 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
559 SDLoc dl(N);
560
561 return DAG.getNode(N->getOpcode(), dl, NVT);
562 }
563
PromoteIntRes_INT_EXTEND(SDNode * N)564 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
565 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
566 SDLoc dl(N);
567
568 if (getTypeAction(N->getOperand(0).getValueType())
569 == TargetLowering::TypePromoteInteger) {
570 SDValue Res = GetPromotedInteger(N->getOperand(0));
571 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
572
573 // If the result and operand types are the same after promotion, simplify
574 // to an in-register extension.
575 if (NVT == Res.getValueType()) {
576 // The high bits are not guaranteed to be anything. Insert an extend.
577 if (N->getOpcode() == ISD::SIGN_EXTEND)
578 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
579 DAG.getValueType(N->getOperand(0).getValueType()));
580 if (N->getOpcode() == ISD::ZERO_EXTEND)
581 return DAG.getZeroExtendInReg(Res, dl,
582 N->getOperand(0).getValueType().getScalarType());
583 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
584 return Res;
585 }
586 }
587
588 // Otherwise, just extend the original operand all the way to the larger type.
589 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
590 }
591
PromoteIntRes_LOAD(LoadSDNode * N)592 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
593 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
594 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
595 ISD::LoadExtType ExtType =
596 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
597 SDLoc dl(N);
598 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
599 N->getMemoryVT(), N->getMemOperand());
600
601 // Legalize the chain result - switch anything that used the old chain to
602 // use the new one.
603 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
604 return Res;
605 }
606
PromoteIntRes_MLOAD(MaskedLoadSDNode * N)607 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
608 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
609 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
610
611 SDLoc dl(N);
612 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
613 N->getOffset(), N->getMask(), ExtPassThru,
614 N->getMemoryVT(), N->getMemOperand(),
615 N->getAddressingMode(), ISD::EXTLOAD);
616 // Legalize the chain result - switch anything that used the old chain to
617 // use the new one.
618 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
619 return Res;
620 }
621
PromoteIntRes_MGATHER(MaskedGatherSDNode * N)622 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
623 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
624 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
625 assert(NVT == ExtPassThru.getValueType() &&
626 "Gather result type and the passThru argument type should be the same");
627
628 SDLoc dl(N);
629 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
630 N->getIndex(), N->getScale() };
631 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
632 N->getMemoryVT(), dl, Ops,
633 N->getMemOperand(), N->getIndexType());
634 // Legalize the chain result - switch anything that used the old chain to
635 // use the new one.
636 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
637 return Res;
638 }
639
640 /// Promote the overflow flag of an overflowing arithmetic node.
PromoteIntRes_Overflow(SDNode * N)641 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
642 // Change the return type of the boolean result while obeying
643 // getSetCCResultType.
644 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
645 EVT VT = N->getValueType(0);
646 EVT SVT = getSetCCResultType(VT);
647 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
648 unsigned NumOps = N->getNumOperands();
649 assert(NumOps <= 3 && "Too many operands");
650 if (NumOps == 3)
651 Ops[2] = N->getOperand(2);
652
653 SDLoc dl(N);
654 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
655 makeArrayRef(Ops, NumOps));
656
657 // Modified the sum result - switch anything that used the old sum to use
658 // the new one.
659 ReplaceValueWith(SDValue(N, 0), Res);
660
661 // Convert to the expected type.
662 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
663 }
664
PromoteIntRes_ADDSUBSAT(SDNode * N)665 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
666 // If the promoted type is legal, we can convert this to:
667 // 1. ANY_EXTEND iN to iM
668 // 2. SHL by M-N
669 // 3. [US][ADD|SUB]SAT
670 // 4. L/ASHR by M-N
671 // Else it is more efficient to convert this to a min and a max
672 // operation in the higher precision arithmetic.
673 SDLoc dl(N);
674 SDValue Op1 = N->getOperand(0);
675 SDValue Op2 = N->getOperand(1);
676 unsigned OldBits = Op1.getScalarValueSizeInBits();
677
678 unsigned Opcode = N->getOpcode();
679
680 SDValue Op1Promoted, Op2Promoted;
681 if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
682 Op1Promoted = ZExtPromotedInteger(Op1);
683 Op2Promoted = ZExtPromotedInteger(Op2);
684 } else {
685 Op1Promoted = SExtPromotedInteger(Op1);
686 Op2Promoted = SExtPromotedInteger(Op2);
687 }
688 EVT PromotedType = Op1Promoted.getValueType();
689 unsigned NewBits = PromotedType.getScalarSizeInBits();
690
691 if (TLI.isOperationLegalOrCustom(Opcode, PromotedType)) {
692 unsigned ShiftOp;
693 switch (Opcode) {
694 case ISD::SADDSAT:
695 case ISD::SSUBSAT:
696 ShiftOp = ISD::SRA;
697 break;
698 case ISD::UADDSAT:
699 case ISD::USUBSAT:
700 ShiftOp = ISD::SRL;
701 break;
702 default:
703 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
704 "addition or subtraction");
705 }
706
707 unsigned SHLAmount = NewBits - OldBits;
708 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
709 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
710 Op1Promoted =
711 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
712 Op2Promoted =
713 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
714
715 SDValue Result =
716 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
717 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
718 } else {
719 if (Opcode == ISD::USUBSAT) {
720 SDValue Max =
721 DAG.getNode(ISD::UMAX, dl, PromotedType, Op1Promoted, Op2Promoted);
722 return DAG.getNode(ISD::SUB, dl, PromotedType, Max, Op2Promoted);
723 }
724
725 if (Opcode == ISD::UADDSAT) {
726 APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits);
727 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
728 SDValue Add =
729 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
730 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
731 }
732
733 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
734 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
735 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
736 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
737 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
738 SDValue Result =
739 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
740 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
741 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
742 return Result;
743 }
744 }
745
PromoteIntRes_MULFIX(SDNode * N)746 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
747 // Can just promote the operands then continue with operation.
748 SDLoc dl(N);
749 SDValue Op1Promoted, Op2Promoted;
750 bool Signed =
751 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
752 bool Saturating =
753 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
754 if (Signed) {
755 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
756 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
757 } else {
758 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
759 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
760 }
761 EVT OldType = N->getOperand(0).getValueType();
762 EVT PromotedType = Op1Promoted.getValueType();
763 unsigned DiffSize =
764 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
765
766 if (Saturating) {
767 // Promoting the operand and result values changes the saturation width,
768 // which is extends the values that we clamp to on saturation. This could be
769 // resolved by shifting one of the operands the same amount, which would
770 // also shift the result we compare against, then shifting back.
771 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
772 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
773 DAG.getConstant(DiffSize, dl, ShiftTy));
774 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
775 Op2Promoted, N->getOperand(2));
776 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
777 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
778 DAG.getConstant(DiffSize, dl, ShiftTy));
779 }
780 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
781 N->getOperand(2));
782 }
783
earlyExpandDIVFIX(SDNode * N,SDValue LHS,SDValue RHS,unsigned Scale,const TargetLowering & TLI,SelectionDAG & DAG)784 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
785 unsigned Scale, const TargetLowering &TLI,
786 SelectionDAG &DAG) {
787 EVT VT = LHS.getValueType();
788 bool Signed = N->getOpcode() == ISD::SDIVFIX;
789
790 SDLoc dl(N);
791 // See if we can perform the division in this type without widening.
792 if (SDValue V = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
793 DAG))
794 return V;
795
796 // If that didn't work, double the type width and try again. That must work,
797 // or something is wrong.
798 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(),
799 VT.getScalarSizeInBits() * 2);
800 if (Signed) {
801 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
802 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
803 } else {
804 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
805 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
806 }
807
808 // TODO: Saturation.
809
810 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
811 DAG);
812 assert(Res && "Expanding DIVFIX with wide type failed?");
813 return DAG.getZExtOrTrunc(Res, dl, VT);
814 }
815
PromoteIntRes_DIVFIX(SDNode * N)816 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
817 SDLoc dl(N);
818 SDValue Op1Promoted, Op2Promoted;
819 bool Signed = N->getOpcode() == ISD::SDIVFIX;
820 if (Signed) {
821 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
822 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
823 } else {
824 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
825 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
826 }
827 EVT PromotedType = Op1Promoted.getValueType();
828 unsigned Scale = N->getConstantOperandVal(2);
829
830 SDValue Res;
831 // If the type is already legal and the operation is legal in that type, we
832 // should not early expand.
833 if (TLI.isTypeLegal(PromotedType)) {
834 TargetLowering::LegalizeAction Action =
835 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
836 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom)
837 Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
838 Op2Promoted, N->getOperand(2));
839 }
840
841 if (!Res)
842 Res = earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG);
843
844 // TODO: Saturation.
845
846 return Res;
847 }
848
PromoteIntRes_SADDSUBO(SDNode * N,unsigned ResNo)849 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
850 if (ResNo == 1)
851 return PromoteIntRes_Overflow(N);
852
853 // The operation overflowed iff the result in the larger type is not the
854 // sign extension of its truncation to the original type.
855 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
856 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
857 EVT OVT = N->getOperand(0).getValueType();
858 EVT NVT = LHS.getValueType();
859 SDLoc dl(N);
860
861 // Do the arithmetic in the larger type.
862 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
863 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
864
865 // Calculate the overflow flag: sign extend the arithmetic result from
866 // the original type.
867 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
868 DAG.getValueType(OVT));
869 // Overflowed if and only if this is not equal to Res.
870 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
871
872 // Use the calculated overflow everywhere.
873 ReplaceValueWith(SDValue(N, 1), Ofl);
874
875 return Res;
876 }
877
PromoteIntRes_SELECT(SDNode * N)878 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
879 SDValue LHS = GetPromotedInteger(N->getOperand(1));
880 SDValue RHS = GetPromotedInteger(N->getOperand(2));
881 return DAG.getSelect(SDLoc(N),
882 LHS.getValueType(), N->getOperand(0), LHS, RHS);
883 }
884
PromoteIntRes_VSELECT(SDNode * N)885 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
886 SDValue Mask = N->getOperand(0);
887
888 SDValue LHS = GetPromotedInteger(N->getOperand(1));
889 SDValue RHS = GetPromotedInteger(N->getOperand(2));
890 return DAG.getNode(ISD::VSELECT, SDLoc(N),
891 LHS.getValueType(), Mask, LHS, RHS);
892 }
893
PromoteIntRes_SELECT_CC(SDNode * N)894 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
895 SDValue LHS = GetPromotedInteger(N->getOperand(2));
896 SDValue RHS = GetPromotedInteger(N->getOperand(3));
897 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
898 LHS.getValueType(), N->getOperand(0),
899 N->getOperand(1), LHS, RHS, N->getOperand(4));
900 }
901
PromoteIntRes_SETCC(SDNode * N)902 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
903 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
904 EVT InVT = N->getOperand(OpNo).getValueType();
905 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
906
907 EVT SVT = getSetCCResultType(InVT);
908
909 // If we got back a type that needs to be promoted, this likely means the
910 // the input type also needs to be promoted. So get the promoted type for
911 // the input and try the query again.
912 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
913 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
914 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
915 SVT = getSetCCResultType(InVT);
916 } else {
917 // Input type isn't promoted, just use the default promoted type.
918 SVT = NVT;
919 }
920 }
921
922 SDLoc dl(N);
923 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
924 "Vector compare must return a vector result!");
925
926 // Get the SETCC result using the canonical SETCC type.
927 SDValue SetCC;
928 if (N->isStrictFPOpcode()) {
929 EVT VTs[] = {SVT, MVT::Other};
930 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
931 N->getOperand(2), N->getOperand(3)};
932 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
933 // Legalize the chain result - switch anything that used the old chain to
934 // use the new one.
935 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
936 } else
937 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
938 N->getOperand(1), N->getOperand(2));
939
940 // Convert to the expected type.
941 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
942 }
943
PromoteIntRes_SHL(SDNode * N)944 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
945 SDValue LHS = GetPromotedInteger(N->getOperand(0));
946 SDValue RHS = N->getOperand(1);
947 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
948 RHS = ZExtPromotedInteger(RHS);
949 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
950 }
951
PromoteIntRes_SIGN_EXTEND_INREG(SDNode * N)952 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
953 SDValue Op = GetPromotedInteger(N->getOperand(0));
954 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
955 Op.getValueType(), Op, N->getOperand(1));
956 }
957
PromoteIntRes_SimpleIntBinOp(SDNode * N)958 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
959 // The input may have strange things in the top bits of the registers, but
960 // these operations don't care. They may have weird bits going out, but
961 // that too is okay if they are integer operations.
962 SDValue LHS = GetPromotedInteger(N->getOperand(0));
963 SDValue RHS = GetPromotedInteger(N->getOperand(1));
964 return DAG.getNode(N->getOpcode(), SDLoc(N),
965 LHS.getValueType(), LHS, RHS);
966 }
967
PromoteIntRes_SExtIntBinOp(SDNode * N)968 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
969 // Sign extend the input.
970 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
971 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
972 return DAG.getNode(N->getOpcode(), SDLoc(N),
973 LHS.getValueType(), LHS, RHS);
974 }
975
PromoteIntRes_ZExtIntBinOp(SDNode * N)976 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
977 // Zero extend the input.
978 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
979 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
980 return DAG.getNode(N->getOpcode(), SDLoc(N),
981 LHS.getValueType(), LHS, RHS);
982 }
983
PromoteIntRes_SRA(SDNode * N)984 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
985 // The input value must be properly sign extended.
986 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
987 SDValue RHS = N->getOperand(1);
988 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
989 RHS = ZExtPromotedInteger(RHS);
990 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
991 }
992
PromoteIntRes_SRL(SDNode * N)993 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
994 // The input value must be properly zero extended.
995 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
996 SDValue RHS = N->getOperand(1);
997 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
998 RHS = ZExtPromotedInteger(RHS);
999 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
1000 }
1001
PromoteIntRes_TRUNCATE(SDNode * N)1002 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1003 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1004 SDValue Res;
1005 SDValue InOp = N->getOperand(0);
1006 SDLoc dl(N);
1007
1008 switch (getTypeAction(InOp.getValueType())) {
1009 default: llvm_unreachable("Unknown type action!");
1010 case TargetLowering::TypeLegal:
1011 case TargetLowering::TypeExpandInteger:
1012 Res = InOp;
1013 break;
1014 case TargetLowering::TypePromoteInteger:
1015 Res = GetPromotedInteger(InOp);
1016 break;
1017 case TargetLowering::TypeSplitVector: {
1018 EVT InVT = InOp.getValueType();
1019 assert(InVT.isVector() && "Cannot split scalar types");
1020 unsigned NumElts = InVT.getVectorNumElements();
1021 assert(NumElts == NVT.getVectorNumElements() &&
1022 "Dst and Src must have the same number of elements");
1023 assert(isPowerOf2_32(NumElts) &&
1024 "Promoted vector type must be a power of two");
1025
1026 SDValue EOp1, EOp2;
1027 GetSplitVector(InOp, EOp1, EOp2);
1028
1029 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1030 NumElts/2);
1031 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1032 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1033
1034 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1035 }
1036 case TargetLowering::TypeWidenVector: {
1037 SDValue WideInOp = GetWidenedVector(InOp);
1038
1039 // Truncate widened InOp.
1040 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1041 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1042 N->getValueType(0).getScalarType(), NumElem);
1043 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1044
1045 // Zero extend so that the elements are of same type as those of NVT
1046 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1047 NumElem);
1048 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1049
1050 // Extract the low NVT subvector.
1051 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
1052 SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
1053 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1054 }
1055 }
1056
1057 // Truncate to NVT instead of VT
1058 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1059 }
1060
PromoteIntRes_UADDSUBO(SDNode * N,unsigned ResNo)1061 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1062 if (ResNo == 1)
1063 return PromoteIntRes_Overflow(N);
1064
1065 // The operation overflowed iff the result in the larger type is not the
1066 // zero extension of its truncation to the original type.
1067 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1068 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1069 EVT OVT = N->getOperand(0).getValueType();
1070 EVT NVT = LHS.getValueType();
1071 SDLoc dl(N);
1072
1073 // Do the arithmetic in the larger type.
1074 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1075 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1076
1077 // Calculate the overflow flag: zero extend the arithmetic result from
1078 // the original type.
1079 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
1080 // Overflowed if and only if this is not equal to Res.
1081 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1082
1083 // Use the calculated overflow everywhere.
1084 ReplaceValueWith(SDValue(N, 1), Ofl);
1085
1086 return Res;
1087 }
1088
1089 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1090 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
1091 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
PromoteIntRes_ADDSUBCARRY(SDNode * N,unsigned ResNo)1092 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1093 if (ResNo == 1)
1094 return PromoteIntRes_Overflow(N);
1095
1096 // We need to sign-extend the operands so the carry value computed by the
1097 // wide operation will be equivalent to the carry value computed by the
1098 // narrow operation.
1099 // An ADDCARRY can generate carry only if any of the operands has its
1100 // most significant bit set. Sign extension propagates the most significant
1101 // bit into the higher bits which means the extra bit that the narrow
1102 // addition would need (i.e. the carry) will be propagated through the higher
1103 // bits of the wide addition.
1104 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1105 // preserved by sign extension.
1106 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1107 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1108
1109 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1110
1111 // Do the arithmetic in the wide type.
1112 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1113 LHS, RHS, N->getOperand(2));
1114
1115 // Update the users of the original carry/borrow value.
1116 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1117
1118 return SDValue(Res.getNode(), 0);
1119 }
1120
PromoteIntRes_ABS(SDNode * N)1121 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1122 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1123 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1124 }
1125
PromoteIntRes_XMULO(SDNode * N,unsigned ResNo)1126 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1127 // Promote the overflow bit trivially.
1128 if (ResNo == 1)
1129 return PromoteIntRes_Overflow(N);
1130
1131 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1132 SDLoc DL(N);
1133 EVT SmallVT = LHS.getValueType();
1134
1135 // To determine if the result overflowed in a larger type, we extend the
1136 // input to the larger type, do the multiply (checking if it overflows),
1137 // then also check the high bits of the result to see if overflow happened
1138 // there.
1139 if (N->getOpcode() == ISD::SMULO) {
1140 LHS = SExtPromotedInteger(LHS);
1141 RHS = SExtPromotedInteger(RHS);
1142 } else {
1143 LHS = ZExtPromotedInteger(LHS);
1144 RHS = ZExtPromotedInteger(RHS);
1145 }
1146 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1147 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1148
1149 // Overflow occurred if it occurred in the larger type, or if the high part
1150 // of the result does not zero/sign-extend the low part. Check this second
1151 // possibility first.
1152 SDValue Overflow;
1153 if (N->getOpcode() == ISD::UMULO) {
1154 // Unsigned overflow occurred if the high part is non-zero.
1155 unsigned Shift = SmallVT.getScalarSizeInBits();
1156 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG);
1157 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1158 DAG.getConstant(Shift, DL, ShiftTy));
1159 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1160 DAG.getConstant(0, DL, Hi.getValueType()),
1161 ISD::SETNE);
1162 } else {
1163 // Signed overflow occurred if the high part does not sign extend the low.
1164 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1165 Mul, DAG.getValueType(SmallVT));
1166 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1167 }
1168
1169 // The only other way for overflow to occur is if the multiplication in the
1170 // larger type itself overflowed.
1171 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1172 SDValue(Mul.getNode(), 1));
1173
1174 // Use the calculated overflow everywhere.
1175 ReplaceValueWith(SDValue(N, 1), Overflow);
1176 return Mul;
1177 }
1178
PromoteIntRes_UNDEF(SDNode * N)1179 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1180 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1181 N->getValueType(0)));
1182 }
1183
PromoteIntRes_VAARG(SDNode * N)1184 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1185 SDValue Chain = N->getOperand(0); // Get the chain.
1186 SDValue Ptr = N->getOperand(1); // Get the pointer.
1187 EVT VT = N->getValueType(0);
1188 SDLoc dl(N);
1189
1190 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1191 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1192 // The argument is passed as NumRegs registers of type RegVT.
1193
1194 SmallVector<SDValue, 8> Parts(NumRegs);
1195 for (unsigned i = 0; i < NumRegs; ++i) {
1196 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1197 N->getConstantOperandVal(3));
1198 Chain = Parts[i].getValue(1);
1199 }
1200
1201 // Handle endianness of the load.
1202 if (DAG.getDataLayout().isBigEndian())
1203 std::reverse(Parts.begin(), Parts.end());
1204
1205 // Assemble the parts in the promoted type.
1206 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1207 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1208 for (unsigned i = 1; i < NumRegs; ++i) {
1209 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1210 // Shift it to the right position and "or" it in.
1211 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1212 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1213 TLI.getPointerTy(DAG.getDataLayout())));
1214 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1215 }
1216
1217 // Modified the chain result - switch anything that used the old chain to
1218 // use the new one.
1219 ReplaceValueWith(SDValue(N, 1), Chain);
1220
1221 return Res;
1222 }
1223
1224 //===----------------------------------------------------------------------===//
1225 // Integer Operand Promotion
1226 //===----------------------------------------------------------------------===//
1227
1228 /// PromoteIntegerOperand - This method is called when the specified operand of
1229 /// the specified node is found to need promotion. At this point, all of the
1230 /// result types of the node are known to be legal, but other operands of the
1231 /// node may need promotion or expansion as well as the specified one.
PromoteIntegerOperand(SDNode * N,unsigned OpNo)1232 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1233 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1234 dbgs() << "\n");
1235 SDValue Res = SDValue();
1236 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1237 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1238 return false;
1239 }
1240
1241 switch (N->getOpcode()) {
1242 default:
1243 #ifndef NDEBUG
1244 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1245 N->dump(&DAG); dbgs() << "\n";
1246 #endif
1247 llvm_unreachable("Do not know how to promote this operator's operand!");
1248
1249 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1250 case ISD::ATOMIC_STORE:
1251 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1252 break;
1253 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1254 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1255 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1256 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1257 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1258 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1259 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1260 case ISD::INSERT_VECTOR_ELT:
1261 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1262 case ISD::SCALAR_TO_VECTOR:
1263 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1264 case ISD::SPLAT_VECTOR:
1265 Res = PromoteIntOp_SPLAT_VECTOR(N); break;
1266 case ISD::VSELECT:
1267 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1268 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1269 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1270 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1271 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1272 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1273 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1274 OpNo); break;
1275 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1276 OpNo); break;
1277 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1278 OpNo); break;
1279 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1280 OpNo); break;
1281 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1282 OpNo); break;
1283 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1284 case ISD::FP16_TO_FP:
1285 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1286 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1287 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1288 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1289
1290 case ISD::SHL:
1291 case ISD::SRA:
1292 case ISD::SRL:
1293 case ISD::ROTL:
1294 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1295
1296 case ISD::ADDCARRY:
1297 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1298
1299 case ISD::FRAMEADDR:
1300 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1301
1302 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1303
1304 case ISD::SMULFIX:
1305 case ISD::SMULFIXSAT:
1306 case ISD::UMULFIX:
1307 case ISD::UMULFIXSAT:
1308 case ISD::SDIVFIX:
1309 case ISD::UDIVFIX: Res = PromoteIntOp_FIX(N); break;
1310
1311 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1312
1313 case ISD::VECREDUCE_ADD:
1314 case ISD::VECREDUCE_MUL:
1315 case ISD::VECREDUCE_AND:
1316 case ISD::VECREDUCE_OR:
1317 case ISD::VECREDUCE_XOR:
1318 case ISD::VECREDUCE_SMAX:
1319 case ISD::VECREDUCE_SMIN:
1320 case ISD::VECREDUCE_UMAX:
1321 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1322 }
1323
1324 // If the result is null, the sub-method took care of registering results etc.
1325 if (!Res.getNode()) return false;
1326
1327 // If the result is N, the sub-method updated N in place. Tell the legalizer
1328 // core about this.
1329 if (Res.getNode() == N)
1330 return true;
1331
1332 const bool IsStrictFp = N->isStrictFPOpcode();
1333 assert(Res.getValueType() == N->getValueType(0) &&
1334 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1335 "Invalid operand expansion");
1336 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1337 Res.dump());
1338
1339 ReplaceValueWith(SDValue(N, 0), Res);
1340 if (IsStrictFp)
1341 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1342
1343 return false;
1344 }
1345
1346 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1347 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
PromoteSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode CCCode)1348 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1349 ISD::CondCode CCCode) {
1350 // We have to insert explicit sign or zero extends. Note that we could
1351 // insert sign extends for ALL conditions. For those operations where either
1352 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1353 // which will choose the cheapest for the target.
1354 switch (CCCode) {
1355 default: llvm_unreachable("Unknown integer comparison!");
1356 case ISD::SETEQ:
1357 case ISD::SETNE: {
1358 SDValue OpL = GetPromotedInteger(NewLHS);
1359 SDValue OpR = GetPromotedInteger(NewRHS);
1360
1361 // We would prefer to promote the comparison operand with sign extension.
1362 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1363 // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1364 // instruction, which is redundant eventually.
1365 unsigned OpLEffectiveBits =
1366 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1367 unsigned OpREffectiveBits =
1368 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1369 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1370 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1371 NewLHS = OpL;
1372 NewRHS = OpR;
1373 } else {
1374 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1375 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1376 }
1377 break;
1378 }
1379 case ISD::SETUGE:
1380 case ISD::SETUGT:
1381 case ISD::SETULE:
1382 case ISD::SETULT:
1383 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1384 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1385 break;
1386 case ISD::SETGE:
1387 case ISD::SETGT:
1388 case ISD::SETLT:
1389 case ISD::SETLE:
1390 NewLHS = SExtPromotedInteger(NewLHS);
1391 NewRHS = SExtPromotedInteger(NewRHS);
1392 break;
1393 }
1394 }
1395
PromoteIntOp_ANY_EXTEND(SDNode * N)1396 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1397 SDValue Op = GetPromotedInteger(N->getOperand(0));
1398 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1399 }
1400
PromoteIntOp_ATOMIC_STORE(AtomicSDNode * N)1401 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1402 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1403 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1404 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1405 }
1406
PromoteIntOp_BITCAST(SDNode * N)1407 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1408 // This should only occur in unusual situations like bitcasting to an
1409 // x86_fp80, so just turn it into a store+load
1410 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1411 }
1412
PromoteIntOp_BR_CC(SDNode * N,unsigned OpNo)1413 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1414 assert(OpNo == 2 && "Don't know how to promote this operand!");
1415
1416 SDValue LHS = N->getOperand(2);
1417 SDValue RHS = N->getOperand(3);
1418 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1419
1420 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1421 // legal types.
1422 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1423 N->getOperand(1), LHS, RHS, N->getOperand(4)),
1424 0);
1425 }
1426
PromoteIntOp_BRCOND(SDNode * N,unsigned OpNo)1427 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1428 assert(OpNo == 1 && "only know how to promote condition");
1429
1430 // Promote all the way up to the canonical SetCC type.
1431 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1432
1433 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1434 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1435 N->getOperand(2)), 0);
1436 }
1437
PromoteIntOp_BUILD_PAIR(SDNode * N)1438 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1439 // Since the result type is legal, the operands must promote to it.
1440 EVT OVT = N->getOperand(0).getValueType();
1441 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1442 SDValue Hi = GetPromotedInteger(N->getOperand(1));
1443 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1444 SDLoc dl(N);
1445
1446 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1447 DAG.getConstant(OVT.getSizeInBits(), dl,
1448 TLI.getPointerTy(DAG.getDataLayout())));
1449 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1450 }
1451
PromoteIntOp_BUILD_VECTOR(SDNode * N)1452 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1453 // The vector type is legal but the element type is not. This implies
1454 // that the vector is a power-of-two in length and that the element
1455 // type does not have a strange size (eg: it is not i1).
1456 EVT VecVT = N->getValueType(0);
1457 unsigned NumElts = VecVT.getVectorNumElements();
1458 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1459 "Legal vector of one illegal element?");
1460
1461 // Promote the inserted value. The type does not need to match the
1462 // vector element type. Check that any extra bits introduced will be
1463 // truncated away.
1464 assert(N->getOperand(0).getValueSizeInBits() >=
1465 N->getValueType(0).getScalarSizeInBits() &&
1466 "Type of inserted value narrower than vector element type!");
1467
1468 SmallVector<SDValue, 16> NewOps;
1469 for (unsigned i = 0; i < NumElts; ++i)
1470 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1471
1472 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1473 }
1474
PromoteIntOp_INSERT_VECTOR_ELT(SDNode * N,unsigned OpNo)1475 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1476 unsigned OpNo) {
1477 if (OpNo == 1) {
1478 // Promote the inserted value. This is valid because the type does not
1479 // have to match the vector element type.
1480
1481 // Check that any extra bits introduced will be truncated away.
1482 assert(N->getOperand(1).getValueSizeInBits() >=
1483 N->getValueType(0).getScalarSizeInBits() &&
1484 "Type of inserted value narrower than vector element type!");
1485 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1486 GetPromotedInteger(N->getOperand(1)),
1487 N->getOperand(2)),
1488 0);
1489 }
1490
1491 assert(OpNo == 2 && "Different operand and result vector types?");
1492
1493 // Promote the index.
1494 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1495 TLI.getVectorIdxTy(DAG.getDataLayout()));
1496 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1497 N->getOperand(1), Idx), 0);
1498 }
1499
PromoteIntOp_SCALAR_TO_VECTOR(SDNode * N)1500 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1501 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1502 // the operand in place.
1503 return SDValue(DAG.UpdateNodeOperands(N,
1504 GetPromotedInteger(N->getOperand(0))), 0);
1505 }
1506
PromoteIntOp_SPLAT_VECTOR(SDNode * N)1507 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1508 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
1509 // operand in place.
1510 return SDValue(
1511 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1512 }
1513
PromoteIntOp_SELECT(SDNode * N,unsigned OpNo)1514 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1515 assert(OpNo == 0 && "Only know how to promote the condition!");
1516 SDValue Cond = N->getOperand(0);
1517 EVT OpTy = N->getOperand(1).getValueType();
1518
1519 if (N->getOpcode() == ISD::VSELECT)
1520 if (SDValue Res = WidenVSELECTAndMask(N))
1521 return Res;
1522
1523 // Promote all the way up to the canonical SetCC type.
1524 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1525 Cond = PromoteTargetBoolean(Cond, OpVT);
1526
1527 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1528 N->getOperand(2)), 0);
1529 }
1530
PromoteIntOp_SELECT_CC(SDNode * N,unsigned OpNo)1531 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1532 assert(OpNo == 0 && "Don't know how to promote this operand!");
1533
1534 SDValue LHS = N->getOperand(0);
1535 SDValue RHS = N->getOperand(1);
1536 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1537
1538 // The CC (#4) and the possible return values (#2 and #3) have legal types.
1539 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1540 N->getOperand(3), N->getOperand(4)), 0);
1541 }
1542
PromoteIntOp_SETCC(SDNode * N,unsigned OpNo)1543 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1544 assert(OpNo == 0 && "Don't know how to promote this operand!");
1545
1546 SDValue LHS = N->getOperand(0);
1547 SDValue RHS = N->getOperand(1);
1548 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1549
1550 // The CC (#2) is always legal.
1551 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1552 }
1553
PromoteIntOp_Shift(SDNode * N)1554 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1555 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1556 ZExtPromotedInteger(N->getOperand(1))), 0);
1557 }
1558
PromoteIntOp_SIGN_EXTEND(SDNode * N)1559 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1560 SDValue Op = GetPromotedInteger(N->getOperand(0));
1561 SDLoc dl(N);
1562 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1563 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1564 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1565 }
1566
PromoteIntOp_SINT_TO_FP(SDNode * N)1567 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1568 return SDValue(DAG.UpdateNodeOperands(N,
1569 SExtPromotedInteger(N->getOperand(0))), 0);
1570 }
1571
PromoteIntOp_STRICT_SINT_TO_FP(SDNode * N)1572 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
1573 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1574 SExtPromotedInteger(N->getOperand(1))), 0);
1575 }
1576
PromoteIntOp_STORE(StoreSDNode * N,unsigned OpNo)1577 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1578 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1579 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1580 SDLoc dl(N);
1581
1582 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1583
1584 // Truncate the value and store the result.
1585 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1586 N->getMemoryVT(), N->getMemOperand());
1587 }
1588
PromoteIntOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)1589 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1590 unsigned OpNo) {
1591
1592 SDValue DataOp = N->getValue();
1593 EVT DataVT = DataOp.getValueType();
1594 SDValue Mask = N->getMask();
1595 SDLoc dl(N);
1596
1597 bool TruncateStore = false;
1598 if (OpNo == 4) {
1599 Mask = PromoteTargetBoolean(Mask, DataVT);
1600 // Update in place.
1601 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1602 NewOps[4] = Mask;
1603 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1604 } else { // Data operand
1605 assert(OpNo == 1 && "Unexpected operand for promotion");
1606 DataOp = GetPromotedInteger(DataOp);
1607 TruncateStore = true;
1608 }
1609
1610 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(),
1611 N->getOffset(), Mask, N->getMemoryVT(),
1612 N->getMemOperand(), N->getAddressingMode(),
1613 TruncateStore, N->isCompressingStore());
1614 }
1615
PromoteIntOp_MLOAD(MaskedLoadSDNode * N,unsigned OpNo)1616 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1617 unsigned OpNo) {
1618 assert(OpNo == 3 && "Only know how to promote the mask!");
1619 EVT DataVT = N->getValueType(0);
1620 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1621 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1622 NewOps[OpNo] = Mask;
1623 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1624 }
1625
PromoteIntOp_MGATHER(MaskedGatherSDNode * N,unsigned OpNo)1626 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1627 unsigned OpNo) {
1628
1629 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1630 if (OpNo == 2) {
1631 // The Mask
1632 EVT DataVT = N->getValueType(0);
1633 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1634 } else if (OpNo == 4) {
1635 // The Index
1636 if (N->isIndexSigned())
1637 // Need to sign extend the index since the bits will likely be used.
1638 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1639 else
1640 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1641 } else
1642 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1643
1644 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1645 }
1646
PromoteIntOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)1647 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1648 unsigned OpNo) {
1649 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1650 if (OpNo == 2) {
1651 // The Mask
1652 EVT DataVT = N->getValue().getValueType();
1653 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1654 } else if (OpNo == 4) {
1655 // The Index
1656 if (N->isIndexSigned())
1657 // Need to sign extend the index since the bits will likely be used.
1658 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1659 else
1660 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1661 } else
1662 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1663 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1664 }
1665
PromoteIntOp_TRUNCATE(SDNode * N)1666 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1667 SDValue Op = GetPromotedInteger(N->getOperand(0));
1668 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1669 }
1670
PromoteIntOp_UINT_TO_FP(SDNode * N)1671 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1672 return SDValue(DAG.UpdateNodeOperands(N,
1673 ZExtPromotedInteger(N->getOperand(0))), 0);
1674 }
1675
PromoteIntOp_STRICT_UINT_TO_FP(SDNode * N)1676 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
1677 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1678 ZExtPromotedInteger(N->getOperand(1))), 0);
1679 }
1680
PromoteIntOp_ZERO_EXTEND(SDNode * N)1681 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1682 SDLoc dl(N);
1683 SDValue Op = GetPromotedInteger(N->getOperand(0));
1684 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1685 return DAG.getZeroExtendInReg(Op, dl,
1686 N->getOperand(0).getValueType().getScalarType());
1687 }
1688
PromoteIntOp_ADDSUBCARRY(SDNode * N,unsigned OpNo)1689 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1690 assert(OpNo == 2 && "Don't know how to promote this operand!");
1691
1692 SDValue LHS = N->getOperand(0);
1693 SDValue RHS = N->getOperand(1);
1694 SDValue Carry = N->getOperand(2);
1695 SDLoc DL(N);
1696
1697 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1698
1699 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1700 }
1701
PromoteIntOp_FIX(SDNode * N)1702 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
1703 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1704 return SDValue(
1705 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1706 }
1707
PromoteIntOp_FRAMERETURNADDR(SDNode * N)1708 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1709 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1710 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1711 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1712 }
1713
PromoteIntOp_PREFETCH(SDNode * N,unsigned OpNo)1714 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1715 assert(OpNo > 1 && "Don't know how to promote this operand!");
1716 // Promote the rw, locality, and cache type arguments to a supported integer
1717 // width.
1718 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1719 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1720 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1721 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1722 Op2, Op3, Op4),
1723 0);
1724 }
1725
PromoteIntOp_FPOWI(SDNode * N)1726 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1727 SDValue Op = SExtPromotedInteger(N->getOperand(1));
1728 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1729 }
1730
PromoteIntOp_VECREDUCE(SDNode * N)1731 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1732 SDLoc dl(N);
1733 SDValue Op;
1734 switch (N->getOpcode()) {
1735 default: llvm_unreachable("Expected integer vector reduction");
1736 case ISD::VECREDUCE_ADD:
1737 case ISD::VECREDUCE_MUL:
1738 case ISD::VECREDUCE_AND:
1739 case ISD::VECREDUCE_OR:
1740 case ISD::VECREDUCE_XOR:
1741 Op = GetPromotedInteger(N->getOperand(0));
1742 break;
1743 case ISD::VECREDUCE_SMAX:
1744 case ISD::VECREDUCE_SMIN:
1745 Op = SExtPromotedInteger(N->getOperand(0));
1746 break;
1747 case ISD::VECREDUCE_UMAX:
1748 case ISD::VECREDUCE_UMIN:
1749 Op = ZExtPromotedInteger(N->getOperand(0));
1750 break;
1751 }
1752
1753 EVT EltVT = Op.getValueType().getVectorElementType();
1754 EVT VT = N->getValueType(0);
1755 if (VT.bitsGE(EltVT))
1756 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1757
1758 // Result size must be >= element size. If this is not the case after
1759 // promotion, also promote the result type and then truncate.
1760 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1761 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1762 }
1763
1764 //===----------------------------------------------------------------------===//
1765 // Integer Result Expansion
1766 //===----------------------------------------------------------------------===//
1767
1768 /// ExpandIntegerResult - This method is called when the specified result of the
1769 /// specified node is found to need expansion. At this point, the node may also
1770 /// have invalid operands or may have other results that need promotion, we just
1771 /// know that (at least) one result needs expansion.
ExpandIntegerResult(SDNode * N,unsigned ResNo)1772 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1773 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1774 dbgs() << "\n");
1775 SDValue Lo, Hi;
1776 Lo = Hi = SDValue();
1777
1778 // See if the target wants to custom expand this node.
1779 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1780 return;
1781
1782 switch (N->getOpcode()) {
1783 default:
1784 #ifndef NDEBUG
1785 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1786 N->dump(&DAG); dbgs() << "\n";
1787 #endif
1788 report_fatal_error("Do not know how to expand the result of this "
1789 "operator!");
1790
1791 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1792 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1793 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1794 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1795
1796 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1797 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1798 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1799 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1800 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1801
1802 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1803 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1804 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1805 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1806 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1807 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1808 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
1809 case ISD::CTLZ_ZERO_UNDEF:
1810 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1811 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1812 case ISD::CTTZ_ZERO_UNDEF:
1813 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1814 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1815 case ISD::STRICT_FP_TO_SINT:
1816 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1817 case ISD::STRICT_FP_TO_UINT:
1818 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1819 case ISD::STRICT_LLROUND:
1820 case ISD::STRICT_LLRINT:
1821 case ISD::LLROUND:
1822 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
1823 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1824 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1825 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1826 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1827 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1828 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1829 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1830 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1831 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1832 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1833 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1834 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1835
1836 case ISD::ATOMIC_LOAD_ADD:
1837 case ISD::ATOMIC_LOAD_SUB:
1838 case ISD::ATOMIC_LOAD_AND:
1839 case ISD::ATOMIC_LOAD_CLR:
1840 case ISD::ATOMIC_LOAD_OR:
1841 case ISD::ATOMIC_LOAD_XOR:
1842 case ISD::ATOMIC_LOAD_NAND:
1843 case ISD::ATOMIC_LOAD_MIN:
1844 case ISD::ATOMIC_LOAD_MAX:
1845 case ISD::ATOMIC_LOAD_UMIN:
1846 case ISD::ATOMIC_LOAD_UMAX:
1847 case ISD::ATOMIC_SWAP:
1848 case ISD::ATOMIC_CMP_SWAP: {
1849 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1850 SplitInteger(Tmp.first, Lo, Hi);
1851 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1852 break;
1853 }
1854 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
1855 AtomicSDNode *AN = cast<AtomicSDNode>(N);
1856 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1857 SDValue Tmp = DAG.getAtomicCmpSwap(
1858 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1859 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1860 AN->getMemOperand());
1861
1862 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1863 // success simply by comparing the loaded value against the ingoing
1864 // comparison.
1865 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1866 N->getOperand(2), ISD::SETEQ);
1867
1868 SplitInteger(Tmp, Lo, Hi);
1869 ReplaceValueWith(SDValue(N, 1), Success);
1870 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1871 break;
1872 }
1873
1874 case ISD::AND:
1875 case ISD::OR:
1876 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1877
1878 case ISD::UMAX:
1879 case ISD::SMAX:
1880 case ISD::UMIN:
1881 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1882
1883 case ISD::ADD:
1884 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1885
1886 case ISD::ADDC:
1887 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1888
1889 case ISD::ADDE:
1890 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1891
1892 case ISD::ADDCARRY:
1893 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1894
1895 case ISD::SHL:
1896 case ISD::SRA:
1897 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1898
1899 case ISD::SADDO:
1900 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1901 case ISD::UADDO:
1902 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1903 case ISD::UMULO:
1904 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1905
1906 case ISD::SADDSAT:
1907 case ISD::UADDSAT:
1908 case ISD::SSUBSAT:
1909 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1910
1911 case ISD::SMULFIX:
1912 case ISD::SMULFIXSAT:
1913 case ISD::UMULFIX:
1914 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
1915
1916 case ISD::SDIVFIX:
1917 case ISD::UDIVFIX: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
1918
1919 case ISD::VECREDUCE_ADD:
1920 case ISD::VECREDUCE_MUL:
1921 case ISD::VECREDUCE_AND:
1922 case ISD::VECREDUCE_OR:
1923 case ISD::VECREDUCE_XOR:
1924 case ISD::VECREDUCE_SMAX:
1925 case ISD::VECREDUCE_SMIN:
1926 case ISD::VECREDUCE_UMAX:
1927 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
1928 }
1929
1930 // If Lo/Hi is null, the sub-method took care of registering results etc.
1931 if (Lo.getNode())
1932 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1933 }
1934
1935 /// Lower an atomic node to the appropriate builtin call.
ExpandAtomic(SDNode * Node)1936 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1937 unsigned Opc = Node->getOpcode();
1938 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1939 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1940 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1941
1942 EVT RetVT = Node->getValueType(0);
1943 SmallVector<SDValue, 4> Ops(Node->op_begin() + 1, Node->op_end());
1944 TargetLowering::MakeLibCallOptions CallOptions;
1945 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
1946 Node->getOperand(0));
1947 }
1948
1949 /// N is a shift by a value that needs to be expanded,
1950 /// and the shift amount is a constant 'Amt'. Expand the operation.
ExpandShiftByConstant(SDNode * N,const APInt & Amt,SDValue & Lo,SDValue & Hi)1951 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1952 SDValue &Lo, SDValue &Hi) {
1953 SDLoc DL(N);
1954 // Expand the incoming operand to be shifted, so that we have its parts
1955 SDValue InL, InH;
1956 GetExpandedInteger(N->getOperand(0), InL, InH);
1957
1958 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1959 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1960 if (!Amt) {
1961 Lo = InL;
1962 Hi = InH;
1963 return;
1964 }
1965
1966 EVT NVT = InL.getValueType();
1967 unsigned VTBits = N->getValueType(0).getSizeInBits();
1968 unsigned NVTBits = NVT.getSizeInBits();
1969 EVT ShTy = N->getOperand(1).getValueType();
1970
1971 if (N->getOpcode() == ISD::SHL) {
1972 if (Amt.ugt(VTBits)) {
1973 Lo = Hi = DAG.getConstant(0, DL, NVT);
1974 } else if (Amt.ugt(NVTBits)) {
1975 Lo = DAG.getConstant(0, DL, NVT);
1976 Hi = DAG.getNode(ISD::SHL, DL,
1977 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1978 } else if (Amt == NVTBits) {
1979 Lo = DAG.getConstant(0, DL, NVT);
1980 Hi = InL;
1981 } else {
1982 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1983 Hi = DAG.getNode(ISD::OR, DL, NVT,
1984 DAG.getNode(ISD::SHL, DL, NVT, InH,
1985 DAG.getConstant(Amt, DL, ShTy)),
1986 DAG.getNode(ISD::SRL, DL, NVT, InL,
1987 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1988 }
1989 return;
1990 }
1991
1992 if (N->getOpcode() == ISD::SRL) {
1993 if (Amt.ugt(VTBits)) {
1994 Lo = Hi = DAG.getConstant(0, DL, NVT);
1995 } else if (Amt.ugt(NVTBits)) {
1996 Lo = DAG.getNode(ISD::SRL, DL,
1997 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1998 Hi = DAG.getConstant(0, DL, NVT);
1999 } else if (Amt == NVTBits) {
2000 Lo = InH;
2001 Hi = DAG.getConstant(0, DL, NVT);
2002 } else {
2003 Lo = DAG.getNode(ISD::OR, DL, NVT,
2004 DAG.getNode(ISD::SRL, DL, NVT, InL,
2005 DAG.getConstant(Amt, DL, ShTy)),
2006 DAG.getNode(ISD::SHL, DL, NVT, InH,
2007 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2008 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2009 }
2010 return;
2011 }
2012
2013 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2014 if (Amt.ugt(VTBits)) {
2015 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2016 DAG.getConstant(NVTBits - 1, DL, ShTy));
2017 } else if (Amt.ugt(NVTBits)) {
2018 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2019 DAG.getConstant(Amt - NVTBits, DL, ShTy));
2020 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2021 DAG.getConstant(NVTBits - 1, DL, ShTy));
2022 } else if (Amt == NVTBits) {
2023 Lo = InH;
2024 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2025 DAG.getConstant(NVTBits - 1, DL, ShTy));
2026 } else {
2027 Lo = DAG.getNode(ISD::OR, DL, NVT,
2028 DAG.getNode(ISD::SRL, DL, NVT, InL,
2029 DAG.getConstant(Amt, DL, ShTy)),
2030 DAG.getNode(ISD::SHL, DL, NVT, InH,
2031 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2032 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2033 }
2034 }
2035
2036 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2037 /// this shift based on knowledge of the high bit of the shift amount. If we
2038 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2039 /// shift amount.
2040 bool DAGTypeLegalizer::
ExpandShiftWithKnownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2041 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2042 SDValue Amt = N->getOperand(1);
2043 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2044 EVT ShTy = Amt.getValueType();
2045 unsigned ShBits = ShTy.getScalarSizeInBits();
2046 unsigned NVTBits = NVT.getScalarSizeInBits();
2047 assert(isPowerOf2_32(NVTBits) &&
2048 "Expanded integer type size not a power of two!");
2049 SDLoc dl(N);
2050
2051 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2052 KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2053
2054 // If we don't know anything about the high bits, exit.
2055 if (((Known.Zero|Known.One) & HighBitMask) == 0)
2056 return false;
2057
2058 // Get the incoming operand to be shifted.
2059 SDValue InL, InH;
2060 GetExpandedInteger(N->getOperand(0), InL, InH);
2061
2062 // If we know that any of the high bits of the shift amount are one, then we
2063 // can do this as a couple of simple shifts.
2064 if (Known.One.intersects(HighBitMask)) {
2065 // Mask out the high bit, which we know is set.
2066 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2067 DAG.getConstant(~HighBitMask, dl, ShTy));
2068
2069 switch (N->getOpcode()) {
2070 default: llvm_unreachable("Unknown shift");
2071 case ISD::SHL:
2072 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2073 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2074 return true;
2075 case ISD::SRL:
2076 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2077 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2078 return true;
2079 case ISD::SRA:
2080 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2081 DAG.getConstant(NVTBits - 1, dl, ShTy));
2082 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2083 return true;
2084 }
2085 }
2086
2087 // If we know that all of the high bits of the shift amount are zero, then we
2088 // can do this as a couple of simple shifts.
2089 if (HighBitMask.isSubsetOf(Known.Zero)) {
2090 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2091 // shift if x is zero. We can use XOR here because x is known to be smaller
2092 // than 32.
2093 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2094 DAG.getConstant(NVTBits - 1, dl, ShTy));
2095
2096 unsigned Op1, Op2;
2097 switch (N->getOpcode()) {
2098 default: llvm_unreachable("Unknown shift");
2099 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2100 case ISD::SRL:
2101 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2102 }
2103
2104 // When shifting right the arithmetic for Lo and Hi is swapped.
2105 if (N->getOpcode() != ISD::SHL)
2106 std::swap(InL, InH);
2107
2108 // Use a little trick to get the bits that move from Lo to Hi. First
2109 // shift by one bit.
2110 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2111 // Then compute the remaining shift with amount-1.
2112 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2113
2114 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2115 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2116
2117 if (N->getOpcode() != ISD::SHL)
2118 std::swap(Hi, Lo);
2119 return true;
2120 }
2121
2122 return false;
2123 }
2124
2125 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2126 /// of any size.
2127 bool DAGTypeLegalizer::
ExpandShiftWithUnknownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2128 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2129 SDValue Amt = N->getOperand(1);
2130 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2131 EVT ShTy = Amt.getValueType();
2132 unsigned NVTBits = NVT.getSizeInBits();
2133 assert(isPowerOf2_32(NVTBits) &&
2134 "Expanded integer type size not a power of two!");
2135 SDLoc dl(N);
2136
2137 // Get the incoming operand to be shifted.
2138 SDValue InL, InH;
2139 GetExpandedInteger(N->getOperand(0), InL, InH);
2140
2141 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2142 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2143 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2144 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2145 Amt, NVBitsNode, ISD::SETULT);
2146 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2147 Amt, DAG.getConstant(0, dl, ShTy),
2148 ISD::SETEQ);
2149
2150 SDValue LoS, HiS, LoL, HiL;
2151 switch (N->getOpcode()) {
2152 default: llvm_unreachable("Unknown shift");
2153 case ISD::SHL:
2154 // Short: ShAmt < NVTBits
2155 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2156 HiS = DAG.getNode(ISD::OR, dl, NVT,
2157 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2158 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2159
2160 // Long: ShAmt >= NVTBits
2161 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2162 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2163
2164 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2165 Hi = DAG.getSelect(dl, NVT, isZero, InH,
2166 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2167 return true;
2168 case ISD::SRL:
2169 // Short: ShAmt < NVTBits
2170 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2171 LoS = DAG.getNode(ISD::OR, dl, NVT,
2172 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2173 // FIXME: If Amt is zero, the following shift generates an undefined result
2174 // on some architectures.
2175 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2176
2177 // Long: ShAmt >= NVTBits
2178 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2179 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2180
2181 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2182 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2183 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2184 return true;
2185 case ISD::SRA:
2186 // Short: ShAmt < NVTBits
2187 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2188 LoS = DAG.getNode(ISD::OR, dl, NVT,
2189 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2190 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2191
2192 // Long: ShAmt >= NVTBits
2193 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2194 DAG.getConstant(NVTBits - 1, dl, ShTy));
2195 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2196
2197 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2198 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2199 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2200 return true;
2201 }
2202 }
2203
getExpandedMinMaxOps(int Op)2204 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2205
2206 switch (Op) {
2207 default: llvm_unreachable("invalid min/max opcode");
2208 case ISD::SMAX:
2209 return std::make_pair(ISD::SETGT, ISD::UMAX);
2210 case ISD::UMAX:
2211 return std::make_pair(ISD::SETUGT, ISD::UMAX);
2212 case ISD::SMIN:
2213 return std::make_pair(ISD::SETLT, ISD::UMIN);
2214 case ISD::UMIN:
2215 return std::make_pair(ISD::SETULT, ISD::UMIN);
2216 }
2217 }
2218
ExpandIntRes_MINMAX(SDNode * N,SDValue & Lo,SDValue & Hi)2219 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2220 SDValue &Lo, SDValue &Hi) {
2221 SDLoc DL(N);
2222 ISD::NodeType LoOpc;
2223 ISD::CondCode CondC;
2224 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2225
2226 // Expand the subcomponents.
2227 SDValue LHSL, LHSH, RHSL, RHSH;
2228 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2229 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2230
2231 // Value types
2232 EVT NVT = LHSL.getValueType();
2233 EVT CCT = getSetCCResultType(NVT);
2234
2235 // Hi part is always the same op
2236 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2237
2238 // We need to know whether to select Lo part that corresponds to 'winning'
2239 // Hi part or if Hi parts are equal.
2240 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2241 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2242
2243 // Lo part corresponding to the 'winning' Hi part
2244 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2245
2246 // Recursed Lo part if Hi parts are equal, this uses unsigned version
2247 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2248
2249 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2250 }
2251
ExpandIntRes_ADDSUB(SDNode * N,SDValue & Lo,SDValue & Hi)2252 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2253 SDValue &Lo, SDValue &Hi) {
2254 SDLoc dl(N);
2255 // Expand the subcomponents.
2256 SDValue LHSL, LHSH, RHSL, RHSH;
2257 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2258 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2259
2260 EVT NVT = LHSL.getValueType();
2261 SDValue LoOps[2] = { LHSL, RHSL };
2262 SDValue HiOps[3] = { LHSH, RHSH };
2263
2264 bool HasOpCarry = TLI.isOperationLegalOrCustom(
2265 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2266 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2267 if (HasOpCarry) {
2268 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2269 if (N->getOpcode() == ISD::ADD) {
2270 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2271 HiOps[2] = Lo.getValue(1);
2272 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2273 } else {
2274 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2275 HiOps[2] = Lo.getValue(1);
2276 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2277 }
2278 return;
2279 }
2280
2281 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2282 // them. TODO: Teach operation legalization how to expand unsupported
2283 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2284 // a carry of type MVT::Glue, but there doesn't seem to be any way to
2285 // generate a value of this type in the expanded code sequence.
2286 bool hasCarry =
2287 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2288 ISD::ADDC : ISD::SUBC,
2289 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2290
2291 if (hasCarry) {
2292 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2293 if (N->getOpcode() == ISD::ADD) {
2294 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2295 HiOps[2] = Lo.getValue(1);
2296 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2297 } else {
2298 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2299 HiOps[2] = Lo.getValue(1);
2300 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2301 }
2302 return;
2303 }
2304
2305 bool hasOVF =
2306 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2307 ISD::UADDO : ISD::USUBO,
2308 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2309 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
2310
2311 if (hasOVF) {
2312 EVT OvfVT = getSetCCResultType(NVT);
2313 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2314 int RevOpc;
2315 if (N->getOpcode() == ISD::ADD) {
2316 RevOpc = ISD::SUB;
2317 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2318 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2319 } else {
2320 RevOpc = ISD::ADD;
2321 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2322 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2323 }
2324 SDValue OVF = Lo.getValue(1);
2325
2326 switch (BoolType) {
2327 case TargetLoweringBase::UndefinedBooleanContent:
2328 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2329 LLVM_FALLTHROUGH;
2330 case TargetLoweringBase::ZeroOrOneBooleanContent:
2331 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2332 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2333 break;
2334 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2335 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2336 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2337 }
2338 return;
2339 }
2340
2341 if (N->getOpcode() == ISD::ADD) {
2342 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2343 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2344 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2345 ISD::SETULT);
2346
2347 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
2348 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2349 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2350 return;
2351 }
2352
2353 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2354 DAG.getConstant(1, dl, NVT),
2355 DAG.getConstant(0, dl, NVT));
2356 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2357 ISD::SETULT);
2358 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2359 DAG.getConstant(1, dl, NVT), Carry1);
2360 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2361 } else {
2362 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2363 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2364 SDValue Cmp =
2365 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2366 LoOps[0], LoOps[1], ISD::SETULT);
2367
2368 SDValue Borrow;
2369 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
2370 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2371 else
2372 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2373 DAG.getConstant(0, dl, NVT));
2374
2375 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2376 }
2377 }
2378
ExpandIntRes_ADDSUBC(SDNode * N,SDValue & Lo,SDValue & Hi)2379 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2380 SDValue &Lo, SDValue &Hi) {
2381 // Expand the subcomponents.
2382 SDValue LHSL, LHSH, RHSL, RHSH;
2383 SDLoc dl(N);
2384 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2385 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2386 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2387 SDValue LoOps[2] = { LHSL, RHSL };
2388 SDValue HiOps[3] = { LHSH, RHSH };
2389
2390 if (N->getOpcode() == ISD::ADDC) {
2391 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2392 HiOps[2] = Lo.getValue(1);
2393 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2394 } else {
2395 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2396 HiOps[2] = Lo.getValue(1);
2397 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2398 }
2399
2400 // Legalized the flag result - switch anything that used the old flag to
2401 // use the new one.
2402 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2403 }
2404
ExpandIntRes_ADDSUBE(SDNode * N,SDValue & Lo,SDValue & Hi)2405 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2406 SDValue &Lo, SDValue &Hi) {
2407 // Expand the subcomponents.
2408 SDValue LHSL, LHSH, RHSL, RHSH;
2409 SDLoc dl(N);
2410 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2411 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2412 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2413 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2414 SDValue HiOps[3] = { LHSH, RHSH };
2415
2416 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2417 HiOps[2] = Lo.getValue(1);
2418 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2419
2420 // Legalized the flag result - switch anything that used the old flag to
2421 // use the new one.
2422 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2423 }
2424
ExpandIntRes_UADDSUBO(SDNode * N,SDValue & Lo,SDValue & Hi)2425 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2426 SDValue &Lo, SDValue &Hi) {
2427 SDValue LHS = N->getOperand(0);
2428 SDValue RHS = N->getOperand(1);
2429 SDLoc dl(N);
2430
2431 SDValue Ovf;
2432
2433 unsigned CarryOp, NoCarryOp;
2434 ISD::CondCode Cond;
2435 switch(N->getOpcode()) {
2436 case ISD::UADDO:
2437 CarryOp = ISD::ADDCARRY;
2438 NoCarryOp = ISD::ADD;
2439 Cond = ISD::SETULT;
2440 break;
2441 case ISD::USUBO:
2442 CarryOp = ISD::SUBCARRY;
2443 NoCarryOp = ISD::SUB;
2444 Cond = ISD::SETUGT;
2445 break;
2446 default:
2447 llvm_unreachable("Node has unexpected Opcode");
2448 }
2449
2450 bool HasCarryOp = TLI.isOperationLegalOrCustom(
2451 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2452
2453 if (HasCarryOp) {
2454 // Expand the subcomponents.
2455 SDValue LHSL, LHSH, RHSL, RHSH;
2456 GetExpandedInteger(LHS, LHSL, LHSH);
2457 GetExpandedInteger(RHS, RHSL, RHSH);
2458 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2459 SDValue LoOps[2] = { LHSL, RHSL };
2460 SDValue HiOps[3] = { LHSH, RHSH };
2461
2462 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2463 HiOps[2] = Lo.getValue(1);
2464 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
2465
2466 Ovf = Hi.getValue(1);
2467 } else {
2468 // Expand the result by simply replacing it with the equivalent
2469 // non-overflow-checking operation.
2470 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2471 SplitInteger(Sum, Lo, Hi);
2472
2473 // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2474 // overflows iff a - b > a.
2475 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2476 }
2477
2478 // Legalized the flag result - switch anything that used the old flag to
2479 // use the new one.
2480 ReplaceValueWith(SDValue(N, 1), Ovf);
2481 }
2482
ExpandIntRes_ADDSUBCARRY(SDNode * N,SDValue & Lo,SDValue & Hi)2483 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2484 SDValue &Lo, SDValue &Hi) {
2485 // Expand the subcomponents.
2486 SDValue LHSL, LHSH, RHSL, RHSH;
2487 SDLoc dl(N);
2488 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2489 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2490 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2491 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2492 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2493
2494 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2495 HiOps[2] = Lo.getValue(1);
2496 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2497
2498 // Legalized the flag result - switch anything that used the old flag to
2499 // use the new one.
2500 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2501 }
2502
ExpandIntRes_ANY_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)2503 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2504 SDValue &Lo, SDValue &Hi) {
2505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2506 SDLoc dl(N);
2507 SDValue Op = N->getOperand(0);
2508 if (Op.getValueType().bitsLE(NVT)) {
2509 // The low part is any extension of the input (which degenerates to a copy).
2510 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2511 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2512 } else {
2513 // For example, extension of an i48 to an i64. The operand type necessarily
2514 // promotes to the result type, so will end up being expanded too.
2515 assert(getTypeAction(Op.getValueType()) ==
2516 TargetLowering::TypePromoteInteger &&
2517 "Only know how to promote this result!");
2518 SDValue Res = GetPromotedInteger(Op);
2519 assert(Res.getValueType() == N->getValueType(0) &&
2520 "Operand over promoted?");
2521 // Split the promoted operand. This will simplify when it is expanded.
2522 SplitInteger(Res, Lo, Hi);
2523 }
2524 }
2525
ExpandIntRes_AssertSext(SDNode * N,SDValue & Lo,SDValue & Hi)2526 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2527 SDValue &Lo, SDValue &Hi) {
2528 SDLoc dl(N);
2529 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2530 EVT NVT = Lo.getValueType();
2531 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2532 unsigned NVTBits = NVT.getSizeInBits();
2533 unsigned EVTBits = EVT.getSizeInBits();
2534
2535 if (NVTBits < EVTBits) {
2536 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2537 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2538 EVTBits - NVTBits)));
2539 } else {
2540 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2541 // The high part replicates the sign bit of Lo, make it explicit.
2542 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2543 DAG.getConstant(NVTBits - 1, dl,
2544 TLI.getPointerTy(DAG.getDataLayout())));
2545 }
2546 }
2547
ExpandIntRes_AssertZext(SDNode * N,SDValue & Lo,SDValue & Hi)2548 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2549 SDValue &Lo, SDValue &Hi) {
2550 SDLoc dl(N);
2551 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2552 EVT NVT = Lo.getValueType();
2553 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2554 unsigned NVTBits = NVT.getSizeInBits();
2555 unsigned EVTBits = EVT.getSizeInBits();
2556
2557 if (NVTBits < EVTBits) {
2558 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2559 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2560 EVTBits - NVTBits)));
2561 } else {
2562 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2563 // The high part must be zero, make it explicit.
2564 Hi = DAG.getConstant(0, dl, NVT);
2565 }
2566 }
2567
ExpandIntRes_BITREVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)2568 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2569 SDValue &Lo, SDValue &Hi) {
2570 SDLoc dl(N);
2571 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2572 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2573 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2574 }
2575
ExpandIntRes_BSWAP(SDNode * N,SDValue & Lo,SDValue & Hi)2576 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2577 SDValue &Lo, SDValue &Hi) {
2578 SDLoc dl(N);
2579 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2580 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2581 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2582 }
2583
ExpandIntRes_Constant(SDNode * N,SDValue & Lo,SDValue & Hi)2584 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2585 SDValue &Lo, SDValue &Hi) {
2586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2587 unsigned NBitWidth = NVT.getSizeInBits();
2588 auto Constant = cast<ConstantSDNode>(N);
2589 const APInt &Cst = Constant->getAPIntValue();
2590 bool IsTarget = Constant->isTargetOpcode();
2591 bool IsOpaque = Constant->isOpaque();
2592 SDLoc dl(N);
2593 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2594 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2595 IsOpaque);
2596 }
2597
ExpandIntRes_ABS(SDNode * N,SDValue & Lo,SDValue & Hi)2598 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2599 SDLoc dl(N);
2600
2601 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2602 EVT VT = N->getValueType(0);
2603 SDValue N0 = N->getOperand(0);
2604 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2605 DAG.getConstant(0, dl, VT), N0);
2606 SDValue NegLo, NegHi;
2607 SplitInteger(Neg, NegLo, NegHi);
2608
2609 GetExpandedInteger(N0, Lo, Hi);
2610 EVT NVT = Lo.getValueType();
2611 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2612 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2613 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2614 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2615 }
2616
ExpandIntRes_CTLZ(SDNode * N,SDValue & Lo,SDValue & Hi)2617 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2618 SDValue &Lo, SDValue &Hi) {
2619 SDLoc dl(N);
2620 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2621 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2622 EVT NVT = Lo.getValueType();
2623
2624 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2625 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2626
2627 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2628 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2629
2630 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2631 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2632 DAG.getConstant(NVT.getSizeInBits(), dl,
2633 NVT)));
2634 Hi = DAG.getConstant(0, dl, NVT);
2635 }
2636
ExpandIntRes_CTPOP(SDNode * N,SDValue & Lo,SDValue & Hi)2637 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2638 SDValue &Lo, SDValue &Hi) {
2639 SDLoc dl(N);
2640 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2641 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2642 EVT NVT = Lo.getValueType();
2643 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2644 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2645 Hi = DAG.getConstant(0, dl, NVT);
2646 }
2647
ExpandIntRes_CTTZ(SDNode * N,SDValue & Lo,SDValue & Hi)2648 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2649 SDValue &Lo, SDValue &Hi) {
2650 SDLoc dl(N);
2651 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2652 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2653 EVT NVT = Lo.getValueType();
2654
2655 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2656 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2657
2658 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2659 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2660
2661 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2662 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2663 DAG.getConstant(NVT.getSizeInBits(), dl,
2664 NVT)));
2665 Hi = DAG.getConstant(0, dl, NVT);
2666 }
2667
ExpandIntRes_FLT_ROUNDS(SDNode * N,SDValue & Lo,SDValue & Hi)2668 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2669 SDValue &Hi) {
2670 SDLoc dl(N);
2671 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2672 unsigned NBitWidth = NVT.getSizeInBits();
2673
2674 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2675 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2676 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2677 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2678 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2679 }
2680
ExpandIntRes_FP_TO_SINT(SDNode * N,SDValue & Lo,SDValue & Hi)2681 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2682 SDValue &Hi) {
2683 SDLoc dl(N);
2684 EVT VT = N->getValueType(0);
2685
2686 bool IsStrict = N->isStrictFPOpcode();
2687 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2688 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2689 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2690 Op = GetPromotedFloat(Op);
2691
2692 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
2693 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2694 TargetLowering::MakeLibCallOptions CallOptions;
2695 CallOptions.setSExt(true);
2696 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
2697 CallOptions, dl, Chain);
2698 SplitInteger(Tmp.first, Lo, Hi);
2699
2700 if (IsStrict)
2701 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2702 }
2703
ExpandIntRes_FP_TO_UINT(SDNode * N,SDValue & Lo,SDValue & Hi)2704 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2705 SDValue &Hi) {
2706 SDLoc dl(N);
2707 EVT VT = N->getValueType(0);
2708
2709 bool IsStrict = N->isStrictFPOpcode();
2710 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2711 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2712 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2713 Op = GetPromotedFloat(Op);
2714
2715 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
2716 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2717 TargetLowering::MakeLibCallOptions CallOptions;
2718 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
2719 CallOptions, dl, Chain);
2720 SplitInteger(Tmp.first, Lo, Hi);
2721
2722 if (IsStrict)
2723 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2724 }
2725
ExpandIntRes_LLROUND_LLRINT(SDNode * N,SDValue & Lo,SDValue & Hi)2726 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
2727 SDValue &Hi) {
2728 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
2729
2730 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
2731 "Input type needs to be promoted!");
2732
2733 EVT VT = Op.getValueType();
2734
2735 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2736 if (N->getOpcode() == ISD::LLROUND ||
2737 N->getOpcode() == ISD::STRICT_LLROUND) {
2738 if (VT == MVT::f32)
2739 LC = RTLIB::LLROUND_F32;
2740 else if (VT == MVT::f64)
2741 LC = RTLIB::LLROUND_F64;
2742 else if (VT == MVT::f80)
2743 LC = RTLIB::LLROUND_F80;
2744 else if (VT == MVT::f128)
2745 LC = RTLIB::LLROUND_F128;
2746 else if (VT == MVT::ppcf128)
2747 LC = RTLIB::LLROUND_PPCF128;
2748 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
2749 } else if (N->getOpcode() == ISD::LLRINT ||
2750 N->getOpcode() == ISD::STRICT_LLRINT) {
2751 if (VT == MVT::f32)
2752 LC = RTLIB::LLRINT_F32;
2753 else if (VT == MVT::f64)
2754 LC = RTLIB::LLRINT_F64;
2755 else if (VT == MVT::f80)
2756 LC = RTLIB::LLRINT_F80;
2757 else if (VT == MVT::f128)
2758 LC = RTLIB::LLRINT_F128;
2759 else if (VT == MVT::ppcf128)
2760 LC = RTLIB::LLRINT_PPCF128;
2761 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
2762 } else
2763 llvm_unreachable("Unexpected opcode!");
2764
2765 SDLoc dl(N);
2766 EVT RetVT = N->getValueType(0);
2767 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();
2768
2769 TargetLowering::MakeLibCallOptions CallOptions;
2770 CallOptions.setSExt(true);
2771 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2772 Op, CallOptions, dl,
2773 Chain);
2774 SplitInteger(Tmp.first, Lo, Hi);
2775
2776 if (N->isStrictFPOpcode())
2777 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2778 }
2779
ExpandIntRes_LOAD(LoadSDNode * N,SDValue & Lo,SDValue & Hi)2780 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2781 SDValue &Lo, SDValue &Hi) {
2782 if (N->isAtomic()) {
2783 // It's typical to have larger CAS than atomic load instructions.
2784 SDLoc dl(N);
2785 EVT VT = N->getMemoryVT();
2786 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2787 SDValue Zero = DAG.getConstant(0, dl, VT);
2788 SDValue Swap = DAG.getAtomicCmpSwap(
2789 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
2790 VT, VTs, N->getOperand(0),
2791 N->getOperand(1), Zero, Zero, N->getMemOperand());
2792 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2793 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2794 return;
2795 }
2796
2797 if (ISD::isNormalLoad(N)) {
2798 ExpandRes_NormalLoad(N, Lo, Hi);
2799 return;
2800 }
2801
2802 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2803
2804 EVT VT = N->getValueType(0);
2805 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2806 SDValue Ch = N->getChain();
2807 SDValue Ptr = N->getBasePtr();
2808 ISD::LoadExtType ExtType = N->getExtensionType();
2809 unsigned Alignment = N->getAlignment();
2810 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2811 AAMDNodes AAInfo = N->getAAInfo();
2812 SDLoc dl(N);
2813
2814 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2815
2816 if (N->getMemoryVT().bitsLE(NVT)) {
2817 EVT MemVT = N->getMemoryVT();
2818
2819 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2820 Alignment, MMOFlags, AAInfo);
2821
2822 // Remember the chain.
2823 Ch = Lo.getValue(1);
2824
2825 if (ExtType == ISD::SEXTLOAD) {
2826 // The high part is obtained by SRA'ing all but one of the bits of the
2827 // lo part.
2828 unsigned LoSize = Lo.getValueSizeInBits();
2829 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2830 DAG.getConstant(LoSize - 1, dl,
2831 TLI.getPointerTy(DAG.getDataLayout())));
2832 } else if (ExtType == ISD::ZEXTLOAD) {
2833 // The high part is just a zero.
2834 Hi = DAG.getConstant(0, dl, NVT);
2835 } else {
2836 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2837 // The high part is undefined.
2838 Hi = DAG.getUNDEF(NVT);
2839 }
2840 } else if (DAG.getDataLayout().isLittleEndian()) {
2841 // Little-endian - low bits are at low addresses.
2842 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2843 AAInfo);
2844
2845 unsigned ExcessBits =
2846 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2847 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2848
2849 // Increment the pointer to the other half.
2850 unsigned IncrementSize = NVT.getSizeInBits()/8;
2851 Ptr = DAG.getMemBasePlusOffset(Ptr, IncrementSize, dl);
2852 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2853 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2854 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2855
2856 // Build a factor node to remember that this load is independent of the
2857 // other one.
2858 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2859 Hi.getValue(1));
2860 } else {
2861 // Big-endian - high bits are at low addresses. Favor aligned loads at
2862 // the cost of some bit-fiddling.
2863 EVT MemVT = N->getMemoryVT();
2864 unsigned EBytes = MemVT.getStoreSize();
2865 unsigned IncrementSize = NVT.getSizeInBits()/8;
2866 unsigned ExcessBits = (EBytes - IncrementSize)*8;
2867
2868 // Load both the high bits and maybe some of the low bits.
2869 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2870 EVT::getIntegerVT(*DAG.getContext(),
2871 MemVT.getSizeInBits() - ExcessBits),
2872 Alignment, MMOFlags, AAInfo);
2873
2874 // Increment the pointer to the other half.
2875 Ptr = DAG.getMemBasePlusOffset(Ptr, IncrementSize, dl);
2876 // Load the rest of the low bits.
2877 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2878 N->getPointerInfo().getWithOffset(IncrementSize),
2879 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2880 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2881
2882 // Build a factor node to remember that this load is independent of the
2883 // other one.
2884 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2885 Hi.getValue(1));
2886
2887 if (ExcessBits < NVT.getSizeInBits()) {
2888 // Transfer low bits from the bottom of Hi to the top of Lo.
2889 Lo = DAG.getNode(
2890 ISD::OR, dl, NVT, Lo,
2891 DAG.getNode(ISD::SHL, dl, NVT, Hi,
2892 DAG.getConstant(ExcessBits, dl,
2893 TLI.getPointerTy(DAG.getDataLayout()))));
2894 // Move high bits to the right position in Hi.
2895 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2896 Hi,
2897 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2898 TLI.getPointerTy(DAG.getDataLayout())));
2899 }
2900 }
2901
2902 // Legalize the chain result - switch anything that used the old chain to
2903 // use the new one.
2904 ReplaceValueWith(SDValue(N, 1), Ch);
2905 }
2906
ExpandIntRes_Logical(SDNode * N,SDValue & Lo,SDValue & Hi)2907 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2908 SDValue &Lo, SDValue &Hi) {
2909 SDLoc dl(N);
2910 SDValue LL, LH, RL, RH;
2911 GetExpandedInteger(N->getOperand(0), LL, LH);
2912 GetExpandedInteger(N->getOperand(1), RL, RH);
2913 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2914 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2915 }
2916
ExpandIntRes_MUL(SDNode * N,SDValue & Lo,SDValue & Hi)2917 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2918 SDValue &Lo, SDValue &Hi) {
2919 EVT VT = N->getValueType(0);
2920 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2921 SDLoc dl(N);
2922
2923 SDValue LL, LH, RL, RH;
2924 GetExpandedInteger(N->getOperand(0), LL, LH);
2925 GetExpandedInteger(N->getOperand(1), RL, RH);
2926
2927 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2928 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
2929 LL, LH, RL, RH))
2930 return;
2931
2932 // If nothing else, we can make a libcall.
2933 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2934 if (VT == MVT::i16)
2935 LC = RTLIB::MUL_I16;
2936 else if (VT == MVT::i32)
2937 LC = RTLIB::MUL_I32;
2938 else if (VT == MVT::i64)
2939 LC = RTLIB::MUL_I64;
2940 else if (VT == MVT::i128)
2941 LC = RTLIB::MUL_I128;
2942
2943 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2944 // We'll expand the multiplication by brute force because we have no other
2945 // options. This is a trivially-generalized version of the code from
2946 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2947 // 4.3.1).
2948 unsigned Bits = NVT.getSizeInBits();
2949 unsigned HalfBits = Bits >> 1;
2950 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2951 NVT);
2952 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2953 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2954
2955 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2956 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2957
2958 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2959 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2960 // The type from TLI is too small to fit the shift amount we want.
2961 // Override it with i32. The shift will have to be legalized.
2962 ShiftAmtTy = MVT::i32;
2963 }
2964 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2965 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2966 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2967 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2968
2969 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2970 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2971 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2972 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2973
2974 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2975 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2976 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2977
2978 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2979 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2980 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2981 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2982 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2983
2984 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2985 DAG.getNode(ISD::ADD, dl, NVT,
2986 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2987 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2988 return;
2989 }
2990
2991 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2992 TargetLowering::MakeLibCallOptions CallOptions;
2993 CallOptions.setSExt(true);
2994 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
2995 Lo, Hi);
2996 }
2997
ExpandIntRes_READCYCLECOUNTER(SDNode * N,SDValue & Lo,SDValue & Hi)2998 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2999 SDValue &Hi) {
3000 SDLoc DL(N);
3001 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3002 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
3003 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
3004 Lo = R.getValue(0);
3005 Hi = R.getValue(1);
3006 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3007 }
3008
ExpandIntRes_ADDSUBSAT(SDNode * N,SDValue & Lo,SDValue & Hi)3009 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
3010 SDValue &Hi) {
3011 SDValue Result = TLI.expandAddSubSat(N, DAG);
3012 SplitInteger(Result, Lo, Hi);
3013 }
3014
3015 /// This performs an expansion of the integer result for a fixed point
3016 /// multiplication. The default expansion performs rounding down towards
3017 /// negative infinity, though targets that do care about rounding should specify
3018 /// a target hook for rounding and provide their own expansion or lowering of
3019 /// fixed point multiplication to be consistent with rounding.
ExpandIntRes_MULFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3020 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
3021 SDValue &Hi) {
3022 SDLoc dl(N);
3023 EVT VT = N->getValueType(0);
3024 unsigned VTSize = VT.getScalarSizeInBits();
3025 SDValue LHS = N->getOperand(0);
3026 SDValue RHS = N->getOperand(1);
3027 uint64_t Scale = N->getConstantOperandVal(2);
3028 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
3029 N->getOpcode() == ISD::UMULFIXSAT);
3030 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
3031 N->getOpcode() == ISD::SMULFIXSAT);
3032
3033 // Handle special case when scale is equal to zero.
3034 if (!Scale) {
3035 SDValue Result;
3036 if (!Saturating) {
3037 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3038 } else {
3039 EVT BoolVT = getSetCCResultType(VT);
3040 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
3041 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
3042 SDValue Product = Result.getValue(0);
3043 SDValue Overflow = Result.getValue(1);
3044 if (Signed) {
3045 APInt MinVal = APInt::getSignedMinValue(VTSize);
3046 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
3047 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
3048 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3049 SDValue Zero = DAG.getConstant(0, dl, VT);
3050 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
3051 Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
3052 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
3053 } else {
3054 // For unsigned multiplication, we only need to check the max since we
3055 // can't really overflow towards zero.
3056 APInt MaxVal = APInt::getMaxValue(VTSize);
3057 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3058 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
3059 }
3060 }
3061 SplitInteger(Result, Lo, Hi);
3062 return;
3063 }
3064
3065 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
3066 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
3067 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
3068
3069 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3070 SDValue LL, LH, RL, RH;
3071 GetExpandedInteger(LHS, LL, LH);
3072 GetExpandedInteger(RHS, RL, RH);
3073 SmallVector<SDValue, 4> Result;
3074
3075 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
3076 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
3077 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3078 LL, LH, RL, RH)) {
3079 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
3080 return;
3081 }
3082
3083 unsigned NVTSize = NVT.getScalarSizeInBits();
3084 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
3085 "the size of the current value type");
3086 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3087
3088 // After getting the multiplication result in 4 parts, we need to perform a
3089 // shift right by the amount of the scale to get the result in that scale.
3090 //
3091 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
3092 // 128 bits that are cut into 4 32-bit parts:
3093 //
3094 // HH HL LH LL
3095 // |---32---|---32---|---32---|---32---|
3096 // 128 96 64 32 0
3097 //
3098 // |------VTSize-----|
3099 //
3100 // |NVTSize-|
3101 //
3102 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
3103 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
3104 // amount and get Lo and Hi using two funnel shifts. Or for the special case
3105 // when Scale is a multiple of NVTSize we can just pick the result without
3106 // shifting.
3107 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
3108 if (Scale % NVTSize) {
3109 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
3110 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
3111 ShiftAmount);
3112 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
3113 ShiftAmount);
3114 } else {
3115 Lo = Result[Part0];
3116 Hi = Result[Part0 + 1];
3117 }
3118
3119 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
3120 if (!Saturating)
3121 return;
3122
3123 // Can not overflow when there is no integer part.
3124 if (Scale == VTSize)
3125 return;
3126
3127 // To handle saturation we must check for overflow in the multiplication.
3128 //
3129 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
3130 // aren't all zeroes.
3131 //
3132 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
3133 // aren't all ones or all zeroes.
3134 //
3135 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
3136 // highest bit of HH determines saturation direction in the event of signed
3137 // saturation.
3138
3139 SDValue ResultHL = Result[2];
3140 SDValue ResultHH = Result[3];
3141
3142 SDValue SatMax, SatMin;
3143 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
3144 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3145 EVT BoolNVT = getSetCCResultType(NVT);
3146
3147 if (!Signed) {
3148 if (Scale < NVTSize) {
3149 // Overflow happened if ((HH | (HL >> Scale)) != 0).
3150 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3151 DAG.getConstant(Scale, dl, ShiftTy));
3152 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3153 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3154 } else if (Scale == NVTSize) {
3155 // Overflow happened if (HH != 0).
3156 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3157 } else if (Scale < VTSize) {
3158 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
3159 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3160 DAG.getConstant(Scale - NVTSize, dl,
3161 ShiftTy));
3162 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3163 } else
3164 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
3165 "(and saturation can't happen with Scale==VTSize).");
3166
3167 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3168 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3169 return;
3170 }
3171
3172 if (Scale < NVTSize) {
3173 // The number of overflow bits we can check are VTSize - Scale + 1 (we
3174 // include the sign bit). If these top bits are > 0, then we overflowed past
3175 // the max value. If these top bits are < -1, then we overflowed past the
3176 // min value. Otherwise, we did not overflow.
3177 unsigned OverflowBits = VTSize - Scale + 1;
3178 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3179 "Extent of overflow bits must start within HL");
3180 SDValue HLHiMask = DAG.getConstant(
3181 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3182 SDValue HLLoMask = DAG.getConstant(
3183 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3184 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3185 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3186 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3187 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3188 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3189 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3190 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3191 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3192 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3193 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3194 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3195 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3196 } else if (Scale == NVTSize) {
3197 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3198 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3199 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3200 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3201 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3202 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3203 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3204 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3205 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3206 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3207 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3208 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3209 } else if (Scale < VTSize) {
3210 // This is similar to the case when we saturate if Scale < NVTSize, but we
3211 // only need to check HH.
3212 unsigned OverflowBits = VTSize - Scale + 1;
3213 SDValue HHHiMask = DAG.getConstant(
3214 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3215 SDValue HHLoMask = DAG.getConstant(
3216 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3217 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3218 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3219 } else
3220 llvm_unreachable("Illegal scale for signed fixed point mul.");
3221
3222 // Saturate to signed maximum.
3223 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3224 APInt MaxLo = APInt::getAllOnesValue(NVTSize);
3225 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3226 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3227 // Saturate to signed minimum.
3228 APInt MinHi = APInt::getSignedMinValue(NVTSize);
3229 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3230 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3231 }
3232
ExpandIntRes_DIVFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3233 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
3234 SDValue &Hi) {
3235 SDValue Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
3236 N->getConstantOperandVal(2), TLI, DAG);
3237 SplitInteger(Res, Lo, Hi);
3238 }
3239
ExpandIntRes_SADDSUBO(SDNode * Node,SDValue & Lo,SDValue & Hi)3240 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3241 SDValue &Lo, SDValue &Hi) {
3242 SDValue LHS = Node->getOperand(0);
3243 SDValue RHS = Node->getOperand(1);
3244 SDLoc dl(Node);
3245
3246 // Expand the result by simply replacing it with the equivalent
3247 // non-overflow-checking operation.
3248 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3249 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3250 LHS, RHS);
3251 SplitInteger(Sum, Lo, Hi);
3252
3253 // Compute the overflow.
3254 //
3255 // LHSSign -> LHS >= 0
3256 // RHSSign -> RHS >= 0
3257 // SumSign -> Sum >= 0
3258 //
3259 // Add:
3260 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3261 // Sub:
3262 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3263 //
3264 EVT OType = Node->getValueType(1);
3265 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3266
3267 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3268 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3269 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3270 Node->getOpcode() == ISD::SADDO ?
3271 ISD::SETEQ : ISD::SETNE);
3272
3273 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3274 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3275
3276 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3277
3278 // Use the calculated overflow everywhere.
3279 ReplaceValueWith(SDValue(Node, 1), Cmp);
3280 }
3281
ExpandIntRes_SDIV(SDNode * N,SDValue & Lo,SDValue & Hi)3282 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3283 SDValue &Lo, SDValue &Hi) {
3284 EVT VT = N->getValueType(0);
3285 SDLoc dl(N);
3286 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3287
3288 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3289 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3290 SplitInteger(Res.getValue(0), Lo, Hi);
3291 return;
3292 }
3293
3294 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3295 if (VT == MVT::i16)
3296 LC = RTLIB::SDIV_I16;
3297 else if (VT == MVT::i32)
3298 LC = RTLIB::SDIV_I32;
3299 else if (VT == MVT::i64)
3300 LC = RTLIB::SDIV_I64;
3301 else if (VT == MVT::i128)
3302 LC = RTLIB::SDIV_I128;
3303 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3304
3305 TargetLowering::MakeLibCallOptions CallOptions;
3306 CallOptions.setSExt(true);
3307 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3308 }
3309
ExpandIntRes_Shift(SDNode * N,SDValue & Lo,SDValue & Hi)3310 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3311 SDValue &Lo, SDValue &Hi) {
3312 EVT VT = N->getValueType(0);
3313 SDLoc dl(N);
3314
3315 // If we can emit an efficient shift operation, do so now. Check to see if
3316 // the RHS is a constant.
3317 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3318 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3319
3320 // If we can determine that the high bit of the shift is zero or one, even if
3321 // the low bits are variable, emit this shift in an optimized form.
3322 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3323 return;
3324
3325 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3326 unsigned PartsOpc;
3327 if (N->getOpcode() == ISD::SHL) {
3328 PartsOpc = ISD::SHL_PARTS;
3329 } else if (N->getOpcode() == ISD::SRL) {
3330 PartsOpc = ISD::SRL_PARTS;
3331 } else {
3332 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3333 PartsOpc = ISD::SRA_PARTS;
3334 }
3335
3336 // Next check to see if the target supports this SHL_PARTS operation or if it
3337 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3338 // size, but create a libcall instead.
3339 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3340 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3341 const bool LegalOrCustom =
3342 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3343 Action == TargetLowering::Custom;
3344
3345 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3346 // Expand the subcomponents.
3347 SDValue LHSL, LHSH;
3348 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3349 EVT VT = LHSL.getValueType();
3350
3351 // If the shift amount operand is coming from a vector legalization it may
3352 // have an illegal type. Fix that first by casting the operand, otherwise
3353 // the new SHL_PARTS operation would need further legalization.
3354 SDValue ShiftOp = N->getOperand(1);
3355 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3356 assert(ShiftTy.getScalarSizeInBits() >=
3357 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3358 "ShiftAmountTy is too small to cover the range of this type!");
3359 if (ShiftOp.getValueType() != ShiftTy)
3360 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3361
3362 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3363 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3364 Hi = Lo.getValue(1);
3365 return;
3366 }
3367
3368 // Otherwise, emit a libcall.
3369 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3370 bool isSigned;
3371 if (N->getOpcode() == ISD::SHL) {
3372 isSigned = false; /*sign irrelevant*/
3373 if (VT == MVT::i16)
3374 LC = RTLIB::SHL_I16;
3375 else if (VT == MVT::i32)
3376 LC = RTLIB::SHL_I32;
3377 else if (VT == MVT::i64)
3378 LC = RTLIB::SHL_I64;
3379 else if (VT == MVT::i128)
3380 LC = RTLIB::SHL_I128;
3381 } else if (N->getOpcode() == ISD::SRL) {
3382 isSigned = false;
3383 if (VT == MVT::i16)
3384 LC = RTLIB::SRL_I16;
3385 else if (VT == MVT::i32)
3386 LC = RTLIB::SRL_I32;
3387 else if (VT == MVT::i64)
3388 LC = RTLIB::SRL_I64;
3389 else if (VT == MVT::i128)
3390 LC = RTLIB::SRL_I128;
3391 } else {
3392 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3393 isSigned = true;
3394 if (VT == MVT::i16)
3395 LC = RTLIB::SRA_I16;
3396 else if (VT == MVT::i32)
3397 LC = RTLIB::SRA_I32;
3398 else if (VT == MVT::i64)
3399 LC = RTLIB::SRA_I64;
3400 else if (VT == MVT::i128)
3401 LC = RTLIB::SRA_I128;
3402 }
3403
3404 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3405 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3406 TargetLowering::MakeLibCallOptions CallOptions;
3407 CallOptions.setSExt(isSigned);
3408 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3409 return;
3410 }
3411
3412 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3413 llvm_unreachable("Unsupported shift!");
3414 }
3415
ExpandIntRes_SIGN_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)3416 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3417 SDValue &Lo, SDValue &Hi) {
3418 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3419 SDLoc dl(N);
3420 SDValue Op = N->getOperand(0);
3421 if (Op.getValueType().bitsLE(NVT)) {
3422 // The low part is sign extension of the input (degenerates to a copy).
3423 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3424 // The high part is obtained by SRA'ing all but one of the bits of low part.
3425 unsigned LoSize = NVT.getSizeInBits();
3426 Hi = DAG.getNode(
3427 ISD::SRA, dl, NVT, Lo,
3428 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3429 } else {
3430 // For example, extension of an i48 to an i64. The operand type necessarily
3431 // promotes to the result type, so will end up being expanded too.
3432 assert(getTypeAction(Op.getValueType()) ==
3433 TargetLowering::TypePromoteInteger &&
3434 "Only know how to promote this result!");
3435 SDValue Res = GetPromotedInteger(Op);
3436 assert(Res.getValueType() == N->getValueType(0) &&
3437 "Operand over promoted?");
3438 // Split the promoted operand. This will simplify when it is expanded.
3439 SplitInteger(Res, Lo, Hi);
3440 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3441 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3442 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3443 ExcessBits)));
3444 }
3445 }
3446
3447 void DAGTypeLegalizer::
ExpandIntRes_SIGN_EXTEND_INREG(SDNode * N,SDValue & Lo,SDValue & Hi)3448 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3449 SDLoc dl(N);
3450 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3451 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3452
3453 if (EVT.bitsLE(Lo.getValueType())) {
3454 // sext_inreg the low part if needed.
3455 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3456 N->getOperand(1));
3457
3458 // The high part gets the sign extension from the lo-part. This handles
3459 // things like sextinreg V:i64 from i8.
3460 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3461 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3462 TLI.getPointerTy(DAG.getDataLayout())));
3463 } else {
3464 // For example, extension of an i48 to an i64. Leave the low part alone,
3465 // sext_inreg the high part.
3466 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3467 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3468 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3469 ExcessBits)));
3470 }
3471 }
3472
ExpandIntRes_SREM(SDNode * N,SDValue & Lo,SDValue & Hi)3473 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3474 SDValue &Lo, SDValue &Hi) {
3475 EVT VT = N->getValueType(0);
3476 SDLoc dl(N);
3477 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3478
3479 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3480 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3481 SplitInteger(Res.getValue(1), Lo, Hi);
3482 return;
3483 }
3484
3485 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3486 if (VT == MVT::i16)
3487 LC = RTLIB::SREM_I16;
3488 else if (VT == MVT::i32)
3489 LC = RTLIB::SREM_I32;
3490 else if (VT == MVT::i64)
3491 LC = RTLIB::SREM_I64;
3492 else if (VT == MVT::i128)
3493 LC = RTLIB::SREM_I128;
3494 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3495
3496 TargetLowering::MakeLibCallOptions CallOptions;
3497 CallOptions.setSExt(true);
3498 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3499 }
3500
ExpandIntRes_TRUNCATE(SDNode * N,SDValue & Lo,SDValue & Hi)3501 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3502 SDValue &Lo, SDValue &Hi) {
3503 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3504 SDLoc dl(N);
3505 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3506 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3507 N->getOperand(0),
3508 DAG.getConstant(NVT.getSizeInBits(), dl,
3509 TLI.getPointerTy(DAG.getDataLayout())));
3510 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3511 }
3512
ExpandIntRes_XMULO(SDNode * N,SDValue & Lo,SDValue & Hi)3513 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3514 SDValue &Lo, SDValue &Hi) {
3515 EVT VT = N->getValueType(0);
3516 SDLoc dl(N);
3517
3518 if (N->getOpcode() == ISD::UMULO) {
3519 // This section expands the operation into the following sequence of
3520 // instructions. `iNh` here refers to a type which has half the bit width of
3521 // the type the original operation operated on.
3522 //
3523 // %0 = %LHS.HI != 0 && %RHS.HI != 0
3524 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3525 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3526 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3527 // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3528 // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3529 //
3530 // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3531 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3532 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3533 SplitInteger(LHS, LHSLow, LHSHigh);
3534 SplitInteger(RHS, RHSLow, RHSHigh);
3535 EVT HalfVT = LHSLow.getValueType()
3536 , BitVT = N->getValueType(1);
3537 SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3538 SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3539
3540 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3541 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3542 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3543 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3544
3545 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3546 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3547 SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3548 One.getValue(0));
3549
3550 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3551 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3552 SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3553 Two.getValue(0));
3554
3555 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3556 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3557 // operation recursively legalized?).
3558 //
3559 // Many backends understand this pattern and will convert into LOHI
3560 // themselves, if applicable.
3561 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3562 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3563 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3564 SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3565 SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3566 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3567 SplitInteger(Five, Lo, Hi);
3568 ReplaceValueWith(SDValue(N, 1), Overflow);
3569 return;
3570 }
3571
3572 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3573 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3574 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3575
3576 // Replace this with a libcall that will check overflow.
3577 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3578 if (VT == MVT::i32)
3579 LC = RTLIB::MULO_I32;
3580 else if (VT == MVT::i64)
3581 LC = RTLIB::MULO_I64;
3582 else if (VT == MVT::i128)
3583 LC = RTLIB::MULO_I128;
3584 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3585
3586 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3587 // Temporary for the overflow value, default it to zero.
3588 SDValue Chain =
3589 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3590 MachinePointerInfo());
3591
3592 TargetLowering::ArgListTy Args;
3593 TargetLowering::ArgListEntry Entry;
3594 for (const SDValue &Op : N->op_values()) {
3595 EVT ArgVT = Op.getValueType();
3596 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3597 Entry.Node = Op;
3598 Entry.Ty = ArgTy;
3599 Entry.IsSExt = true;
3600 Entry.IsZExt = false;
3601 Args.push_back(Entry);
3602 }
3603
3604 // Also pass the address of the overflow check.
3605 Entry.Node = Temp;
3606 Entry.Ty = PtrTy->getPointerTo();
3607 Entry.IsSExt = true;
3608 Entry.IsZExt = false;
3609 Args.push_back(Entry);
3610
3611 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3612
3613 TargetLowering::CallLoweringInfo CLI(DAG);
3614 CLI.setDebugLoc(dl)
3615 .setChain(Chain)
3616 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3617 .setSExtResult();
3618
3619 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3620
3621 SplitInteger(CallInfo.first, Lo, Hi);
3622 SDValue Temp2 =
3623 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3624 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3625 DAG.getConstant(0, dl, PtrVT),
3626 ISD::SETNE);
3627 // Use the overflow from the libcall everywhere.
3628 ReplaceValueWith(SDValue(N, 1), Ofl);
3629 }
3630
ExpandIntRes_UDIV(SDNode * N,SDValue & Lo,SDValue & Hi)3631 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3632 SDValue &Lo, SDValue &Hi) {
3633 EVT VT = N->getValueType(0);
3634 SDLoc dl(N);
3635 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3636
3637 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
3638 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3639 SplitInteger(Res.getValue(0), Lo, Hi);
3640 return;
3641 }
3642
3643 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3644 if (VT == MVT::i16)
3645 LC = RTLIB::UDIV_I16;
3646 else if (VT == MVT::i32)
3647 LC = RTLIB::UDIV_I32;
3648 else if (VT == MVT::i64)
3649 LC = RTLIB::UDIV_I64;
3650 else if (VT == MVT::i128)
3651 LC = RTLIB::UDIV_I128;
3652 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3653
3654 TargetLowering::MakeLibCallOptions CallOptions;
3655 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3656 }
3657
ExpandIntRes_UREM(SDNode * N,SDValue & Lo,SDValue & Hi)3658 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3659 SDValue &Lo, SDValue &Hi) {
3660 EVT VT = N->getValueType(0);
3661 SDLoc dl(N);
3662 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3663
3664 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
3665 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3666 SplitInteger(Res.getValue(1), Lo, Hi);
3667 return;
3668 }
3669
3670 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3671 if (VT == MVT::i16)
3672 LC = RTLIB::UREM_I16;
3673 else if (VT == MVT::i32)
3674 LC = RTLIB::UREM_I32;
3675 else if (VT == MVT::i64)
3676 LC = RTLIB::UREM_I64;
3677 else if (VT == MVT::i128)
3678 LC = RTLIB::UREM_I128;
3679 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3680
3681 TargetLowering::MakeLibCallOptions CallOptions;
3682 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3683 }
3684
ExpandIntRes_ZERO_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)3685 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3686 SDValue &Lo, SDValue &Hi) {
3687 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3688 SDLoc dl(N);
3689 SDValue Op = N->getOperand(0);
3690 if (Op.getValueType().bitsLE(NVT)) {
3691 // The low part is zero extension of the input (degenerates to a copy).
3692 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3693 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3694 } else {
3695 // For example, extension of an i48 to an i64. The operand type necessarily
3696 // promotes to the result type, so will end up being expanded too.
3697 assert(getTypeAction(Op.getValueType()) ==
3698 TargetLowering::TypePromoteInteger &&
3699 "Only know how to promote this result!");
3700 SDValue Res = GetPromotedInteger(Op);
3701 assert(Res.getValueType() == N->getValueType(0) &&
3702 "Operand over promoted?");
3703 // Split the promoted operand. This will simplify when it is expanded.
3704 SplitInteger(Res, Lo, Hi);
3705 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3706 Hi = DAG.getZeroExtendInReg(Hi, dl,
3707 EVT::getIntegerVT(*DAG.getContext(),
3708 ExcessBits));
3709 }
3710 }
3711
ExpandIntRes_ATOMIC_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)3712 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3713 SDValue &Lo, SDValue &Hi) {
3714 SDLoc dl(N);
3715 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3716 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3717 SDValue Zero = DAG.getConstant(0, dl, VT);
3718 SDValue Swap = DAG.getAtomicCmpSwap(
3719 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
3720 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3721 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3722
3723 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3724 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3725 }
3726
ExpandIntRes_VECREDUCE(SDNode * N,SDValue & Lo,SDValue & Hi)3727 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
3728 SDValue &Lo, SDValue &Hi) {
3729 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
3730 // both halves independently.
3731 SDValue Res = TLI.expandVecReduce(N, DAG);
3732 SplitInteger(Res, Lo, Hi);
3733 }
3734
3735 //===----------------------------------------------------------------------===//
3736 // Integer Operand Expansion
3737 //===----------------------------------------------------------------------===//
3738
3739 /// ExpandIntegerOperand - This method is called when the specified operand of
3740 /// the specified node is found to need expansion. At this point, all of the
3741 /// result types of the node are known to be legal, but other operands of the
3742 /// node may need promotion or expansion as well as the specified one.
ExpandIntegerOperand(SDNode * N,unsigned OpNo)3743 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3744 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3745 dbgs() << "\n");
3746 SDValue Res = SDValue();
3747
3748 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3749 return false;
3750
3751 switch (N->getOpcode()) {
3752 default:
3753 #ifndef NDEBUG
3754 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3755 N->dump(&DAG); dbgs() << "\n";
3756 #endif
3757 report_fatal_error("Do not know how to expand this operator's operand!");
3758
3759 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3760 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3761 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3762 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3763 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3764 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3765 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3766 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3767 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3768 case ISD::STRICT_SINT_TO_FP:
3769 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3770 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3771 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3772 case ISD::STRICT_UINT_TO_FP:
3773 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3774
3775 case ISD::SHL:
3776 case ISD::SRA:
3777 case ISD::SRL:
3778 case ISD::ROTL:
3779 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3780 case ISD::RETURNADDR:
3781 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3782
3783 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3784 }
3785
3786 // If the result is null, the sub-method took care of registering results etc.
3787 if (!Res.getNode()) return false;
3788
3789 // If the result is N, the sub-method updated N in place. Tell the legalizer
3790 // core about this.
3791 if (Res.getNode() == N)
3792 return true;
3793
3794 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3795 "Invalid operand expansion");
3796
3797 ReplaceValueWith(SDValue(N, 0), Res);
3798 return false;
3799 }
3800
3801 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3802 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
IntegerExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl)3803 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3804 SDValue &NewRHS,
3805 ISD::CondCode &CCCode,
3806 const SDLoc &dl) {
3807 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3808 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3809 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3810
3811 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3812 if (RHSLo == RHSHi) {
3813 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3814 if (RHSCST->isAllOnesValue()) {
3815 // Equality comparison to -1.
3816 NewLHS = DAG.getNode(ISD::AND, dl,
3817 LHSLo.getValueType(), LHSLo, LHSHi);
3818 NewRHS = RHSLo;
3819 return;
3820 }
3821 }
3822 }
3823
3824 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3825 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3826 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3827 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3828 return;
3829 }
3830
3831 // If this is a comparison of the sign bit, just look at the top part.
3832 // X > -1, x < 0
3833 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3834 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3835 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3836 NewLHS = LHSHi;
3837 NewRHS = RHSHi;
3838 return;
3839 }
3840
3841 // FIXME: This generated code sucks.
3842 ISD::CondCode LowCC;
3843 switch (CCCode) {
3844 default: llvm_unreachable("Unknown integer setcc!");
3845 case ISD::SETLT:
3846 case ISD::SETULT: LowCC = ISD::SETULT; break;
3847 case ISD::SETGT:
3848 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3849 case ISD::SETLE:
3850 case ISD::SETULE: LowCC = ISD::SETULE; break;
3851 case ISD::SETGE:
3852 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3853 }
3854
3855 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3856 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3857 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3858
3859 // NOTE: on targets without efficient SELECT of bools, we can always use
3860 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3861 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3862 nullptr);
3863 SDValue LoCmp, HiCmp;
3864 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3865 TLI.isTypeLegal(RHSLo.getValueType()))
3866 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3867 RHSLo, LowCC, false, DagCombineInfo, dl);
3868 if (!LoCmp.getNode())
3869 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3870 RHSLo, LowCC);
3871 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3872 TLI.isTypeLegal(RHSHi.getValueType()))
3873 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3874 RHSHi, CCCode, false, DagCombineInfo, dl);
3875 if (!HiCmp.getNode())
3876 HiCmp =
3877 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3878 LHSHi, RHSHi, DAG.getCondCode(CCCode));
3879
3880 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3881 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3882
3883 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3884 CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3885
3886 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3887 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3888 (LoCmpC && LoCmpC->isNullValue())))) {
3889 // For LE / GE, if high part is known false, ignore the low part.
3890 // For LT / GT: if low part is known false, return the high part.
3891 // if high part is known true, ignore the low part.
3892 NewLHS = HiCmp;
3893 NewRHS = SDValue();
3894 return;
3895 }
3896
3897 if (LHSHi == RHSHi) {
3898 // Comparing the low bits is enough.
3899 NewLHS = LoCmp;
3900 NewRHS = SDValue();
3901 return;
3902 }
3903
3904 // Lower with SETCCCARRY if the target supports it.
3905 EVT HiVT = LHSHi.getValueType();
3906 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3907 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3908
3909 // FIXME: Make all targets support this, then remove the other lowering.
3910 if (HasSETCCCARRY) {
3911 // SETCCCARRY can detect < and >= directly. For > and <=, flip
3912 // operands and condition code.
3913 bool FlipOperands = false;
3914 switch (CCCode) {
3915 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3916 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3917 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3918 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3919 default: break;
3920 }
3921 if (FlipOperands) {
3922 std::swap(LHSLo, RHSLo);
3923 std::swap(LHSHi, RHSHi);
3924 }
3925 // Perform a wide subtraction, feeding the carry from the low part into
3926 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3927 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3928 // zero or positive iff LHS >= RHS.
3929 EVT LoVT = LHSLo.getValueType();
3930 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3931 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3932 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3933 LHSHi, RHSHi, LowCmp.getValue(1),
3934 DAG.getCondCode(CCCode));
3935 NewLHS = Res;
3936 NewRHS = SDValue();
3937 return;
3938 }
3939
3940 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3941 false, DagCombineInfo, dl);
3942 if (!NewLHS.getNode())
3943 NewLHS =
3944 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3945 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3946 NewRHS = SDValue();
3947 }
3948
ExpandIntOp_BR_CC(SDNode * N)3949 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3950 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3951 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3952 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3953
3954 // If ExpandSetCCOperands returned a scalar, we need to compare the result
3955 // against zero to select between true and false values.
3956 if (!NewRHS.getNode()) {
3957 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3958 CCCode = ISD::SETNE;
3959 }
3960
3961 // Update N to have the operands specified.
3962 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3963 DAG.getCondCode(CCCode), NewLHS, NewRHS,
3964 N->getOperand(4)), 0);
3965 }
3966
ExpandIntOp_SELECT_CC(SDNode * N)3967 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3968 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3969 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3970 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3971
3972 // If ExpandSetCCOperands returned a scalar, we need to compare the result
3973 // against zero to select between true and false values.
3974 if (!NewRHS.getNode()) {
3975 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3976 CCCode = ISD::SETNE;
3977 }
3978
3979 // Update N to have the operands specified.
3980 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3981 N->getOperand(2), N->getOperand(3),
3982 DAG.getCondCode(CCCode)), 0);
3983 }
3984
ExpandIntOp_SETCC(SDNode * N)3985 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3986 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3987 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3988 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3989
3990 // If ExpandSetCCOperands returned a scalar, use it.
3991 if (!NewRHS.getNode()) {
3992 assert(NewLHS.getValueType() == N->getValueType(0) &&
3993 "Unexpected setcc expansion!");
3994 return NewLHS;
3995 }
3996
3997 // Otherwise, update N to have the operands specified.
3998 return SDValue(
3999 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
4000 }
4001
ExpandIntOp_SETCCCARRY(SDNode * N)4002 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
4003 SDValue LHS = N->getOperand(0);
4004 SDValue RHS = N->getOperand(1);
4005 SDValue Carry = N->getOperand(2);
4006 SDValue Cond = N->getOperand(3);
4007 SDLoc dl = SDLoc(N);
4008
4009 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4010 GetExpandedInteger(LHS, LHSLo, LHSHi);
4011 GetExpandedInteger(RHS, RHSLo, RHSHi);
4012
4013 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
4014 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
4015 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
4016 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
4017 LowCmp.getValue(1), Cond);
4018 }
4019
ExpandIntOp_Shift(SDNode * N)4020 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
4021 // The value being shifted is legal, but the shift amount is too big.
4022 // It follows that either the result of the shift is undefined, or the
4023 // upper half of the shift amount is zero. Just use the lower half.
4024 SDValue Lo, Hi;
4025 GetExpandedInteger(N->getOperand(1), Lo, Hi);
4026 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
4027 }
4028
ExpandIntOp_RETURNADDR(SDNode * N)4029 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
4030 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
4031 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
4032 // constant to valid type.
4033 SDValue Lo, Hi;
4034 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4035 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
4036 }
4037
ExpandIntOp_SINT_TO_FP(SDNode * N)4038 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
4039 bool IsStrict = N->isStrictFPOpcode();
4040 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4041 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4042 EVT DstVT = N->getValueType(0);
4043 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
4044 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4045 "Don't know how to expand this SINT_TO_FP!");
4046 TargetLowering::MakeLibCallOptions CallOptions;
4047 CallOptions.setSExt(true);
4048 std::pair<SDValue, SDValue> Tmp =
4049 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4050
4051 if (!IsStrict)
4052 return Tmp.first;
4053
4054 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4055 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4056 return SDValue();
4057 }
4058
ExpandIntOp_STORE(StoreSDNode * N,unsigned OpNo)4059 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
4060 if (N->isAtomic()) {
4061 // It's typical to have larger CAS than atomic store instructions.
4062 SDLoc dl(N);
4063 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4064 N->getMemoryVT(),
4065 N->getOperand(0), N->getOperand(2),
4066 N->getOperand(1),
4067 N->getMemOperand());
4068 return Swap.getValue(1);
4069 }
4070 if (ISD::isNormalStore(N))
4071 return ExpandOp_NormalStore(N, OpNo);
4072
4073 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
4074 assert(OpNo == 1 && "Can only expand the stored value so far");
4075
4076 EVT VT = N->getOperand(1).getValueType();
4077 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4078 SDValue Ch = N->getChain();
4079 SDValue Ptr = N->getBasePtr();
4080 unsigned Alignment = N->getAlignment();
4081 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4082 AAMDNodes AAInfo = N->getAAInfo();
4083 SDLoc dl(N);
4084 SDValue Lo, Hi;
4085
4086 assert(NVT.isByteSized() && "Expanded type not byte sized!");
4087
4088 if (N->getMemoryVT().bitsLE(NVT)) {
4089 GetExpandedInteger(N->getValue(), Lo, Hi);
4090 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4091 N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
4092 }
4093
4094 if (DAG.getDataLayout().isLittleEndian()) {
4095 // Little-endian - low bits are at low addresses.
4096 GetExpandedInteger(N->getValue(), Lo, Hi);
4097
4098 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
4099 AAInfo);
4100
4101 unsigned ExcessBits =
4102 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4103 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
4104
4105 // Increment the pointer to the other half.
4106 unsigned IncrementSize = NVT.getSizeInBits()/8;
4107 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4108 Hi = DAG.getTruncStore(
4109 Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
4110 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
4111 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4112 }
4113
4114 // Big-endian - high bits are at low addresses. Favor aligned stores at
4115 // the cost of some bit-fiddling.
4116 GetExpandedInteger(N->getValue(), Lo, Hi);
4117
4118 EVT ExtVT = N->getMemoryVT();
4119 unsigned EBytes = ExtVT.getStoreSize();
4120 unsigned IncrementSize = NVT.getSizeInBits()/8;
4121 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4122 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
4123 ExtVT.getSizeInBits() - ExcessBits);
4124
4125 if (ExcessBits < NVT.getSizeInBits()) {
4126 // Transfer high bits from the top of Lo to the bottom of Hi.
4127 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
4128 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
4129 TLI.getPointerTy(DAG.getDataLayout())));
4130 Hi = DAG.getNode(
4131 ISD::OR, dl, NVT, Hi,
4132 DAG.getNode(ISD::SRL, dl, NVT, Lo,
4133 DAG.getConstant(ExcessBits, dl,
4134 TLI.getPointerTy(DAG.getDataLayout()))));
4135 }
4136
4137 // Store both the high bits and maybe some of the low bits.
4138 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
4139 MMOFlags, AAInfo);
4140
4141 // Increment the pointer to the other half.
4142 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4143 // Store the lowest ExcessBits bits in the second half.
4144 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
4145 N->getPointerInfo().getWithOffset(IncrementSize),
4146 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
4147 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
4148 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4149 }
4150
ExpandIntOp_TRUNCATE(SDNode * N)4151 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
4152 SDValue InL, InH;
4153 GetExpandedInteger(N->getOperand(0), InL, InH);
4154 // Just truncate the low part of the source.
4155 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
4156 }
4157
ExpandIntOp_UINT_TO_FP(SDNode * N)4158 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
4159 bool IsStrict = N->isStrictFPOpcode();
4160 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4161 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4162 EVT DstVT = N->getValueType(0);
4163 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
4164 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4165 "Don't know how to expand this UINT_TO_FP!");
4166 TargetLowering::MakeLibCallOptions CallOptions;
4167 CallOptions.setSExt(true);
4168 std::pair<SDValue, SDValue> Tmp =
4169 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4170
4171 if (!IsStrict)
4172 return Tmp.first;
4173
4174 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4175 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4176 return SDValue();
4177 }
4178
ExpandIntOp_ATOMIC_STORE(SDNode * N)4179 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
4180 SDLoc dl(N);
4181 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4182 cast<AtomicSDNode>(N)->getMemoryVT(),
4183 N->getOperand(0),
4184 N->getOperand(1), N->getOperand(2),
4185 cast<AtomicSDNode>(N)->getMemOperand());
4186 return Swap.getValue(1);
4187 }
4188
4189
PromoteIntRes_EXTRACT_SUBVECTOR(SDNode * N)4190 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4191
4192 EVT OutVT = N->getValueType(0);
4193 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4194 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4195 unsigned OutNumElems = OutVT.getVectorNumElements();
4196 EVT NOutVTElem = NOutVT.getVectorElementType();
4197
4198 SDLoc dl(N);
4199 SDValue BaseIdx = N->getOperand(1);
4200
4201 SDValue InOp0 = N->getOperand(0);
4202 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
4203 InOp0 = GetPromotedInteger(N->getOperand(0));
4204
4205 EVT InVT = InOp0.getValueType();
4206
4207 SmallVector<SDValue, 8> Ops;
4208 Ops.reserve(OutNumElems);
4209 for (unsigned i = 0; i != OutNumElems; ++i) {
4210
4211 // Extract the element from the original vector.
4212 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4213 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4214 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4215 InVT.getVectorElementType(), N->getOperand(0), Index);
4216
4217 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4218 // Insert the converted element to the new vector.
4219 Ops.push_back(Op);
4220 }
4221
4222 return DAG.getBuildVector(NOutVT, dl, Ops);
4223 }
4224
4225
PromoteIntRes_VECTOR_SHUFFLE(SDNode * N)4226 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4227 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4228 EVT VT = N->getValueType(0);
4229 SDLoc dl(N);
4230
4231 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4232
4233 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4234 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4235 EVT OutVT = V0.getValueType();
4236
4237 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4238 }
4239
4240
PromoteIntRes_BUILD_VECTOR(SDNode * N)4241 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4242 EVT OutVT = N->getValueType(0);
4243 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4244 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4245 unsigned NumElems = N->getNumOperands();
4246 EVT NOutVTElem = NOutVT.getVectorElementType();
4247
4248 SDLoc dl(N);
4249
4250 SmallVector<SDValue, 8> Ops;
4251 Ops.reserve(NumElems);
4252 for (unsigned i = 0; i != NumElems; ++i) {
4253 SDValue Op;
4254 // BUILD_VECTOR integer operand types are allowed to be larger than the
4255 // result's element type. This may still be true after the promotion. For
4256 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4257 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4258 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4259 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4260 else
4261 Op = N->getOperand(i);
4262 Ops.push_back(Op);
4263 }
4264
4265 return DAG.getBuildVector(NOutVT, dl, Ops);
4266 }
4267
PromoteIntRes_SCALAR_TO_VECTOR(SDNode * N)4268 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4269
4270 SDLoc dl(N);
4271
4272 assert(!N->getOperand(0).getValueType().isVector() &&
4273 "Input must be a scalar");
4274
4275 EVT OutVT = N->getValueType(0);
4276 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4277 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4278 EVT NOutVTElem = NOutVT.getVectorElementType();
4279
4280 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4281
4282 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4283 }
4284
PromoteIntRes_SPLAT_VECTOR(SDNode * N)4285 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
4286 SDLoc dl(N);
4287
4288 SDValue SplatVal = N->getOperand(0);
4289
4290 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar");
4291
4292 EVT OutVT = N->getValueType(0);
4293 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4294 assert(NOutVT.isVector() && "Type must be promoted to a vector type");
4295 EVT NOutElemVT = NOutVT.getVectorElementType();
4296
4297 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
4298
4299 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
4300 }
4301
PromoteIntRes_CONCAT_VECTORS(SDNode * N)4302 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4303 SDLoc dl(N);
4304
4305 EVT OutVT = N->getValueType(0);
4306 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4307 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4308
4309 EVT OutElemTy = NOutVT.getVectorElementType();
4310
4311 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4312 unsigned NumOutElem = NOutVT.getVectorNumElements();
4313 unsigned NumOperands = N->getNumOperands();
4314 assert(NumElem * NumOperands == NumOutElem &&
4315 "Unexpected number of elements");
4316
4317 // Take the elements from the first vector.
4318 SmallVector<SDValue, 8> Ops(NumOutElem);
4319 for (unsigned i = 0; i < NumOperands; ++i) {
4320 SDValue Op = N->getOperand(i);
4321 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4322 Op = GetPromotedInteger(Op);
4323 EVT SclrTy = Op.getValueType().getVectorElementType();
4324 assert(NumElem == Op.getValueType().getVectorNumElements() &&
4325 "Unexpected number of elements");
4326
4327 for (unsigned j = 0; j < NumElem; ++j) {
4328 SDValue Ext = DAG.getNode(
4329 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4330 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4331 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4332 }
4333 }
4334
4335 return DAG.getBuildVector(NOutVT, dl, Ops);
4336 }
4337
PromoteIntRes_EXTEND_VECTOR_INREG(SDNode * N)4338 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4339 EVT VT = N->getValueType(0);
4340 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4341 assert(NVT.isVector() && "This type must be promoted to a vector type");
4342
4343 SDLoc dl(N);
4344
4345 // For operands whose TypeAction is to promote, extend the promoted node
4346 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4347 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4348 // type..
4349 if (getTypeAction(N->getOperand(0).getValueType())
4350 == TargetLowering::TypePromoteInteger) {
4351 SDValue Promoted;
4352
4353 switch(N->getOpcode()) {
4354 case ISD::SIGN_EXTEND_VECTOR_INREG:
4355 Promoted = SExtPromotedInteger(N->getOperand(0));
4356 break;
4357 case ISD::ZERO_EXTEND_VECTOR_INREG:
4358 Promoted = ZExtPromotedInteger(N->getOperand(0));
4359 break;
4360 case ISD::ANY_EXTEND_VECTOR_INREG:
4361 Promoted = GetPromotedInteger(N->getOperand(0));
4362 break;
4363 default:
4364 llvm_unreachable("Node has unexpected Opcode");
4365 }
4366 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4367 }
4368
4369 // Directly extend to the appropriate transform-to type.
4370 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4371 }
4372
PromoteIntRes_INSERT_VECTOR_ELT(SDNode * N)4373 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4374 EVT OutVT = N->getValueType(0);
4375 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4376 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4377
4378 EVT NOutVTElem = NOutVT.getVectorElementType();
4379
4380 SDLoc dl(N);
4381 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4382
4383 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4384 NOutVTElem, N->getOperand(1));
4385 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4386 V0, ConvElem, N->getOperand(2));
4387 }
4388
PromoteIntRes_VECREDUCE(SDNode * N)4389 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4390 // The VECREDUCE result size may be larger than the element size, so
4391 // we can simply change the result type.
4392 SDLoc dl(N);
4393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4394 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4395 }
4396
PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode * N)4397 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4398 SDLoc dl(N);
4399 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4400 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4401 TLI.getVectorIdxTy(DAG.getDataLayout()));
4402 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4403 V0->getValueType(0).getScalarType(), V0, V1);
4404
4405 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4406 // element types. If this is the case then we need to expand the outgoing
4407 // value and not truncate it.
4408 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4409 }
4410
PromoteIntOp_EXTRACT_SUBVECTOR(SDNode * N)4411 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4412 SDLoc dl(N);
4413 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4414 MVT InVT = V0.getValueType().getSimpleVT();
4415 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4416 N->getValueType(0).getVectorNumElements());
4417 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4418 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4419 }
4420
PromoteIntOp_CONCAT_VECTORS(SDNode * N)4421 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4422 SDLoc dl(N);
4423 unsigned NumElems = N->getNumOperands();
4424
4425 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4426
4427 SmallVector<SDValue, 8> NewOps;
4428 NewOps.reserve(NumElems);
4429
4430 // For each incoming vector
4431 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
4432 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4433 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4434 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4435
4436 for (unsigned i=0; i<NumElem; ++i) {
4437 // Extract element from incoming vector
4438 SDValue Ex = DAG.getNode(
4439 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4440 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4441 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4442 NewOps.push_back(Tr);
4443 }
4444 }
4445
4446 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4447 }
4448