1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include "llvm/Support/KnownBits.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
PromoteIntegerResult(SDNode * N,unsigned ResNo)37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39 dbgs() << "\n");
40 SDValue Res = SDValue();
41
42 // See if the target wants to custom expand this node.
43 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45 return;
46 }
47
48 switch (N->getOpcode()) {
49 default:
50 #ifndef NDEBUG
51 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52 N->dump(&DAG); dbgs() << "\n";
53 #endif
54 llvm_unreachable("Do not know how to promote this operator!");
55 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
63 case ISD::CTLZ_ZERO_UNDEF:
64 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65 case ISD::PARITY:
66 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
67 case ISD::CTTZ_ZERO_UNDEF:
68 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69 case ISD::EXTRACT_VECTOR_ELT:
70 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
71 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
72 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
73 break;
74 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
75 break;
76 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
77 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
78 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
79 case ISD::STRICT_FSETCC:
80 case ISD::STRICT_FSETCCS:
81 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
82 case ISD::SMIN:
83 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
84 case ISD::UMIN:
85 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
86
87 case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
88 case ISD::SIGN_EXTEND_INREG:
89 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
90 case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
91 case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
92 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
93 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
94 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
95 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
96
97 case ISD::EXTRACT_SUBVECTOR:
98 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
99 case ISD::VECTOR_SHUFFLE:
100 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
101 case ISD::INSERT_VECTOR_ELT:
102 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
103 case ISD::BUILD_VECTOR:
104 Res = PromoteIntRes_BUILD_VECTOR(N); break;
105 case ISD::SCALAR_TO_VECTOR:
106 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
107 case ISD::SPLAT_VECTOR:
108 Res = PromoteIntRes_SPLAT_VECTOR(N); break;
109 case ISD::CONCAT_VECTORS:
110 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
111
112 case ISD::ANY_EXTEND_VECTOR_INREG:
113 case ISD::SIGN_EXTEND_VECTOR_INREG:
114 case ISD::ZERO_EXTEND_VECTOR_INREG:
115 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
116
117 case ISD::SIGN_EXTEND:
118 case ISD::ZERO_EXTEND:
119 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
120
121 case ISD::STRICT_FP_TO_SINT:
122 case ISD::STRICT_FP_TO_UINT:
123 case ISD::FP_TO_SINT:
124 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
125
126 case ISD::FP_TO_SINT_SAT:
127 case ISD::FP_TO_UINT_SAT:
128 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
129
130 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
131
132 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
133
134 case ISD::AND:
135 case ISD::OR:
136 case ISD::XOR:
137 case ISD::ADD:
138 case ISD::SUB:
139 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
140
141 case ISD::SDIV:
142 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
143
144 case ISD::UDIV:
145 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
146
147 case ISD::SADDO:
148 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
149 case ISD::UADDO:
150 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
151 case ISD::SMULO:
152 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
153
154 case ISD::ADDE:
155 case ISD::SUBE:
156 case ISD::ADDCARRY:
157 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
158
159 case ISD::SADDO_CARRY:
160 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
161
162 case ISD::SADDSAT:
163 case ISD::UADDSAT:
164 case ISD::SSUBSAT:
165 case ISD::USUBSAT:
166 case ISD::SSHLSAT:
167 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break;
168
169 case ISD::SMULFIX:
170 case ISD::SMULFIXSAT:
171 case ISD::UMULFIX:
172 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
173
174 case ISD::SDIVFIX:
175 case ISD::SDIVFIXSAT:
176 case ISD::UDIVFIX:
177 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
178
179 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
180
181 case ISD::ATOMIC_LOAD:
182 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
183
184 case ISD::ATOMIC_LOAD_ADD:
185 case ISD::ATOMIC_LOAD_SUB:
186 case ISD::ATOMIC_LOAD_AND:
187 case ISD::ATOMIC_LOAD_CLR:
188 case ISD::ATOMIC_LOAD_OR:
189 case ISD::ATOMIC_LOAD_XOR:
190 case ISD::ATOMIC_LOAD_NAND:
191 case ISD::ATOMIC_LOAD_MIN:
192 case ISD::ATOMIC_LOAD_MAX:
193 case ISD::ATOMIC_LOAD_UMIN:
194 case ISD::ATOMIC_LOAD_UMAX:
195 case ISD::ATOMIC_SWAP:
196 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
197
198 case ISD::ATOMIC_CMP_SWAP:
199 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
200 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
201 break;
202
203 case ISD::VECREDUCE_ADD:
204 case ISD::VECREDUCE_MUL:
205 case ISD::VECREDUCE_AND:
206 case ISD::VECREDUCE_OR:
207 case ISD::VECREDUCE_XOR:
208 case ISD::VECREDUCE_SMAX:
209 case ISD::VECREDUCE_SMIN:
210 case ISD::VECREDUCE_UMAX:
211 case ISD::VECREDUCE_UMIN:
212 Res = PromoteIntRes_VECREDUCE(N);
213 break;
214
215 case ISD::FREEZE:
216 Res = PromoteIntRes_FREEZE(N);
217 break;
218
219 case ISD::ROTL:
220 case ISD::ROTR:
221 Res = PromoteIntRes_Rotate(N);
222 break;
223
224 case ISD::FSHL:
225 case ISD::FSHR:
226 Res = PromoteIntRes_FunnelShift(N);
227 break;
228 }
229
230 // If the result is null then the sub-method took care of registering it.
231 if (Res.getNode())
232 SetPromotedInteger(SDValue(N, ResNo), Res);
233 }
234
PromoteIntRes_MERGE_VALUES(SDNode * N,unsigned ResNo)235 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
236 unsigned ResNo) {
237 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
238 return GetPromotedInteger(Op);
239 }
240
PromoteIntRes_AssertSext(SDNode * N)241 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
242 // Sign-extend the new bits, and continue the assertion.
243 SDValue Op = SExtPromotedInteger(N->getOperand(0));
244 return DAG.getNode(ISD::AssertSext, SDLoc(N),
245 Op.getValueType(), Op, N->getOperand(1));
246 }
247
PromoteIntRes_AssertZext(SDNode * N)248 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
249 // Zero the new bits, and continue the assertion.
250 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
251 return DAG.getNode(ISD::AssertZext, SDLoc(N),
252 Op.getValueType(), Op, N->getOperand(1));
253 }
254
PromoteIntRes_Atomic0(AtomicSDNode * N)255 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
256 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
257 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
258 N->getMemoryVT(), ResVT,
259 N->getChain(), N->getBasePtr(),
260 N->getMemOperand());
261 // Legalize the chain result - switch anything that used the old chain to
262 // use the new one.
263 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
264 return Res;
265 }
266
PromoteIntRes_Atomic1(AtomicSDNode * N)267 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
268 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
269 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
270 N->getMemoryVT(),
271 N->getChain(), N->getBasePtr(),
272 Op2, N->getMemOperand());
273 // Legalize the chain result - switch anything that used the old chain to
274 // use the new one.
275 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
276 return Res;
277 }
278
PromoteIntRes_AtomicCmpSwap(AtomicSDNode * N,unsigned ResNo)279 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
280 unsigned ResNo) {
281 if (ResNo == 1) {
282 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
283 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
284 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
285
286 // Only use the result of getSetCCResultType if it is legal,
287 // otherwise just use the promoted result type (NVT).
288 if (!TLI.isTypeLegal(SVT))
289 SVT = NVT;
290
291 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
292 SDValue Res = DAG.getAtomicCmpSwap(
293 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
294 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
295 N->getMemOperand());
296 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
297 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
298 return Res.getValue(1);
299 }
300
301 // Op2 is used for the comparison and thus must be extended according to the
302 // target's atomic operations. Op3 is merely stored and so can be left alone.
303 SDValue Op2 = N->getOperand(2);
304 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
305 switch (TLI.getExtendForAtomicCmpSwapArg()) {
306 case ISD::SIGN_EXTEND:
307 Op2 = SExtPromotedInteger(Op2);
308 break;
309 case ISD::ZERO_EXTEND:
310 Op2 = ZExtPromotedInteger(Op2);
311 break;
312 case ISD::ANY_EXTEND:
313 Op2 = GetPromotedInteger(Op2);
314 break;
315 default:
316 llvm_unreachable("Invalid atomic op extension");
317 }
318
319 SDVTList VTs =
320 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
321 SDValue Res = DAG.getAtomicCmpSwap(
322 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
323 N->getBasePtr(), Op2, Op3, N->getMemOperand());
324 // Update the use to N with the newly created Res.
325 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
326 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
327 return Res;
328 }
329
PromoteIntRes_BITCAST(SDNode * N)330 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
331 SDValue InOp = N->getOperand(0);
332 EVT InVT = InOp.getValueType();
333 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
334 EVT OutVT = N->getValueType(0);
335 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
336 SDLoc dl(N);
337
338 switch (getTypeAction(InVT)) {
339 case TargetLowering::TypeLegal:
340 break;
341 case TargetLowering::TypePromoteInteger:
342 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
343 // The input promotes to the same size. Convert the promoted value.
344 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
345 break;
346 case TargetLowering::TypeSoftenFloat:
347 // Promote the integer operand by hand.
348 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
349 case TargetLowering::TypeSoftPromoteHalf:
350 // Promote the integer operand by hand.
351 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
352 case TargetLowering::TypePromoteFloat: {
353 // Convert the promoted float by hand.
354 if (!NOutVT.isVector())
355 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
356 break;
357 }
358 case TargetLowering::TypeExpandInteger:
359 case TargetLowering::TypeExpandFloat:
360 break;
361 case TargetLowering::TypeScalarizeVector:
362 // Convert the element to an integer and promote it by hand.
363 if (!NOutVT.isVector())
364 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
365 BitConvertToInteger(GetScalarizedVector(InOp)));
366 break;
367 case TargetLowering::TypeScalarizeScalableVector:
368 report_fatal_error("Scalarization of scalable vectors is not supported.");
369 case TargetLowering::TypeSplitVector: {
370 if (!NOutVT.isVector()) {
371 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
372 // pieces of the input into integers and reassemble in the final type.
373 SDValue Lo, Hi;
374 GetSplitVector(N->getOperand(0), Lo, Hi);
375 Lo = BitConvertToInteger(Lo);
376 Hi = BitConvertToInteger(Hi);
377
378 if (DAG.getDataLayout().isBigEndian())
379 std::swap(Lo, Hi);
380
381 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
382 EVT::getIntegerVT(*DAG.getContext(),
383 NOutVT.getSizeInBits()),
384 JoinIntegers(Lo, Hi));
385 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
386 }
387 break;
388 }
389 case TargetLowering::TypeWidenVector:
390 // The input is widened to the same size. Convert to the widened value.
391 // Make sure that the outgoing value is not a vector, because this would
392 // make us bitcast between two vectors which are legalized in different ways.
393 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
394 SDValue Res =
395 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
396
397 // For big endian targets we need to shift the casted value or the
398 // interesting bits will end up at the wrong place.
399 if (DAG.getDataLayout().isBigEndian()) {
400 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
401 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout());
402 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
403 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
404 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
405 }
406 return Res;
407 }
408 // If the output type is also a vector and widening it to the same size
409 // as the widened input type would be a legal type, we can widen the bitcast
410 // and handle the promotion after.
411 if (NOutVT.isVector()) {
412 unsigned WidenInSize = NInVT.getSizeInBits();
413 unsigned OutSize = OutVT.getSizeInBits();
414 if (WidenInSize % OutSize == 0) {
415 unsigned Scale = WidenInSize / OutSize;
416 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
417 OutVT.getVectorElementType(),
418 OutVT.getVectorNumElements() * Scale);
419 if (isTypeLegal(WideOutVT)) {
420 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
421 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
422 DAG.getVectorIdxConstant(0, dl));
423 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
424 }
425 }
426 }
427 }
428
429 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
430 CreateStackStoreLoad(InOp, OutVT));
431 }
432
433 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount
434 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
getShiftAmountTyForConstant(EVT VT,const TargetLowering & TLI,SelectionDAG & DAG)435 static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI,
436 SelectionDAG &DAG) {
437 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
438 // If any possible shift value won't fit in the prefered type, just use
439 // something safe. It will be legalized when the shift is expanded.
440 if (!ShiftVT.isVector() &&
441 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits()))
442 ShiftVT = MVT::i32;
443 return ShiftVT;
444 }
445
PromoteIntRes_FREEZE(SDNode * N)446 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
447 SDValue V = GetPromotedInteger(N->getOperand(0));
448 return DAG.getNode(ISD::FREEZE, SDLoc(N),
449 V.getValueType(), V);
450 }
451
PromoteIntRes_BSWAP(SDNode * N)452 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
453 SDValue Op = GetPromotedInteger(N->getOperand(0));
454 EVT OVT = N->getValueType(0);
455 EVT NVT = Op.getValueType();
456 SDLoc dl(N);
457
458 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
459 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
460 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
461 DAG.getConstant(DiffBits, dl, ShiftVT));
462 }
463
PromoteIntRes_BITREVERSE(SDNode * N)464 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
465 SDValue Op = GetPromotedInteger(N->getOperand(0));
466 EVT OVT = N->getValueType(0);
467 EVT NVT = Op.getValueType();
468 SDLoc dl(N);
469
470 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
471 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
472 return DAG.getNode(ISD::SRL, dl, NVT,
473 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
474 DAG.getConstant(DiffBits, dl, ShiftVT));
475 }
476
PromoteIntRes_BUILD_PAIR(SDNode * N)477 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
478 // The pair element type may be legal, or may not promote to the same type as
479 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
480 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
481 TLI.getTypeToTransformTo(*DAG.getContext(),
482 N->getValueType(0)), JoinIntegers(N->getOperand(0),
483 N->getOperand(1)));
484 }
485
PromoteIntRes_Constant(SDNode * N)486 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
487 EVT VT = N->getValueType(0);
488 // FIXME there is no actual debug info here
489 SDLoc dl(N);
490 // Zero extend things like i1, sign extend everything else. It shouldn't
491 // matter in theory which one we pick, but this tends to give better code?
492 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
493 SDValue Result = DAG.getNode(Opc, dl,
494 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
495 SDValue(N, 0));
496 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
497 return Result;
498 }
499
PromoteIntRes_CTLZ(SDNode * N)500 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
501 // Zero extend to the promoted type and do the count there.
502 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
503 SDLoc dl(N);
504 EVT OVT = N->getValueType(0);
505 EVT NVT = Op.getValueType();
506 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
507 // Subtract off the extra leading bits in the bigger type.
508 return DAG.getNode(
509 ISD::SUB, dl, NVT, Op,
510 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
511 NVT));
512 }
513
PromoteIntRes_CTPOP_PARITY(SDNode * N)514 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
515 // Zero extend to the promoted type and do the count or parity there.
516 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
517 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
518 }
519
PromoteIntRes_CTTZ(SDNode * N)520 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
521 SDValue Op = GetPromotedInteger(N->getOperand(0));
522 EVT OVT = N->getValueType(0);
523 EVT NVT = Op.getValueType();
524 SDLoc dl(N);
525 if (N->getOpcode() == ISD::CTTZ) {
526 // The count is the same in the promoted type except if the original
527 // value was zero. This can be handled by setting the bit just off
528 // the top of the original type.
529 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
530 OVT.getScalarSizeInBits());
531 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
532 }
533 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
534 }
535
PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode * N)536 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
537 SDLoc dl(N);
538 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
539
540 SDValue Op0 = N->getOperand(0);
541 SDValue Op1 = N->getOperand(1);
542
543 // If the input also needs to be promoted, do that first so we can get a
544 // get a good idea for the output type.
545 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
546 == TargetLowering::TypePromoteInteger) {
547 SDValue In = GetPromotedInteger(Op0);
548
549 // If the new type is larger than NVT, use it. We probably won't need to
550 // promote it again.
551 EVT SVT = In.getValueType().getScalarType();
552 if (SVT.bitsGE(NVT)) {
553 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
554 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
555 }
556 }
557
558 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
559 }
560
PromoteIntRes_FP_TO_XINT(SDNode * N)561 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
562 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
563 unsigned NewOpc = N->getOpcode();
564 SDLoc dl(N);
565
566 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
567 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
568 // and SINT conversions are Custom, there is no way to tell which is
569 // preferable. We choose SINT because that's the right thing on PPC.)
570 if (N->getOpcode() == ISD::FP_TO_UINT &&
571 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
572 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
573 NewOpc = ISD::FP_TO_SINT;
574
575 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
576 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
577 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
578 NewOpc = ISD::STRICT_FP_TO_SINT;
579
580 SDValue Res;
581 if (N->isStrictFPOpcode()) {
582 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
583 {N->getOperand(0), N->getOperand(1)});
584 // Legalize the chain result - switch anything that used the old chain to
585 // use the new one.
586 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
587 } else
588 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
589
590 // Assert that the converted value fits in the original type. If it doesn't
591 // (eg: because the value being converted is too big), then the result of the
592 // original operation was undefined anyway, so the assert is still correct.
593 //
594 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
595 // before legalization: fp-to-uint16, 65534. -> 0xfffe
596 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
597 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
598 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ?
599 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
600 DAG.getValueType(N->getValueType(0).getScalarType()));
601 }
602
PromoteIntRes_FP_TO_XINT_SAT(SDNode * N)603 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
604 // Promote the result type, while keeping the original width in Op1.
605 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
606 SDLoc dl(N);
607 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
608 N->getOperand(1));
609 }
610
PromoteIntRes_FP_TO_FP16(SDNode * N)611 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
612 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
613 SDLoc dl(N);
614
615 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
616 }
617
PromoteIntRes_FLT_ROUNDS(SDNode * N)618 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
619 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
620 SDLoc dl(N);
621
622 SDValue Res =
623 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
624
625 // Legalize the chain result - switch anything that used the old chain to
626 // use the new one.
627 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
628 return Res;
629 }
630
PromoteIntRes_INT_EXTEND(SDNode * N)631 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
632 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
633 SDLoc dl(N);
634
635 if (getTypeAction(N->getOperand(0).getValueType())
636 == TargetLowering::TypePromoteInteger) {
637 SDValue Res = GetPromotedInteger(N->getOperand(0));
638 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
639
640 // If the result and operand types are the same after promotion, simplify
641 // to an in-register extension.
642 if (NVT == Res.getValueType()) {
643 // The high bits are not guaranteed to be anything. Insert an extend.
644 if (N->getOpcode() == ISD::SIGN_EXTEND)
645 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
646 DAG.getValueType(N->getOperand(0).getValueType()));
647 if (N->getOpcode() == ISD::ZERO_EXTEND)
648 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
649 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
650 return Res;
651 }
652 }
653
654 // Otherwise, just extend the original operand all the way to the larger type.
655 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
656 }
657
PromoteIntRes_LOAD(LoadSDNode * N)658 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
659 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
661 ISD::LoadExtType ExtType =
662 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
663 SDLoc dl(N);
664 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
665 N->getMemoryVT(), N->getMemOperand());
666
667 // Legalize the chain result - switch anything that used the old chain to
668 // use the new one.
669 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
670 return Res;
671 }
672
PromoteIntRes_MLOAD(MaskedLoadSDNode * N)673 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
674 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
675 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
676
677 SDLoc dl(N);
678 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
679 N->getOffset(), N->getMask(), ExtPassThru,
680 N->getMemoryVT(), N->getMemOperand(),
681 N->getAddressingMode(), ISD::EXTLOAD);
682 // Legalize the chain result - switch anything that used the old chain to
683 // use the new one.
684 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
685 return Res;
686 }
687
PromoteIntRes_MGATHER(MaskedGatherSDNode * N)688 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
689 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
690 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
691 assert(NVT == ExtPassThru.getValueType() &&
692 "Gather result type and the passThru argument type should be the same");
693
694 ISD::LoadExtType ExtType = N->getExtensionType();
695 if (ExtType == ISD::NON_EXTLOAD)
696 ExtType = ISD::EXTLOAD;
697
698 SDLoc dl(N);
699 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
700 N->getIndex(), N->getScale() };
701 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
702 N->getMemoryVT(), dl, Ops,
703 N->getMemOperand(), N->getIndexType(),
704 ExtType);
705 // Legalize the chain result - switch anything that used the old chain to
706 // use the new one.
707 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
708 return Res;
709 }
710
711 /// Promote the overflow flag of an overflowing arithmetic node.
PromoteIntRes_Overflow(SDNode * N)712 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
713 // Change the return type of the boolean result while obeying
714 // getSetCCResultType.
715 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
716 EVT VT = N->getValueType(0);
717 EVT SVT = getSetCCResultType(VT);
718 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
719 unsigned NumOps = N->getNumOperands();
720 assert(NumOps <= 3 && "Too many operands");
721 if (NumOps == 3)
722 Ops[2] = N->getOperand(2);
723
724 SDLoc dl(N);
725 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
726 makeArrayRef(Ops, NumOps));
727
728 // Modified the sum result - switch anything that used the old sum to use
729 // the new one.
730 ReplaceValueWith(SDValue(N, 0), Res);
731
732 // Convert to the expected type.
733 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
734 }
735
PromoteIntRes_ADDSUBSHLSAT(SDNode * N)736 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
737 // If the promoted type is legal, we can convert this to:
738 // 1. ANY_EXTEND iN to iM
739 // 2. SHL by M-N
740 // 3. [US][ADD|SUB|SHL]SAT
741 // 4. L/ASHR by M-N
742 // Else it is more efficient to convert this to a min and a max
743 // operation in the higher precision arithmetic.
744 SDLoc dl(N);
745 SDValue Op1 = N->getOperand(0);
746 SDValue Op2 = N->getOperand(1);
747 unsigned OldBits = Op1.getScalarValueSizeInBits();
748
749 unsigned Opcode = N->getOpcode();
750 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
751
752 SDValue Op1Promoted, Op2Promoted;
753 if (IsShift) {
754 Op1Promoted = GetPromotedInteger(Op1);
755 Op2Promoted = ZExtPromotedInteger(Op2);
756 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
757 Op1Promoted = ZExtPromotedInteger(Op1);
758 Op2Promoted = ZExtPromotedInteger(Op2);
759 } else {
760 Op1Promoted = SExtPromotedInteger(Op1);
761 Op2Promoted = SExtPromotedInteger(Op2);
762 }
763 EVT PromotedType = Op1Promoted.getValueType();
764 unsigned NewBits = PromotedType.getScalarSizeInBits();
765
766 // Shift cannot use a min/max expansion, we can't detect overflow if all of
767 // the bits have been shifted out.
768 if (IsShift || TLI.isOperationLegalOrCustom(Opcode, PromotedType)) {
769 unsigned ShiftOp;
770 switch (Opcode) {
771 case ISD::SADDSAT:
772 case ISD::SSUBSAT:
773 case ISD::SSHLSAT:
774 ShiftOp = ISD::SRA;
775 break;
776 case ISD::UADDSAT:
777 case ISD::USUBSAT:
778 case ISD::USHLSAT:
779 ShiftOp = ISD::SRL;
780 break;
781 default:
782 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
783 "addition, subtraction or left shift");
784 }
785
786 unsigned SHLAmount = NewBits - OldBits;
787 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
788 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
789 Op1Promoted =
790 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
791 if (!IsShift)
792 Op2Promoted =
793 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
794
795 SDValue Result =
796 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
797 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
798 } else {
799 if (Opcode == ISD::USUBSAT) {
800 SDValue Max =
801 DAG.getNode(ISD::UMAX, dl, PromotedType, Op1Promoted, Op2Promoted);
802 return DAG.getNode(ISD::SUB, dl, PromotedType, Max, Op2Promoted);
803 }
804
805 if (Opcode == ISD::UADDSAT) {
806 APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits);
807 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
808 SDValue Add =
809 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
810 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
811 }
812
813 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
814 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
815 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
816 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
817 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
818 SDValue Result =
819 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
820 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
821 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
822 return Result;
823 }
824 }
825
PromoteIntRes_MULFIX(SDNode * N)826 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
827 // Can just promote the operands then continue with operation.
828 SDLoc dl(N);
829 SDValue Op1Promoted, Op2Promoted;
830 bool Signed =
831 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
832 bool Saturating =
833 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
834 if (Signed) {
835 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
836 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
837 } else {
838 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
839 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
840 }
841 EVT OldType = N->getOperand(0).getValueType();
842 EVT PromotedType = Op1Promoted.getValueType();
843 unsigned DiffSize =
844 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
845
846 if (Saturating) {
847 // Promoting the operand and result values changes the saturation width,
848 // which is extends the values that we clamp to on saturation. This could be
849 // resolved by shifting one of the operands the same amount, which would
850 // also shift the result we compare against, then shifting back.
851 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
852 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
853 DAG.getConstant(DiffSize, dl, ShiftTy));
854 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
855 Op2Promoted, N->getOperand(2));
856 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
857 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
858 DAG.getConstant(DiffSize, dl, ShiftTy));
859 }
860 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
861 N->getOperand(2));
862 }
863
SaturateWidenedDIVFIX(SDValue V,SDLoc & dl,unsigned SatW,bool Signed,const TargetLowering & TLI,SelectionDAG & DAG)864 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
865 unsigned SatW, bool Signed,
866 const TargetLowering &TLI,
867 SelectionDAG &DAG) {
868 EVT VT = V.getValueType();
869 unsigned VTW = VT.getScalarSizeInBits();
870
871 if (!Signed) {
872 // Saturate to the unsigned maximum by getting the minimum of V and the
873 // maximum.
874 return DAG.getNode(ISD::UMIN, dl, VT, V,
875 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
876 dl, VT));
877 }
878
879 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
880 // signed minimum of it and V.
881 V = DAG.getNode(ISD::SMIN, dl, VT, V,
882 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
883 dl, VT));
884 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
885 // signed maximum of it and V.
886 V = DAG.getNode(ISD::SMAX, dl, VT, V,
887 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
888 dl, VT));
889 return V;
890 }
891
earlyExpandDIVFIX(SDNode * N,SDValue LHS,SDValue RHS,unsigned Scale,const TargetLowering & TLI,SelectionDAG & DAG,unsigned SatW=0)892 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
893 unsigned Scale, const TargetLowering &TLI,
894 SelectionDAG &DAG, unsigned SatW = 0) {
895 EVT VT = LHS.getValueType();
896 unsigned VTSize = VT.getScalarSizeInBits();
897 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
898 N->getOpcode() == ISD::SDIVFIXSAT;
899 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
900 N->getOpcode() == ISD::UDIVFIXSAT;
901
902 SDLoc dl(N);
903 // Widen the types by a factor of two. This is guaranteed to expand, since it
904 // will always have enough high bits in the LHS to shift into.
905 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
906 if (VT.isVector())
907 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
908 VT.getVectorElementCount());
909 if (Signed) {
910 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
911 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
912 } else {
913 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
914 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
915 }
916
917 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
918 DAG);
919 assert(Res && "Expanding DIVFIX with wide type failed?");
920 if (Saturating) {
921 // If the caller has told us to saturate at something less, use that width
922 // instead of the type before doubling. However, it cannot be more than
923 // what we just widened!
924 assert(SatW <= VTSize &&
925 "Tried to saturate to more than the original type?");
926 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
927 TLI, DAG);
928 }
929 return DAG.getZExtOrTrunc(Res, dl, VT);
930 }
931
PromoteIntRes_DIVFIX(SDNode * N)932 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
933 SDLoc dl(N);
934 SDValue Op1Promoted, Op2Promoted;
935 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
936 N->getOpcode() == ISD::SDIVFIXSAT;
937 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
938 N->getOpcode() == ISD::UDIVFIXSAT;
939 if (Signed) {
940 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
941 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
942 } else {
943 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
944 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
945 }
946 EVT PromotedType = Op1Promoted.getValueType();
947 unsigned Scale = N->getConstantOperandVal(2);
948
949 // If the type is already legal and the operation is legal in that type, we
950 // should not early expand.
951 if (TLI.isTypeLegal(PromotedType)) {
952 TargetLowering::LegalizeAction Action =
953 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
954 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
955 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
956 unsigned Diff = PromotedType.getScalarSizeInBits() -
957 N->getValueType(0).getScalarSizeInBits();
958 if (Saturating)
959 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
960 DAG.getConstant(Diff, dl, ShiftTy));
961 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
962 Op2Promoted, N->getOperand(2));
963 if (Saturating)
964 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
965 DAG.getConstant(Diff, dl, ShiftTy));
966 return Res;
967 }
968 }
969
970 // See if we can perform the division in this type without expanding.
971 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
972 Op2Promoted, Scale, DAG)) {
973 if (Saturating)
974 Res = SaturateWidenedDIVFIX(Res, dl,
975 N->getValueType(0).getScalarSizeInBits(),
976 Signed, TLI, DAG);
977 return Res;
978 }
979 // If we cannot, expand it to twice the type width. If we are saturating, give
980 // it the original width as a saturating width so we don't need to emit
981 // two saturations.
982 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
983 N->getValueType(0).getScalarSizeInBits());
984 }
985
PromoteIntRes_SADDSUBO(SDNode * N,unsigned ResNo)986 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
987 if (ResNo == 1)
988 return PromoteIntRes_Overflow(N);
989
990 // The operation overflowed iff the result in the larger type is not the
991 // sign extension of its truncation to the original type.
992 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
993 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
994 EVT OVT = N->getOperand(0).getValueType();
995 EVT NVT = LHS.getValueType();
996 SDLoc dl(N);
997
998 // Do the arithmetic in the larger type.
999 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1000 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1001
1002 // Calculate the overflow flag: sign extend the arithmetic result from
1003 // the original type.
1004 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1005 DAG.getValueType(OVT));
1006 // Overflowed if and only if this is not equal to Res.
1007 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1008
1009 // Use the calculated overflow everywhere.
1010 ReplaceValueWith(SDValue(N, 1), Ofl);
1011
1012 return Res;
1013 }
1014
PromoteIntRes_SELECT(SDNode * N)1015 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
1016 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1017 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1018 return DAG.getSelect(SDLoc(N),
1019 LHS.getValueType(), N->getOperand(0), LHS, RHS);
1020 }
1021
PromoteIntRes_VSELECT(SDNode * N)1022 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
1023 SDValue Mask = N->getOperand(0);
1024
1025 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1026 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1027 return DAG.getNode(ISD::VSELECT, SDLoc(N),
1028 LHS.getValueType(), Mask, LHS, RHS);
1029 }
1030
PromoteIntRes_SELECT_CC(SDNode * N)1031 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1032 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1033 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1034 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1035 LHS.getValueType(), N->getOperand(0),
1036 N->getOperand(1), LHS, RHS, N->getOperand(4));
1037 }
1038
PromoteIntRes_SETCC(SDNode * N)1039 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1040 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1041 EVT InVT = N->getOperand(OpNo).getValueType();
1042 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1043
1044 EVT SVT = getSetCCResultType(InVT);
1045
1046 // If we got back a type that needs to be promoted, this likely means the
1047 // the input type also needs to be promoted. So get the promoted type for
1048 // the input and try the query again.
1049 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1050 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1051 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1052 SVT = getSetCCResultType(InVT);
1053 } else {
1054 // Input type isn't promoted, just use the default promoted type.
1055 SVT = NVT;
1056 }
1057 }
1058
1059 SDLoc dl(N);
1060 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1061 "Vector compare must return a vector result!");
1062
1063 // Get the SETCC result using the canonical SETCC type.
1064 SDValue SetCC;
1065 if (N->isStrictFPOpcode()) {
1066 EVT VTs[] = {SVT, MVT::Other};
1067 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1068 N->getOperand(2), N->getOperand(3)};
1069 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
1070 // Legalize the chain result - switch anything that used the old chain to
1071 // use the new one.
1072 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1073 } else
1074 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1075 N->getOperand(1), N->getOperand(2));
1076
1077 // Convert to the expected type.
1078 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1079 }
1080
PromoteIntRes_SHL(SDNode * N)1081 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1082 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1083 SDValue RHS = N->getOperand(1);
1084 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1085 RHS = ZExtPromotedInteger(RHS);
1086 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
1087 }
1088
PromoteIntRes_SIGN_EXTEND_INREG(SDNode * N)1089 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1090 SDValue Op = GetPromotedInteger(N->getOperand(0));
1091 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1092 Op.getValueType(), Op, N->getOperand(1));
1093 }
1094
PromoteIntRes_SimpleIntBinOp(SDNode * N)1095 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1096 // The input may have strange things in the top bits of the registers, but
1097 // these operations don't care. They may have weird bits going out, but
1098 // that too is okay if they are integer operations.
1099 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1100 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1101 return DAG.getNode(N->getOpcode(), SDLoc(N),
1102 LHS.getValueType(), LHS, RHS);
1103 }
1104
PromoteIntRes_SExtIntBinOp(SDNode * N)1105 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1106 // Sign extend the input.
1107 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1108 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1109 return DAG.getNode(N->getOpcode(), SDLoc(N),
1110 LHS.getValueType(), LHS, RHS);
1111 }
1112
PromoteIntRes_ZExtIntBinOp(SDNode * N)1113 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1114 // Zero extend the input.
1115 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1116 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1117 return DAG.getNode(N->getOpcode(), SDLoc(N),
1118 LHS.getValueType(), LHS, RHS);
1119 }
1120
PromoteIntRes_UMINUMAX(SDNode * N)1121 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1122 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1123 // whatever is best for the target.
1124 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0));
1125 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1));
1126 return DAG.getNode(N->getOpcode(), SDLoc(N),
1127 LHS.getValueType(), LHS, RHS);
1128 }
1129
PromoteIntRes_SRA(SDNode * N)1130 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1131 // The input value must be properly sign extended.
1132 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1133 SDValue RHS = N->getOperand(1);
1134 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1135 RHS = ZExtPromotedInteger(RHS);
1136 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
1137 }
1138
PromoteIntRes_SRL(SDNode * N)1139 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1140 // The input value must be properly zero extended.
1141 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1142 SDValue RHS = N->getOperand(1);
1143 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1144 RHS = ZExtPromotedInteger(RHS);
1145 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
1146 }
1147
PromoteIntRes_Rotate(SDNode * N)1148 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1149 // Lower the rotate to shifts and ORs which can be promoted.
1150 SDValue Res;
1151 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
1152 ReplaceValueWith(SDValue(N, 0), Res);
1153 return SDValue();
1154 }
1155
PromoteIntRes_FunnelShift(SDNode * N)1156 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1157 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1158 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1159 SDValue Amount = GetPromotedInteger(N->getOperand(2));
1160
1161 SDLoc DL(N);
1162 EVT OldVT = N->getOperand(0).getValueType();
1163 EVT VT = Lo.getValueType();
1164 unsigned Opcode = N->getOpcode();
1165 bool IsFSHR = Opcode == ISD::FSHR;
1166 unsigned OldBits = OldVT.getScalarSizeInBits();
1167 unsigned NewBits = VT.getScalarSizeInBits();
1168
1169 // Amount has to be interpreted modulo the old bit width.
1170 Amount =
1171 DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT));
1172
1173 // If the promoted type is twice the size (or more), then we use the
1174 // traditional funnel 'double' shift codegen. This isn't necessary if the
1175 // shift amount is constant.
1176 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1177 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1178 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) &&
1179 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1180 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1181 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1182 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1183 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1184 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount);
1185 if (!IsFSHR)
1186 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1187 return Res;
1188 }
1189
1190 // Shift Lo up to occupy the upper bits of the promoted type.
1191 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT);
1192 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1193
1194 // Increase Amount to shift the result into the lower bits of the promoted
1195 // type.
1196 if (IsFSHR)
1197 Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset);
1198
1199 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount);
1200 }
1201
PromoteIntRes_TRUNCATE(SDNode * N)1202 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1203 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1204 SDValue Res;
1205 SDValue InOp = N->getOperand(0);
1206 SDLoc dl(N);
1207
1208 switch (getTypeAction(InOp.getValueType())) {
1209 default: llvm_unreachable("Unknown type action!");
1210 case TargetLowering::TypeLegal:
1211 case TargetLowering::TypeExpandInteger:
1212 Res = InOp;
1213 break;
1214 case TargetLowering::TypePromoteInteger:
1215 Res = GetPromotedInteger(InOp);
1216 break;
1217 case TargetLowering::TypeSplitVector: {
1218 EVT InVT = InOp.getValueType();
1219 assert(InVT.isVector() && "Cannot split scalar types");
1220 unsigned NumElts = InVT.getVectorNumElements();
1221 assert(NumElts == NVT.getVectorNumElements() &&
1222 "Dst and Src must have the same number of elements");
1223 assert(isPowerOf2_32(NumElts) &&
1224 "Promoted vector type must be a power of two");
1225
1226 SDValue EOp1, EOp2;
1227 GetSplitVector(InOp, EOp1, EOp2);
1228
1229 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1230 NumElts/2);
1231 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1232 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1233
1234 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1235 }
1236 case TargetLowering::TypeWidenVector: {
1237 SDValue WideInOp = GetWidenedVector(InOp);
1238
1239 // Truncate widened InOp.
1240 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1241 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1242 N->getValueType(0).getScalarType(), NumElem);
1243 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1244
1245 // Zero extend so that the elements are of same type as those of NVT
1246 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1247 NumElem);
1248 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1249
1250 // Extract the low NVT subvector.
1251 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1252 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1253 }
1254 }
1255
1256 // Truncate to NVT instead of VT
1257 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1258 }
1259
PromoteIntRes_UADDSUBO(SDNode * N,unsigned ResNo)1260 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1261 if (ResNo == 1)
1262 return PromoteIntRes_Overflow(N);
1263
1264 // The operation overflowed iff the result in the larger type is not the
1265 // zero extension of its truncation to the original type.
1266 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1267 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1268 EVT OVT = N->getOperand(0).getValueType();
1269 EVT NVT = LHS.getValueType();
1270 SDLoc dl(N);
1271
1272 // Do the arithmetic in the larger type.
1273 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1274 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1275
1276 // Calculate the overflow flag: zero extend the arithmetic result from
1277 // the original type.
1278 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1279 // Overflowed if and only if this is not equal to Res.
1280 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1281
1282 // Use the calculated overflow everywhere.
1283 ReplaceValueWith(SDValue(N, 1), Ofl);
1284
1285 return Res;
1286 }
1287
1288 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1289 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
1290 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
PromoteIntRes_ADDSUBCARRY(SDNode * N,unsigned ResNo)1291 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1292 if (ResNo == 1)
1293 return PromoteIntRes_Overflow(N);
1294
1295 // We need to sign-extend the operands so the carry value computed by the
1296 // wide operation will be equivalent to the carry value computed by the
1297 // narrow operation.
1298 // An ADDCARRY can generate carry only if any of the operands has its
1299 // most significant bit set. Sign extension propagates the most significant
1300 // bit into the higher bits which means the extra bit that the narrow
1301 // addition would need (i.e. the carry) will be propagated through the higher
1302 // bits of the wide addition.
1303 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1304 // preserved by sign extension.
1305 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1306 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1307
1308 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1309
1310 // Do the arithmetic in the wide type.
1311 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1312 LHS, RHS, N->getOperand(2));
1313
1314 // Update the users of the original carry/borrow value.
1315 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1316
1317 return SDValue(Res.getNode(), 0);
1318 }
1319
PromoteIntRes_SADDSUBO_CARRY(SDNode * N,unsigned ResNo)1320 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1321 unsigned ResNo) {
1322 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1323 return PromoteIntRes_Overflow(N);
1324 }
1325
PromoteIntRes_ABS(SDNode * N)1326 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1327 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1328 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1329 }
1330
PromoteIntRes_XMULO(SDNode * N,unsigned ResNo)1331 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1332 // Promote the overflow bit trivially.
1333 if (ResNo == 1)
1334 return PromoteIntRes_Overflow(N);
1335
1336 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1337 SDLoc DL(N);
1338 EVT SmallVT = LHS.getValueType();
1339
1340 // To determine if the result overflowed in a larger type, we extend the
1341 // input to the larger type, do the multiply (checking if it overflows),
1342 // then also check the high bits of the result to see if overflow happened
1343 // there.
1344 if (N->getOpcode() == ISD::SMULO) {
1345 LHS = SExtPromotedInteger(LHS);
1346 RHS = SExtPromotedInteger(RHS);
1347 } else {
1348 LHS = ZExtPromotedInteger(LHS);
1349 RHS = ZExtPromotedInteger(RHS);
1350 }
1351 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1352 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1353
1354 // Overflow occurred if it occurred in the larger type, or if the high part
1355 // of the result does not zero/sign-extend the low part. Check this second
1356 // possibility first.
1357 SDValue Overflow;
1358 if (N->getOpcode() == ISD::UMULO) {
1359 // Unsigned overflow occurred if the high part is non-zero.
1360 unsigned Shift = SmallVT.getScalarSizeInBits();
1361 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG);
1362 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1363 DAG.getConstant(Shift, DL, ShiftTy));
1364 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1365 DAG.getConstant(0, DL, Hi.getValueType()),
1366 ISD::SETNE);
1367 } else {
1368 // Signed overflow occurred if the high part does not sign extend the low.
1369 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1370 Mul, DAG.getValueType(SmallVT));
1371 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1372 }
1373
1374 // The only other way for overflow to occur is if the multiplication in the
1375 // larger type itself overflowed.
1376 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1377 SDValue(Mul.getNode(), 1));
1378
1379 // Use the calculated overflow everywhere.
1380 ReplaceValueWith(SDValue(N, 1), Overflow);
1381 return Mul;
1382 }
1383
PromoteIntRes_UNDEF(SDNode * N)1384 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1385 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1386 N->getValueType(0)));
1387 }
1388
PromoteIntRes_VSCALE(SDNode * N)1389 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1390 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1391
1392 APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
1393 return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1394 }
1395
PromoteIntRes_VAARG(SDNode * N)1396 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1397 SDValue Chain = N->getOperand(0); // Get the chain.
1398 SDValue Ptr = N->getOperand(1); // Get the pointer.
1399 EVT VT = N->getValueType(0);
1400 SDLoc dl(N);
1401
1402 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1403 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1404 // The argument is passed as NumRegs registers of type RegVT.
1405
1406 SmallVector<SDValue, 8> Parts(NumRegs);
1407 for (unsigned i = 0; i < NumRegs; ++i) {
1408 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1409 N->getConstantOperandVal(3));
1410 Chain = Parts[i].getValue(1);
1411 }
1412
1413 // Handle endianness of the load.
1414 if (DAG.getDataLayout().isBigEndian())
1415 std::reverse(Parts.begin(), Parts.end());
1416
1417 // Assemble the parts in the promoted type.
1418 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1419 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1420 for (unsigned i = 1; i < NumRegs; ++i) {
1421 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1422 // Shift it to the right position and "or" it in.
1423 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1424 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1425 TLI.getPointerTy(DAG.getDataLayout())));
1426 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1427 }
1428
1429 // Modified the chain result - switch anything that used the old chain to
1430 // use the new one.
1431 ReplaceValueWith(SDValue(N, 1), Chain);
1432
1433 return Res;
1434 }
1435
1436 //===----------------------------------------------------------------------===//
1437 // Integer Operand Promotion
1438 //===----------------------------------------------------------------------===//
1439
1440 /// PromoteIntegerOperand - This method is called when the specified operand of
1441 /// the specified node is found to need promotion. At this point, all of the
1442 /// result types of the node are known to be legal, but other operands of the
1443 /// node may need promotion or expansion as well as the specified one.
PromoteIntegerOperand(SDNode * N,unsigned OpNo)1444 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1445 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1446 dbgs() << "\n");
1447 SDValue Res = SDValue();
1448 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1449 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1450 return false;
1451 }
1452
1453 switch (N->getOpcode()) {
1454 default:
1455 #ifndef NDEBUG
1456 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1457 N->dump(&DAG); dbgs() << "\n";
1458 #endif
1459 llvm_unreachable("Do not know how to promote this operator's operand!");
1460
1461 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1462 case ISD::ATOMIC_STORE:
1463 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1464 break;
1465 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1466 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1467 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1468 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1469 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1470 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1471 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1472 case ISD::INSERT_VECTOR_ELT:
1473 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1474 case ISD::SCALAR_TO_VECTOR:
1475 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1476 case ISD::SPLAT_VECTOR:
1477 Res = PromoteIntOp_SPLAT_VECTOR(N); break;
1478 case ISD::VSELECT:
1479 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1480 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1481 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1482 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1483 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1484 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1485 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1486 OpNo); break;
1487 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1488 OpNo); break;
1489 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1490 OpNo); break;
1491 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1492 OpNo); break;
1493 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1494 OpNo); break;
1495 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1496 case ISD::FP16_TO_FP:
1497 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1498 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1499 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1500 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1501
1502 case ISD::SHL:
1503 case ISD::SRA:
1504 case ISD::SRL:
1505 case ISD::ROTL:
1506 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1507
1508 case ISD::SADDO_CARRY:
1509 case ISD::SSUBO_CARRY:
1510 case ISD::ADDCARRY:
1511 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1512
1513 case ISD::FRAMEADDR:
1514 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1515
1516 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1517
1518 case ISD::SMULFIX:
1519 case ISD::SMULFIXSAT:
1520 case ISD::UMULFIX:
1521 case ISD::UMULFIXSAT:
1522 case ISD::SDIVFIX:
1523 case ISD::SDIVFIXSAT:
1524 case ISD::UDIVFIX:
1525 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1526
1527 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1528
1529 case ISD::VECREDUCE_ADD:
1530 case ISD::VECREDUCE_MUL:
1531 case ISD::VECREDUCE_AND:
1532 case ISD::VECREDUCE_OR:
1533 case ISD::VECREDUCE_XOR:
1534 case ISD::VECREDUCE_SMAX:
1535 case ISD::VECREDUCE_SMIN:
1536 case ISD::VECREDUCE_UMAX:
1537 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1538 }
1539
1540 // If the result is null, the sub-method took care of registering results etc.
1541 if (!Res.getNode()) return false;
1542
1543 // If the result is N, the sub-method updated N in place. Tell the legalizer
1544 // core about this.
1545 if (Res.getNode() == N)
1546 return true;
1547
1548 const bool IsStrictFp = N->isStrictFPOpcode();
1549 assert(Res.getValueType() == N->getValueType(0) &&
1550 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1551 "Invalid operand expansion");
1552 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1553 Res.dump());
1554
1555 ReplaceValueWith(SDValue(N, 0), Res);
1556 if (IsStrictFp)
1557 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1558
1559 return false;
1560 }
1561
1562 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1563 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
PromoteSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode CCCode)1564 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1565 ISD::CondCode CCCode) {
1566 // We have to insert explicit sign or zero extends. Note that we could
1567 // insert sign extends for ALL conditions. For those operations where either
1568 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1569 // which will choose the cheapest for the target.
1570 switch (CCCode) {
1571 default: llvm_unreachable("Unknown integer comparison!");
1572 case ISD::SETEQ:
1573 case ISD::SETNE: {
1574 SDValue OpL = GetPromotedInteger(NewLHS);
1575 SDValue OpR = GetPromotedInteger(NewRHS);
1576
1577 // We would prefer to promote the comparison operand with sign extension.
1578 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1579 // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1580 // instruction, which is redundant eventually.
1581 unsigned OpLEffectiveBits =
1582 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1583 unsigned OpREffectiveBits =
1584 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1585 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1586 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1587 NewLHS = OpL;
1588 NewRHS = OpR;
1589 } else {
1590 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1591 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1592 }
1593 break;
1594 }
1595 case ISD::SETUGE:
1596 case ISD::SETUGT:
1597 case ISD::SETULE:
1598 case ISD::SETULT:
1599 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1600 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1601 break;
1602 case ISD::SETGE:
1603 case ISD::SETGT:
1604 case ISD::SETLT:
1605 case ISD::SETLE:
1606 NewLHS = SExtPromotedInteger(NewLHS);
1607 NewRHS = SExtPromotedInteger(NewRHS);
1608 break;
1609 }
1610 }
1611
PromoteIntOp_ANY_EXTEND(SDNode * N)1612 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1613 SDValue Op = GetPromotedInteger(N->getOperand(0));
1614 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1615 }
1616
PromoteIntOp_ATOMIC_STORE(AtomicSDNode * N)1617 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1618 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1619 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1620 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1621 }
1622
PromoteIntOp_BITCAST(SDNode * N)1623 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1624 // This should only occur in unusual situations like bitcasting to an
1625 // x86_fp80, so just turn it into a store+load
1626 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1627 }
1628
PromoteIntOp_BR_CC(SDNode * N,unsigned OpNo)1629 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1630 assert(OpNo == 2 && "Don't know how to promote this operand!");
1631
1632 SDValue LHS = N->getOperand(2);
1633 SDValue RHS = N->getOperand(3);
1634 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1635
1636 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1637 // legal types.
1638 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1639 N->getOperand(1), LHS, RHS, N->getOperand(4)),
1640 0);
1641 }
1642
PromoteIntOp_BRCOND(SDNode * N,unsigned OpNo)1643 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1644 assert(OpNo == 1 && "only know how to promote condition");
1645
1646 // Promote all the way up to the canonical SetCC type.
1647 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1648
1649 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1650 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1651 N->getOperand(2)), 0);
1652 }
1653
PromoteIntOp_BUILD_PAIR(SDNode * N)1654 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1655 // Since the result type is legal, the operands must promote to it.
1656 EVT OVT = N->getOperand(0).getValueType();
1657 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1658 SDValue Hi = GetPromotedInteger(N->getOperand(1));
1659 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1660 SDLoc dl(N);
1661
1662 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1663 DAG.getConstant(OVT.getSizeInBits(), dl,
1664 TLI.getPointerTy(DAG.getDataLayout())));
1665 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1666 }
1667
PromoteIntOp_BUILD_VECTOR(SDNode * N)1668 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1669 // The vector type is legal but the element type is not. This implies
1670 // that the vector is a power-of-two in length and that the element
1671 // type does not have a strange size (eg: it is not i1).
1672 EVT VecVT = N->getValueType(0);
1673 unsigned NumElts = VecVT.getVectorNumElements();
1674 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1675 "Legal vector of one illegal element?");
1676
1677 // Promote the inserted value. The type does not need to match the
1678 // vector element type. Check that any extra bits introduced will be
1679 // truncated away.
1680 assert(N->getOperand(0).getValueSizeInBits() >=
1681 N->getValueType(0).getScalarSizeInBits() &&
1682 "Type of inserted value narrower than vector element type!");
1683
1684 SmallVector<SDValue, 16> NewOps;
1685 for (unsigned i = 0; i < NumElts; ++i)
1686 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1687
1688 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1689 }
1690
PromoteIntOp_INSERT_VECTOR_ELT(SDNode * N,unsigned OpNo)1691 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1692 unsigned OpNo) {
1693 if (OpNo == 1) {
1694 // Promote the inserted value. This is valid because the type does not
1695 // have to match the vector element type.
1696
1697 // Check that any extra bits introduced will be truncated away.
1698 assert(N->getOperand(1).getValueSizeInBits() >=
1699 N->getValueType(0).getScalarSizeInBits() &&
1700 "Type of inserted value narrower than vector element type!");
1701 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1702 GetPromotedInteger(N->getOperand(1)),
1703 N->getOperand(2)),
1704 0);
1705 }
1706
1707 assert(OpNo == 2 && "Different operand and result vector types?");
1708
1709 // Promote the index.
1710 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1711 TLI.getVectorIdxTy(DAG.getDataLayout()));
1712 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1713 N->getOperand(1), Idx), 0);
1714 }
1715
PromoteIntOp_SCALAR_TO_VECTOR(SDNode * N)1716 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1717 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1718 // the operand in place.
1719 return SDValue(DAG.UpdateNodeOperands(N,
1720 GetPromotedInteger(N->getOperand(0))), 0);
1721 }
1722
PromoteIntOp_SPLAT_VECTOR(SDNode * N)1723 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1724 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
1725 // operand in place.
1726 return SDValue(
1727 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1728 }
1729
PromoteIntOp_SELECT(SDNode * N,unsigned OpNo)1730 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1731 assert(OpNo == 0 && "Only know how to promote the condition!");
1732 SDValue Cond = N->getOperand(0);
1733 EVT OpTy = N->getOperand(1).getValueType();
1734
1735 if (N->getOpcode() == ISD::VSELECT)
1736 if (SDValue Res = WidenVSELECTMask(N))
1737 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1738 Res, N->getOperand(1), N->getOperand(2));
1739
1740 // Promote all the way up to the canonical SetCC type.
1741 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1742 Cond = PromoteTargetBoolean(Cond, OpVT);
1743
1744 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1745 N->getOperand(2)), 0);
1746 }
1747
PromoteIntOp_SELECT_CC(SDNode * N,unsigned OpNo)1748 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1749 assert(OpNo == 0 && "Don't know how to promote this operand!");
1750
1751 SDValue LHS = N->getOperand(0);
1752 SDValue RHS = N->getOperand(1);
1753 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1754
1755 // The CC (#4) and the possible return values (#2 and #3) have legal types.
1756 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1757 N->getOperand(3), N->getOperand(4)), 0);
1758 }
1759
PromoteIntOp_SETCC(SDNode * N,unsigned OpNo)1760 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1761 assert(OpNo == 0 && "Don't know how to promote this operand!");
1762
1763 SDValue LHS = N->getOperand(0);
1764 SDValue RHS = N->getOperand(1);
1765 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1766
1767 // The CC (#2) is always legal.
1768 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1769 }
1770
PromoteIntOp_Shift(SDNode * N)1771 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1772 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1773 ZExtPromotedInteger(N->getOperand(1))), 0);
1774 }
1775
PromoteIntOp_SIGN_EXTEND(SDNode * N)1776 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1777 SDValue Op = GetPromotedInteger(N->getOperand(0));
1778 SDLoc dl(N);
1779 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1780 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1781 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1782 }
1783
PromoteIntOp_SINT_TO_FP(SDNode * N)1784 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1785 return SDValue(DAG.UpdateNodeOperands(N,
1786 SExtPromotedInteger(N->getOperand(0))), 0);
1787 }
1788
PromoteIntOp_STRICT_SINT_TO_FP(SDNode * N)1789 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
1790 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1791 SExtPromotedInteger(N->getOperand(1))), 0);
1792 }
1793
PromoteIntOp_STORE(StoreSDNode * N,unsigned OpNo)1794 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1795 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1796 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1797 SDLoc dl(N);
1798
1799 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1800
1801 // Truncate the value and store the result.
1802 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1803 N->getMemoryVT(), N->getMemOperand());
1804 }
1805
PromoteIntOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)1806 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1807 unsigned OpNo) {
1808
1809 SDValue DataOp = N->getValue();
1810 EVT DataVT = DataOp.getValueType();
1811 SDValue Mask = N->getMask();
1812 SDLoc dl(N);
1813
1814 bool TruncateStore = false;
1815 if (OpNo == 4) {
1816 Mask = PromoteTargetBoolean(Mask, DataVT);
1817 // Update in place.
1818 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1819 NewOps[4] = Mask;
1820 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1821 } else { // Data operand
1822 assert(OpNo == 1 && "Unexpected operand for promotion");
1823 DataOp = GetPromotedInteger(DataOp);
1824 TruncateStore = true;
1825 }
1826
1827 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(),
1828 N->getOffset(), Mask, N->getMemoryVT(),
1829 N->getMemOperand(), N->getAddressingMode(),
1830 TruncateStore, N->isCompressingStore());
1831 }
1832
PromoteIntOp_MLOAD(MaskedLoadSDNode * N,unsigned OpNo)1833 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1834 unsigned OpNo) {
1835 assert(OpNo == 3 && "Only know how to promote the mask!");
1836 EVT DataVT = N->getValueType(0);
1837 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1838 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1839 NewOps[OpNo] = Mask;
1840 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1841 if (Res == N)
1842 return SDValue(Res, 0);
1843
1844 // Update triggered CSE, do our own replacement since caller can't.
1845 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1846 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1847 return SDValue();
1848 }
1849
PromoteIntOp_MGATHER(MaskedGatherSDNode * N,unsigned OpNo)1850 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1851 unsigned OpNo) {
1852
1853 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1854 if (OpNo == 2) {
1855 // The Mask
1856 EVT DataVT = N->getValueType(0);
1857 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1858 } else if (OpNo == 4) {
1859 // The Index
1860 if (N->isIndexSigned())
1861 // Need to sign extend the index since the bits will likely be used.
1862 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1863 else
1864 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1865 } else
1866 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1867
1868 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1869 if (Res == N)
1870 return SDValue(Res, 0);
1871
1872 // Update triggered CSE, do our own replacement since caller can't.
1873 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1874 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1875 return SDValue();
1876 }
1877
PromoteIntOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)1878 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1879 unsigned OpNo) {
1880 bool TruncateStore = N->isTruncatingStore();
1881 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1882 if (OpNo == 2) {
1883 // The Mask
1884 EVT DataVT = N->getValue().getValueType();
1885 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1886 } else if (OpNo == 4) {
1887 // The Index
1888 if (N->isIndexSigned())
1889 // Need to sign extend the index since the bits will likely be used.
1890 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1891 else
1892 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1893
1894 N->setIndexType(TLI.getCanonicalIndexType(N->getIndexType(),
1895 N->getMemoryVT(), NewOps[OpNo]));
1896 } else {
1897 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1898 TruncateStore = true;
1899 }
1900
1901 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
1902 SDLoc(N), NewOps, N->getMemOperand(),
1903 N->getIndexType(), TruncateStore);
1904 }
1905
PromoteIntOp_TRUNCATE(SDNode * N)1906 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1907 SDValue Op = GetPromotedInteger(N->getOperand(0));
1908 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1909 }
1910
PromoteIntOp_UINT_TO_FP(SDNode * N)1911 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1912 return SDValue(DAG.UpdateNodeOperands(N,
1913 ZExtPromotedInteger(N->getOperand(0))), 0);
1914 }
1915
PromoteIntOp_STRICT_UINT_TO_FP(SDNode * N)1916 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
1917 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1918 ZExtPromotedInteger(N->getOperand(1))), 0);
1919 }
1920
PromoteIntOp_ZERO_EXTEND(SDNode * N)1921 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1922 SDLoc dl(N);
1923 SDValue Op = GetPromotedInteger(N->getOperand(0));
1924 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1925 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
1926 }
1927
PromoteIntOp_ADDSUBCARRY(SDNode * N,unsigned OpNo)1928 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1929 assert(OpNo == 2 && "Don't know how to promote this operand!");
1930
1931 SDValue LHS = N->getOperand(0);
1932 SDValue RHS = N->getOperand(1);
1933 SDValue Carry = N->getOperand(2);
1934 SDLoc DL(N);
1935
1936 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1937
1938 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1939 }
1940
PromoteIntOp_FIX(SDNode * N)1941 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
1942 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1943 return SDValue(
1944 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1945 }
1946
PromoteIntOp_FRAMERETURNADDR(SDNode * N)1947 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1948 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1949 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1950 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1951 }
1952
PromoteIntOp_PREFETCH(SDNode * N,unsigned OpNo)1953 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1954 assert(OpNo > 1 && "Don't know how to promote this operand!");
1955 // Promote the rw, locality, and cache type arguments to a supported integer
1956 // width.
1957 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1958 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1959 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1960 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1961 Op2, Op3, Op4),
1962 0);
1963 }
1964
PromoteIntOp_FPOWI(SDNode * N)1965 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1966 SDValue Op = SExtPromotedInteger(N->getOperand(1));
1967 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1968 }
1969
PromoteIntOp_VECREDUCE(SDNode * N)1970 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1971 SDLoc dl(N);
1972 SDValue Op;
1973 switch (N->getOpcode()) {
1974 default: llvm_unreachable("Expected integer vector reduction");
1975 case ISD::VECREDUCE_ADD:
1976 case ISD::VECREDUCE_MUL:
1977 case ISD::VECREDUCE_AND:
1978 case ISD::VECREDUCE_OR:
1979 case ISD::VECREDUCE_XOR:
1980 Op = GetPromotedInteger(N->getOperand(0));
1981 break;
1982 case ISD::VECREDUCE_SMAX:
1983 case ISD::VECREDUCE_SMIN:
1984 Op = SExtPromotedInteger(N->getOperand(0));
1985 break;
1986 case ISD::VECREDUCE_UMAX:
1987 case ISD::VECREDUCE_UMIN:
1988 Op = ZExtPromotedInteger(N->getOperand(0));
1989 break;
1990 }
1991
1992 EVT EltVT = Op.getValueType().getVectorElementType();
1993 EVT VT = N->getValueType(0);
1994 if (VT.bitsGE(EltVT))
1995 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1996
1997 // Result size must be >= element size. If this is not the case after
1998 // promotion, also promote the result type and then truncate.
1999 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
2000 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
2001 }
2002
2003 //===----------------------------------------------------------------------===//
2004 // Integer Result Expansion
2005 //===----------------------------------------------------------------------===//
2006
2007 /// ExpandIntegerResult - This method is called when the specified result of the
2008 /// specified node is found to need expansion. At this point, the node may also
2009 /// have invalid operands or may have other results that need promotion, we just
2010 /// know that (at least) one result needs expansion.
ExpandIntegerResult(SDNode * N,unsigned ResNo)2011 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2012 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
2013 dbgs() << "\n");
2014 SDValue Lo, Hi;
2015 Lo = Hi = SDValue();
2016
2017 // See if the target wants to custom expand this node.
2018 if (CustomLowerNode(N, N->getValueType(ResNo), true))
2019 return;
2020
2021 switch (N->getOpcode()) {
2022 default:
2023 #ifndef NDEBUG
2024 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2025 N->dump(&DAG); dbgs() << "\n";
2026 #endif
2027 report_fatal_error("Do not know how to expand the result of this "
2028 "operator!");
2029
2030 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2031 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
2032 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2033 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2034 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2035
2036 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2037 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2038 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2039 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2040 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2041
2042 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2043 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2044 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2045 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2046 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2047 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2048 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2049 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2050 case ISD::CTLZ_ZERO_UNDEF:
2051 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2052 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2053 case ISD::CTTZ_ZERO_UNDEF:
2054 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2055 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
2056 case ISD::STRICT_FP_TO_SINT:
2057 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
2058 case ISD::STRICT_FP_TO_UINT:
2059 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
2060 case ISD::FP_TO_SINT_SAT:
2061 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2062 case ISD::STRICT_LLROUND:
2063 case ISD::STRICT_LLRINT:
2064 case ISD::LLROUND:
2065 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
2066 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2067 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2068 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
2069 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2070 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2071 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2072 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2073 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2074 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2075 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2076 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2077 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2078
2079 case ISD::ATOMIC_LOAD_ADD:
2080 case ISD::ATOMIC_LOAD_SUB:
2081 case ISD::ATOMIC_LOAD_AND:
2082 case ISD::ATOMIC_LOAD_CLR:
2083 case ISD::ATOMIC_LOAD_OR:
2084 case ISD::ATOMIC_LOAD_XOR:
2085 case ISD::ATOMIC_LOAD_NAND:
2086 case ISD::ATOMIC_LOAD_MIN:
2087 case ISD::ATOMIC_LOAD_MAX:
2088 case ISD::ATOMIC_LOAD_UMIN:
2089 case ISD::ATOMIC_LOAD_UMAX:
2090 case ISD::ATOMIC_SWAP:
2091 case ISD::ATOMIC_CMP_SWAP: {
2092 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2093 SplitInteger(Tmp.first, Lo, Hi);
2094 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2095 break;
2096 }
2097 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2098 AtomicSDNode *AN = cast<AtomicSDNode>(N);
2099 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2100 SDValue Tmp = DAG.getAtomicCmpSwap(
2101 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
2102 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2103 AN->getMemOperand());
2104
2105 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2106 // success simply by comparing the loaded value against the ingoing
2107 // comparison.
2108 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2109 N->getOperand(2), ISD::SETEQ);
2110
2111 SplitInteger(Tmp, Lo, Hi);
2112 ReplaceValueWith(SDValue(N, 1), Success);
2113 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2114 break;
2115 }
2116
2117 case ISD::AND:
2118 case ISD::OR:
2119 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2120
2121 case ISD::UMAX:
2122 case ISD::SMAX:
2123 case ISD::UMIN:
2124 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2125
2126 case ISD::ADD:
2127 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2128
2129 case ISD::ADDC:
2130 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2131
2132 case ISD::ADDE:
2133 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2134
2135 case ISD::ADDCARRY:
2136 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
2137
2138 case ISD::SADDO_CARRY:
2139 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2140
2141 case ISD::SHL:
2142 case ISD::SRA:
2143 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2144
2145 case ISD::SADDO:
2146 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2147 case ISD::UADDO:
2148 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2149 case ISD::UMULO:
2150 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2151
2152 case ISD::SADDSAT:
2153 case ISD::UADDSAT:
2154 case ISD::SSUBSAT:
2155 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2156
2157 case ISD::SSHLSAT:
2158 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2159
2160 case ISD::SMULFIX:
2161 case ISD::SMULFIXSAT:
2162 case ISD::UMULFIX:
2163 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2164
2165 case ISD::SDIVFIX:
2166 case ISD::SDIVFIXSAT:
2167 case ISD::UDIVFIX:
2168 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2169
2170 case ISD::VECREDUCE_ADD:
2171 case ISD::VECREDUCE_MUL:
2172 case ISD::VECREDUCE_AND:
2173 case ISD::VECREDUCE_OR:
2174 case ISD::VECREDUCE_XOR:
2175 case ISD::VECREDUCE_SMAX:
2176 case ISD::VECREDUCE_SMIN:
2177 case ISD::VECREDUCE_UMAX:
2178 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2179
2180 case ISD::ROTL:
2181 case ISD::ROTR:
2182 ExpandIntRes_Rotate(N, Lo, Hi);
2183 break;
2184
2185 case ISD::FSHL:
2186 case ISD::FSHR:
2187 ExpandIntRes_FunnelShift(N, Lo, Hi);
2188 break;
2189 }
2190
2191 // If Lo/Hi is null, the sub-method took care of registering results etc.
2192 if (Lo.getNode())
2193 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2194 }
2195
2196 /// Lower an atomic node to the appropriate builtin call.
ExpandAtomic(SDNode * Node)2197 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2198 unsigned Opc = Node->getOpcode();
2199 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2200 AtomicOrdering order = cast<AtomicSDNode>(Node)->getOrdering();
2201 // Lower to outline atomic libcall if outline atomics enabled,
2202 // or to sync libcall otherwise
2203 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2204 EVT RetVT = Node->getValueType(0);
2205 TargetLowering::MakeLibCallOptions CallOptions;
2206 SmallVector<SDValue, 4> Ops;
2207 if (TLI.getLibcallName(LC)) {
2208 Ops.append(Node->op_begin() + 2, Node->op_end());
2209 Ops.push_back(Node->getOperand(1));
2210 } else {
2211 LC = RTLIB::getSYNC(Opc, VT);
2212 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2213 "Unexpected atomic op or value type!");
2214 Ops.append(Node->op_begin() + 1, Node->op_end());
2215 }
2216 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2217 Node->getOperand(0));
2218 }
2219
2220 /// N is a shift by a value that needs to be expanded,
2221 /// and the shift amount is a constant 'Amt'. Expand the operation.
ExpandShiftByConstant(SDNode * N,const APInt & Amt,SDValue & Lo,SDValue & Hi)2222 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2223 SDValue &Lo, SDValue &Hi) {
2224 SDLoc DL(N);
2225 // Expand the incoming operand to be shifted, so that we have its parts
2226 SDValue InL, InH;
2227 GetExpandedInteger(N->getOperand(0), InL, InH);
2228
2229 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2230 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2231 if (!Amt) {
2232 Lo = InL;
2233 Hi = InH;
2234 return;
2235 }
2236
2237 EVT NVT = InL.getValueType();
2238 unsigned VTBits = N->getValueType(0).getSizeInBits();
2239 unsigned NVTBits = NVT.getSizeInBits();
2240 EVT ShTy = N->getOperand(1).getValueType();
2241
2242 if (N->getOpcode() == ISD::SHL) {
2243 if (Amt.ugt(VTBits)) {
2244 Lo = Hi = DAG.getConstant(0, DL, NVT);
2245 } else if (Amt.ugt(NVTBits)) {
2246 Lo = DAG.getConstant(0, DL, NVT);
2247 Hi = DAG.getNode(ISD::SHL, DL,
2248 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2249 } else if (Amt == NVTBits) {
2250 Lo = DAG.getConstant(0, DL, NVT);
2251 Hi = InL;
2252 } else {
2253 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
2254 Hi = DAG.getNode(ISD::OR, DL, NVT,
2255 DAG.getNode(ISD::SHL, DL, NVT, InH,
2256 DAG.getConstant(Amt, DL, ShTy)),
2257 DAG.getNode(ISD::SRL, DL, NVT, InL,
2258 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2259 }
2260 return;
2261 }
2262
2263 if (N->getOpcode() == ISD::SRL) {
2264 if (Amt.ugt(VTBits)) {
2265 Lo = Hi = DAG.getConstant(0, DL, NVT);
2266 } else if (Amt.ugt(NVTBits)) {
2267 Lo = DAG.getNode(ISD::SRL, DL,
2268 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2269 Hi = DAG.getConstant(0, DL, NVT);
2270 } else if (Amt == NVTBits) {
2271 Lo = InH;
2272 Hi = DAG.getConstant(0, DL, NVT);
2273 } else {
2274 Lo = DAG.getNode(ISD::OR, DL, NVT,
2275 DAG.getNode(ISD::SRL, DL, NVT, InL,
2276 DAG.getConstant(Amt, DL, ShTy)),
2277 DAG.getNode(ISD::SHL, DL, NVT, InH,
2278 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2279 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2280 }
2281 return;
2282 }
2283
2284 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2285 if (Amt.ugt(VTBits)) {
2286 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2287 DAG.getConstant(NVTBits - 1, DL, ShTy));
2288 } else if (Amt.ugt(NVTBits)) {
2289 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2290 DAG.getConstant(Amt - NVTBits, DL, ShTy));
2291 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2292 DAG.getConstant(NVTBits - 1, DL, ShTy));
2293 } else if (Amt == NVTBits) {
2294 Lo = InH;
2295 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2296 DAG.getConstant(NVTBits - 1, DL, ShTy));
2297 } else {
2298 Lo = DAG.getNode(ISD::OR, DL, NVT,
2299 DAG.getNode(ISD::SRL, DL, NVT, InL,
2300 DAG.getConstant(Amt, DL, ShTy)),
2301 DAG.getNode(ISD::SHL, DL, NVT, InH,
2302 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2303 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2304 }
2305 }
2306
2307 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2308 /// this shift based on knowledge of the high bit of the shift amount. If we
2309 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2310 /// shift amount.
2311 bool DAGTypeLegalizer::
ExpandShiftWithKnownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2312 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2313 SDValue Amt = N->getOperand(1);
2314 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2315 EVT ShTy = Amt.getValueType();
2316 unsigned ShBits = ShTy.getScalarSizeInBits();
2317 unsigned NVTBits = NVT.getScalarSizeInBits();
2318 assert(isPowerOf2_32(NVTBits) &&
2319 "Expanded integer type size not a power of two!");
2320 SDLoc dl(N);
2321
2322 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2323 KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2324
2325 // If we don't know anything about the high bits, exit.
2326 if (((Known.Zero|Known.One) & HighBitMask) == 0)
2327 return false;
2328
2329 // Get the incoming operand to be shifted.
2330 SDValue InL, InH;
2331 GetExpandedInteger(N->getOperand(0), InL, InH);
2332
2333 // If we know that any of the high bits of the shift amount are one, then we
2334 // can do this as a couple of simple shifts.
2335 if (Known.One.intersects(HighBitMask)) {
2336 // Mask out the high bit, which we know is set.
2337 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2338 DAG.getConstant(~HighBitMask, dl, ShTy));
2339
2340 switch (N->getOpcode()) {
2341 default: llvm_unreachable("Unknown shift");
2342 case ISD::SHL:
2343 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2344 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2345 return true;
2346 case ISD::SRL:
2347 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2348 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2349 return true;
2350 case ISD::SRA:
2351 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2352 DAG.getConstant(NVTBits - 1, dl, ShTy));
2353 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2354 return true;
2355 }
2356 }
2357
2358 // If we know that all of the high bits of the shift amount are zero, then we
2359 // can do this as a couple of simple shifts.
2360 if (HighBitMask.isSubsetOf(Known.Zero)) {
2361 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2362 // shift if x is zero. We can use XOR here because x is known to be smaller
2363 // than 32.
2364 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2365 DAG.getConstant(NVTBits - 1, dl, ShTy));
2366
2367 unsigned Op1, Op2;
2368 switch (N->getOpcode()) {
2369 default: llvm_unreachable("Unknown shift");
2370 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2371 case ISD::SRL:
2372 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2373 }
2374
2375 // When shifting right the arithmetic for Lo and Hi is swapped.
2376 if (N->getOpcode() != ISD::SHL)
2377 std::swap(InL, InH);
2378
2379 // Use a little trick to get the bits that move from Lo to Hi. First
2380 // shift by one bit.
2381 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2382 // Then compute the remaining shift with amount-1.
2383 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2384
2385 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2386 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2387
2388 if (N->getOpcode() != ISD::SHL)
2389 std::swap(Hi, Lo);
2390 return true;
2391 }
2392
2393 return false;
2394 }
2395
2396 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2397 /// of any size.
2398 bool DAGTypeLegalizer::
ExpandShiftWithUnknownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2399 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2400 SDValue Amt = N->getOperand(1);
2401 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2402 EVT ShTy = Amt.getValueType();
2403 unsigned NVTBits = NVT.getSizeInBits();
2404 assert(isPowerOf2_32(NVTBits) &&
2405 "Expanded integer type size not a power of two!");
2406 SDLoc dl(N);
2407
2408 // Get the incoming operand to be shifted.
2409 SDValue InL, InH;
2410 GetExpandedInteger(N->getOperand(0), InL, InH);
2411
2412 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2413 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2414 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2415 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2416 Amt, NVBitsNode, ISD::SETULT);
2417 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2418 Amt, DAG.getConstant(0, dl, ShTy),
2419 ISD::SETEQ);
2420
2421 SDValue LoS, HiS, LoL, HiL;
2422 switch (N->getOpcode()) {
2423 default: llvm_unreachable("Unknown shift");
2424 case ISD::SHL:
2425 // Short: ShAmt < NVTBits
2426 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2427 HiS = DAG.getNode(ISD::OR, dl, NVT,
2428 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2429 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2430
2431 // Long: ShAmt >= NVTBits
2432 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2433 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2434
2435 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2436 Hi = DAG.getSelect(dl, NVT, isZero, InH,
2437 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2438 return true;
2439 case ISD::SRL:
2440 // Short: ShAmt < NVTBits
2441 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2442 LoS = DAG.getNode(ISD::OR, dl, NVT,
2443 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2444 // FIXME: If Amt is zero, the following shift generates an undefined result
2445 // on some architectures.
2446 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2447
2448 // Long: ShAmt >= NVTBits
2449 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2450 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2451
2452 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2453 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2454 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2455 return true;
2456 case ISD::SRA:
2457 // Short: ShAmt < NVTBits
2458 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2459 LoS = DAG.getNode(ISD::OR, dl, NVT,
2460 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2461 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2462
2463 // Long: ShAmt >= NVTBits
2464 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2465 DAG.getConstant(NVTBits - 1, dl, ShTy));
2466 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2467
2468 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2469 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2470 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2471 return true;
2472 }
2473 }
2474
getExpandedMinMaxOps(int Op)2475 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2476
2477 switch (Op) {
2478 default: llvm_unreachable("invalid min/max opcode");
2479 case ISD::SMAX:
2480 return std::make_pair(ISD::SETGT, ISD::UMAX);
2481 case ISD::UMAX:
2482 return std::make_pair(ISD::SETUGT, ISD::UMAX);
2483 case ISD::SMIN:
2484 return std::make_pair(ISD::SETLT, ISD::UMIN);
2485 case ISD::UMIN:
2486 return std::make_pair(ISD::SETULT, ISD::UMIN);
2487 }
2488 }
2489
ExpandIntRes_MINMAX(SDNode * N,SDValue & Lo,SDValue & Hi)2490 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2491 SDValue &Lo, SDValue &Hi) {
2492 SDLoc DL(N);
2493 ISD::NodeType LoOpc;
2494 ISD::CondCode CondC;
2495 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2496
2497 // Expand the subcomponents.
2498 SDValue LHSL, LHSH, RHSL, RHSH;
2499 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2500 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2501
2502 // Value types
2503 EVT NVT = LHSL.getValueType();
2504 EVT CCT = getSetCCResultType(NVT);
2505
2506 // Hi part is always the same op
2507 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2508
2509 // We need to know whether to select Lo part that corresponds to 'winning'
2510 // Hi part or if Hi parts are equal.
2511 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2512 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2513
2514 // Lo part corresponding to the 'winning' Hi part
2515 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2516
2517 // Recursed Lo part if Hi parts are equal, this uses unsigned version
2518 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2519
2520 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2521 }
2522
ExpandIntRes_ADDSUB(SDNode * N,SDValue & Lo,SDValue & Hi)2523 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2524 SDValue &Lo, SDValue &Hi) {
2525 SDLoc dl(N);
2526 // Expand the subcomponents.
2527 SDValue LHSL, LHSH, RHSL, RHSH;
2528 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2529 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2530
2531 EVT NVT = LHSL.getValueType();
2532 SDValue LoOps[2] = { LHSL, RHSL };
2533 SDValue HiOps[3] = { LHSH, RHSH };
2534
2535 bool HasOpCarry = TLI.isOperationLegalOrCustom(
2536 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2537 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2538 if (HasOpCarry) {
2539 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2540 if (N->getOpcode() == ISD::ADD) {
2541 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2542 HiOps[2] = Lo.getValue(1);
2543 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2544 } else {
2545 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2546 HiOps[2] = Lo.getValue(1);
2547 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2548 }
2549 return;
2550 }
2551
2552 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2553 // them. TODO: Teach operation legalization how to expand unsupported
2554 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2555 // a carry of type MVT::Glue, but there doesn't seem to be any way to
2556 // generate a value of this type in the expanded code sequence.
2557 bool hasCarry =
2558 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2559 ISD::ADDC : ISD::SUBC,
2560 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2561
2562 if (hasCarry) {
2563 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2564 if (N->getOpcode() == ISD::ADD) {
2565 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2566 HiOps[2] = Lo.getValue(1);
2567 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2568 } else {
2569 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2570 HiOps[2] = Lo.getValue(1);
2571 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2572 }
2573 return;
2574 }
2575
2576 bool hasOVF =
2577 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2578 ISD::UADDO : ISD::USUBO,
2579 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2580 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
2581
2582 if (hasOVF) {
2583 EVT OvfVT = getSetCCResultType(NVT);
2584 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2585 int RevOpc;
2586 if (N->getOpcode() == ISD::ADD) {
2587 RevOpc = ISD::SUB;
2588 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2589 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2590 } else {
2591 RevOpc = ISD::ADD;
2592 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2593 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2594 }
2595 SDValue OVF = Lo.getValue(1);
2596
2597 switch (BoolType) {
2598 case TargetLoweringBase::UndefinedBooleanContent:
2599 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2600 LLVM_FALLTHROUGH;
2601 case TargetLoweringBase::ZeroOrOneBooleanContent:
2602 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2603 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2604 break;
2605 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2606 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2607 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2608 }
2609 return;
2610 }
2611
2612 if (N->getOpcode() == ISD::ADD) {
2613 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2614 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2615 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2616 ISD::SETULT);
2617
2618 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
2619 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2620 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2621 return;
2622 }
2623
2624 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2625 DAG.getConstant(1, dl, NVT),
2626 DAG.getConstant(0, dl, NVT));
2627 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2628 ISD::SETULT);
2629 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2630 DAG.getConstant(1, dl, NVT), Carry1);
2631 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2632 } else {
2633 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2634 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2635 SDValue Cmp =
2636 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2637 LoOps[0], LoOps[1], ISD::SETULT);
2638
2639 SDValue Borrow;
2640 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
2641 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2642 else
2643 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2644 DAG.getConstant(0, dl, NVT));
2645
2646 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2647 }
2648 }
2649
ExpandIntRes_ADDSUBC(SDNode * N,SDValue & Lo,SDValue & Hi)2650 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2651 SDValue &Lo, SDValue &Hi) {
2652 // Expand the subcomponents.
2653 SDValue LHSL, LHSH, RHSL, RHSH;
2654 SDLoc dl(N);
2655 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2656 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2657 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2658 SDValue LoOps[2] = { LHSL, RHSL };
2659 SDValue HiOps[3] = { LHSH, RHSH };
2660
2661 if (N->getOpcode() == ISD::ADDC) {
2662 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2663 HiOps[2] = Lo.getValue(1);
2664 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2665 } else {
2666 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2667 HiOps[2] = Lo.getValue(1);
2668 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2669 }
2670
2671 // Legalized the flag result - switch anything that used the old flag to
2672 // use the new one.
2673 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2674 }
2675
ExpandIntRes_ADDSUBE(SDNode * N,SDValue & Lo,SDValue & Hi)2676 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2677 SDValue &Lo, SDValue &Hi) {
2678 // Expand the subcomponents.
2679 SDValue LHSL, LHSH, RHSL, RHSH;
2680 SDLoc dl(N);
2681 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2682 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2683 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2684 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2685 SDValue HiOps[3] = { LHSH, RHSH };
2686
2687 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2688 HiOps[2] = Lo.getValue(1);
2689 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2690
2691 // Legalized the flag result - switch anything that used the old flag to
2692 // use the new one.
2693 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2694 }
2695
ExpandIntRes_UADDSUBO(SDNode * N,SDValue & Lo,SDValue & Hi)2696 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2697 SDValue &Lo, SDValue &Hi) {
2698 SDValue LHS = N->getOperand(0);
2699 SDValue RHS = N->getOperand(1);
2700 SDLoc dl(N);
2701
2702 SDValue Ovf;
2703
2704 unsigned CarryOp, NoCarryOp;
2705 ISD::CondCode Cond;
2706 switch(N->getOpcode()) {
2707 case ISD::UADDO:
2708 CarryOp = ISD::ADDCARRY;
2709 NoCarryOp = ISD::ADD;
2710 Cond = ISD::SETULT;
2711 break;
2712 case ISD::USUBO:
2713 CarryOp = ISD::SUBCARRY;
2714 NoCarryOp = ISD::SUB;
2715 Cond = ISD::SETUGT;
2716 break;
2717 default:
2718 llvm_unreachable("Node has unexpected Opcode");
2719 }
2720
2721 bool HasCarryOp = TLI.isOperationLegalOrCustom(
2722 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2723
2724 if (HasCarryOp) {
2725 // Expand the subcomponents.
2726 SDValue LHSL, LHSH, RHSL, RHSH;
2727 GetExpandedInteger(LHS, LHSL, LHSH);
2728 GetExpandedInteger(RHS, RHSL, RHSH);
2729 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2730 SDValue LoOps[2] = { LHSL, RHSL };
2731 SDValue HiOps[3] = { LHSH, RHSH };
2732
2733 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2734 HiOps[2] = Lo.getValue(1);
2735 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
2736
2737 Ovf = Hi.getValue(1);
2738 } else {
2739 // Expand the result by simply replacing it with the equivalent
2740 // non-overflow-checking operation.
2741 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2742 SplitInteger(Sum, Lo, Hi);
2743
2744 // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2745 // overflows iff a - b > a.
2746 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2747 }
2748
2749 // Legalized the flag result - switch anything that used the old flag to
2750 // use the new one.
2751 ReplaceValueWith(SDValue(N, 1), Ovf);
2752 }
2753
ExpandIntRes_ADDSUBCARRY(SDNode * N,SDValue & Lo,SDValue & Hi)2754 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2755 SDValue &Lo, SDValue &Hi) {
2756 // Expand the subcomponents.
2757 SDValue LHSL, LHSH, RHSL, RHSH;
2758 SDLoc dl(N);
2759 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2760 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2761 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2762 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2763 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2764
2765 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2766 HiOps[2] = Lo.getValue(1);
2767 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2768
2769 // Legalized the flag result - switch anything that used the old flag to
2770 // use the new one.
2771 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2772 }
2773
ExpandIntRes_SADDSUBO_CARRY(SDNode * N,SDValue & Lo,SDValue & Hi)2774 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
2775 SDValue &Lo, SDValue &Hi) {
2776 // Expand the subcomponents.
2777 SDValue LHSL, LHSH, RHSL, RHSH;
2778 SDLoc dl(N);
2779 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2780 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2781 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2782
2783 // We need to use an unsigned carry op for the lo part.
2784 unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY
2785 : ISD::SUBCARRY;
2786 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
2787 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
2788
2789 // Legalized the flag result - switch anything that used the old flag to
2790 // use the new one.
2791 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2792 }
2793
ExpandIntRes_ANY_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)2794 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2795 SDValue &Lo, SDValue &Hi) {
2796 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2797 SDLoc dl(N);
2798 SDValue Op = N->getOperand(0);
2799 if (Op.getValueType().bitsLE(NVT)) {
2800 // The low part is any extension of the input (which degenerates to a copy).
2801 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2802 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2803 } else {
2804 // For example, extension of an i48 to an i64. The operand type necessarily
2805 // promotes to the result type, so will end up being expanded too.
2806 assert(getTypeAction(Op.getValueType()) ==
2807 TargetLowering::TypePromoteInteger &&
2808 "Only know how to promote this result!");
2809 SDValue Res = GetPromotedInteger(Op);
2810 assert(Res.getValueType() == N->getValueType(0) &&
2811 "Operand over promoted?");
2812 // Split the promoted operand. This will simplify when it is expanded.
2813 SplitInteger(Res, Lo, Hi);
2814 }
2815 }
2816
ExpandIntRes_AssertSext(SDNode * N,SDValue & Lo,SDValue & Hi)2817 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2818 SDValue &Lo, SDValue &Hi) {
2819 SDLoc dl(N);
2820 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2821 EVT NVT = Lo.getValueType();
2822 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2823 unsigned NVTBits = NVT.getSizeInBits();
2824 unsigned EVTBits = EVT.getSizeInBits();
2825
2826 if (NVTBits < EVTBits) {
2827 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2828 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2829 EVTBits - NVTBits)));
2830 } else {
2831 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2832 // The high part replicates the sign bit of Lo, make it explicit.
2833 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2834 DAG.getConstant(NVTBits - 1, dl,
2835 TLI.getPointerTy(DAG.getDataLayout())));
2836 }
2837 }
2838
ExpandIntRes_AssertZext(SDNode * N,SDValue & Lo,SDValue & Hi)2839 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2840 SDValue &Lo, SDValue &Hi) {
2841 SDLoc dl(N);
2842 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2843 EVT NVT = Lo.getValueType();
2844 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2845 unsigned NVTBits = NVT.getSizeInBits();
2846 unsigned EVTBits = EVT.getSizeInBits();
2847
2848 if (NVTBits < EVTBits) {
2849 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2850 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2851 EVTBits - NVTBits)));
2852 } else {
2853 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2854 // The high part must be zero, make it explicit.
2855 Hi = DAG.getConstant(0, dl, NVT);
2856 }
2857 }
2858
ExpandIntRes_BITREVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)2859 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2860 SDValue &Lo, SDValue &Hi) {
2861 SDLoc dl(N);
2862 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2863 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2864 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2865 }
2866
ExpandIntRes_BSWAP(SDNode * N,SDValue & Lo,SDValue & Hi)2867 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2868 SDValue &Lo, SDValue &Hi) {
2869 SDLoc dl(N);
2870 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2871 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2872 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2873 }
2874
ExpandIntRes_PARITY(SDNode * N,SDValue & Lo,SDValue & Hi)2875 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
2876 SDValue &Hi) {
2877 SDLoc dl(N);
2878 // parity(HiLo) -> parity(Lo^Hi)
2879 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2880 EVT NVT = Lo.getValueType();
2881 Lo =
2882 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
2883 Hi = DAG.getConstant(0, dl, NVT);
2884 }
2885
ExpandIntRes_Constant(SDNode * N,SDValue & Lo,SDValue & Hi)2886 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2887 SDValue &Lo, SDValue &Hi) {
2888 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2889 unsigned NBitWidth = NVT.getSizeInBits();
2890 auto Constant = cast<ConstantSDNode>(N);
2891 const APInt &Cst = Constant->getAPIntValue();
2892 bool IsTarget = Constant->isTargetOpcode();
2893 bool IsOpaque = Constant->isOpaque();
2894 SDLoc dl(N);
2895 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2896 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2897 IsOpaque);
2898 }
2899
ExpandIntRes_ABS(SDNode * N,SDValue & Lo,SDValue & Hi)2900 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2901 SDLoc dl(N);
2902
2903 SDValue N0 = N->getOperand(0);
2904 GetExpandedInteger(N0, Lo, Hi);
2905 EVT NVT = Lo.getValueType();
2906
2907 // If we have ADDCARRY, use the expanded form of the sra+add+xor sequence we
2908 // use in LegalizeDAG. The ADD part of the expansion is based on
2909 // ExpandIntRes_ADDSUB which also uses ADDCARRY/UADDO after checking that
2910 // ADDCARRY is LegalOrCustom. Each of the pieces here can be further expanded
2911 // if needed. Shift expansion has a special case for filling with sign bits
2912 // so that we will only end up with one SRA.
2913 bool HasAddCarry = TLI.isOperationLegalOrCustom(
2914 ISD::ADDCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2915 if (HasAddCarry) {
2916 EVT ShiftAmtTy = getShiftAmountTyForConstant(NVT, TLI, DAG);
2917 SDValue Sign =
2918 DAG.getNode(ISD::SRA, dl, NVT, Hi,
2919 DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy));
2920 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2921 Lo = DAG.getNode(ISD::UADDO, dl, VTList, Lo, Sign);
2922 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
2923 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
2924 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
2925 return;
2926 }
2927
2928 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2929 EVT VT = N->getValueType(0);
2930 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2931 DAG.getConstant(0, dl, VT), N0);
2932 SDValue NegLo, NegHi;
2933 SplitInteger(Neg, NegLo, NegHi);
2934
2935 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2936 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2937 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2938 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2939 }
2940
ExpandIntRes_CTLZ(SDNode * N,SDValue & Lo,SDValue & Hi)2941 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2942 SDValue &Lo, SDValue &Hi) {
2943 SDLoc dl(N);
2944 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2945 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2946 EVT NVT = Lo.getValueType();
2947
2948 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2949 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2950
2951 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2952 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2953
2954 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2955 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2956 DAG.getConstant(NVT.getSizeInBits(), dl,
2957 NVT)));
2958 Hi = DAG.getConstant(0, dl, NVT);
2959 }
2960
ExpandIntRes_CTPOP(SDNode * N,SDValue & Lo,SDValue & Hi)2961 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2962 SDValue &Lo, SDValue &Hi) {
2963 SDLoc dl(N);
2964 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2965 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2966 EVT NVT = Lo.getValueType();
2967 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2968 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2969 Hi = DAG.getConstant(0, dl, NVT);
2970 }
2971
ExpandIntRes_CTTZ(SDNode * N,SDValue & Lo,SDValue & Hi)2972 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2973 SDValue &Lo, SDValue &Hi) {
2974 SDLoc dl(N);
2975 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2976 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2977 EVT NVT = Lo.getValueType();
2978
2979 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2980 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2981
2982 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2983 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2984
2985 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2986 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2987 DAG.getConstant(NVT.getSizeInBits(), dl,
2988 NVT)));
2989 Hi = DAG.getConstant(0, dl, NVT);
2990 }
2991
ExpandIntRes_FLT_ROUNDS(SDNode * N,SDValue & Lo,SDValue & Hi)2992 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2993 SDValue &Hi) {
2994 SDLoc dl(N);
2995 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2996 unsigned NBitWidth = NVT.getSizeInBits();
2997
2998 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2999 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0));
3000 SDValue Chain = Lo.getValue(1);
3001 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
3002 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3003 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
3004
3005 // Legalize the chain result - switch anything that used the old chain to
3006 // use the new one.
3007 ReplaceValueWith(SDValue(N, 1), Chain);
3008 }
3009
ExpandIntRes_FP_TO_SINT(SDNode * N,SDValue & Lo,SDValue & Hi)3010 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
3011 SDValue &Hi) {
3012 SDLoc dl(N);
3013 EVT VT = N->getValueType(0);
3014
3015 bool IsStrict = N->isStrictFPOpcode();
3016 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3017 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3018 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3019 Op = GetPromotedFloat(Op);
3020
3021 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3022 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3023 Op = GetSoftPromotedHalf(Op);
3024 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3025 }
3026
3027 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
3028 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
3029 TargetLowering::MakeLibCallOptions CallOptions;
3030 CallOptions.setSExt(true);
3031 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3032 CallOptions, dl, Chain);
3033 SplitInteger(Tmp.first, Lo, Hi);
3034
3035 if (IsStrict)
3036 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3037 }
3038
ExpandIntRes_FP_TO_UINT(SDNode * N,SDValue & Lo,SDValue & Hi)3039 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
3040 SDValue &Hi) {
3041 SDLoc dl(N);
3042 EVT VT = N->getValueType(0);
3043
3044 bool IsStrict = N->isStrictFPOpcode();
3045 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3046 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3047 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3048 Op = GetPromotedFloat(Op);
3049
3050 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3051 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3052 Op = GetSoftPromotedHalf(Op);
3053 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3054 }
3055
3056 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
3057 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
3058 TargetLowering::MakeLibCallOptions CallOptions;
3059 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3060 CallOptions, dl, Chain);
3061 SplitInteger(Tmp.first, Lo, Hi);
3062
3063 if (IsStrict)
3064 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3065 }
3066
ExpandIntRes_FP_TO_XINT_SAT(SDNode * N,SDValue & Lo,SDValue & Hi)3067 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3068 SDValue &Hi) {
3069 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3070 SplitInteger(Res, Lo, Hi);
3071 }
3072
ExpandIntRes_LLROUND_LLRINT(SDNode * N,SDValue & Lo,SDValue & Hi)3073 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
3074 SDValue &Hi) {
3075 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
3076
3077 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
3078 "Input type needs to be promoted!");
3079
3080 EVT VT = Op.getValueType();
3081
3082 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3083 if (N->getOpcode() == ISD::LLROUND ||
3084 N->getOpcode() == ISD::STRICT_LLROUND) {
3085 if (VT == MVT::f32)
3086 LC = RTLIB::LLROUND_F32;
3087 else if (VT == MVT::f64)
3088 LC = RTLIB::LLROUND_F64;
3089 else if (VT == MVT::f80)
3090 LC = RTLIB::LLROUND_F80;
3091 else if (VT == MVT::f128)
3092 LC = RTLIB::LLROUND_F128;
3093 else if (VT == MVT::ppcf128)
3094 LC = RTLIB::LLROUND_PPCF128;
3095 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
3096 } else if (N->getOpcode() == ISD::LLRINT ||
3097 N->getOpcode() == ISD::STRICT_LLRINT) {
3098 if (VT == MVT::f32)
3099 LC = RTLIB::LLRINT_F32;
3100 else if (VT == MVT::f64)
3101 LC = RTLIB::LLRINT_F64;
3102 else if (VT == MVT::f80)
3103 LC = RTLIB::LLRINT_F80;
3104 else if (VT == MVT::f128)
3105 LC = RTLIB::LLRINT_F128;
3106 else if (VT == MVT::ppcf128)
3107 LC = RTLIB::LLRINT_PPCF128;
3108 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
3109 } else
3110 llvm_unreachable("Unexpected opcode!");
3111
3112 SDLoc dl(N);
3113 EVT RetVT = N->getValueType(0);
3114 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();
3115
3116 TargetLowering::MakeLibCallOptions CallOptions;
3117 CallOptions.setSExt(true);
3118 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3119 Op, CallOptions, dl,
3120 Chain);
3121 SplitInteger(Tmp.first, Lo, Hi);
3122
3123 if (N->isStrictFPOpcode())
3124 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3125 }
3126
ExpandIntRes_LOAD(LoadSDNode * N,SDValue & Lo,SDValue & Hi)3127 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
3128 SDValue &Lo, SDValue &Hi) {
3129 if (N->isAtomic()) {
3130 // It's typical to have larger CAS than atomic load instructions.
3131 SDLoc dl(N);
3132 EVT VT = N->getMemoryVT();
3133 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3134 SDValue Zero = DAG.getConstant(0, dl, VT);
3135 SDValue Swap = DAG.getAtomicCmpSwap(
3136 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
3137 VT, VTs, N->getOperand(0),
3138 N->getOperand(1), Zero, Zero, N->getMemOperand());
3139 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3140 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3141 return;
3142 }
3143
3144 if (ISD::isNormalLoad(N)) {
3145 ExpandRes_NormalLoad(N, Lo, Hi);
3146 return;
3147 }
3148
3149 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
3150
3151 EVT VT = N->getValueType(0);
3152 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3153 SDValue Ch = N->getChain();
3154 SDValue Ptr = N->getBasePtr();
3155 ISD::LoadExtType ExtType = N->getExtensionType();
3156 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3157 AAMDNodes AAInfo = N->getAAInfo();
3158 SDLoc dl(N);
3159
3160 assert(NVT.isByteSized() && "Expanded type not byte sized!");
3161
3162 if (N->getMemoryVT().bitsLE(NVT)) {
3163 EVT MemVT = N->getMemoryVT();
3164
3165 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
3166 N->getOriginalAlign(), MMOFlags, AAInfo);
3167
3168 // Remember the chain.
3169 Ch = Lo.getValue(1);
3170
3171 if (ExtType == ISD::SEXTLOAD) {
3172 // The high part is obtained by SRA'ing all but one of the bits of the
3173 // lo part.
3174 unsigned LoSize = Lo.getValueSizeInBits();
3175 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3176 DAG.getConstant(LoSize - 1, dl,
3177 TLI.getPointerTy(DAG.getDataLayout())));
3178 } else if (ExtType == ISD::ZEXTLOAD) {
3179 // The high part is just a zero.
3180 Hi = DAG.getConstant(0, dl, NVT);
3181 } else {
3182 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
3183 // The high part is undefined.
3184 Hi = DAG.getUNDEF(NVT);
3185 }
3186 } else if (DAG.getDataLayout().isLittleEndian()) {
3187 // Little-endian - low bits are at low addresses.
3188 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
3189 N->getOriginalAlign(), MMOFlags, AAInfo);
3190
3191 unsigned ExcessBits =
3192 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3193 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3194
3195 // Increment the pointer to the other half.
3196 unsigned IncrementSize = NVT.getSizeInBits()/8;
3197 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3198 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3199 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3200 N->getOriginalAlign(), MMOFlags, AAInfo);
3201
3202 // Build a factor node to remember that this load is independent of the
3203 // other one.
3204 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3205 Hi.getValue(1));
3206 } else {
3207 // Big-endian - high bits are at low addresses. Favor aligned loads at
3208 // the cost of some bit-fiddling.
3209 EVT MemVT = N->getMemoryVT();
3210 unsigned EBytes = MemVT.getStoreSize();
3211 unsigned IncrementSize = NVT.getSizeInBits()/8;
3212 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3213
3214 // Load both the high bits and maybe some of the low bits.
3215 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3216 EVT::getIntegerVT(*DAG.getContext(),
3217 MemVT.getSizeInBits() - ExcessBits),
3218 N->getOriginalAlign(), MMOFlags, AAInfo);
3219
3220 // Increment the pointer to the other half.
3221 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3222 // Load the rest of the low bits.
3223 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3224 N->getPointerInfo().getWithOffset(IncrementSize),
3225 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3226 N->getOriginalAlign(), MMOFlags, AAInfo);
3227
3228 // Build a factor node to remember that this load is independent of the
3229 // other one.
3230 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3231 Hi.getValue(1));
3232
3233 if (ExcessBits < NVT.getSizeInBits()) {
3234 // Transfer low bits from the bottom of Hi to the top of Lo.
3235 Lo = DAG.getNode(
3236 ISD::OR, dl, NVT, Lo,
3237 DAG.getNode(ISD::SHL, dl, NVT, Hi,
3238 DAG.getConstant(ExcessBits, dl,
3239 TLI.getPointerTy(DAG.getDataLayout()))));
3240 // Move high bits to the right position in Hi.
3241 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
3242 Hi,
3243 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3244 TLI.getPointerTy(DAG.getDataLayout())));
3245 }
3246 }
3247
3248 // Legalize the chain result - switch anything that used the old chain to
3249 // use the new one.
3250 ReplaceValueWith(SDValue(N, 1), Ch);
3251 }
3252
ExpandIntRes_Logical(SDNode * N,SDValue & Lo,SDValue & Hi)3253 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
3254 SDValue &Lo, SDValue &Hi) {
3255 SDLoc dl(N);
3256 SDValue LL, LH, RL, RH;
3257 GetExpandedInteger(N->getOperand(0), LL, LH);
3258 GetExpandedInteger(N->getOperand(1), RL, RH);
3259 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
3260 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3261 }
3262
ExpandIntRes_MUL(SDNode * N,SDValue & Lo,SDValue & Hi)3263 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
3264 SDValue &Lo, SDValue &Hi) {
3265 EVT VT = N->getValueType(0);
3266 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3267 SDLoc dl(N);
3268
3269 SDValue LL, LH, RL, RH;
3270 GetExpandedInteger(N->getOperand(0), LL, LH);
3271 GetExpandedInteger(N->getOperand(1), RL, RH);
3272
3273 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
3274 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3275 LL, LH, RL, RH))
3276 return;
3277
3278 // If nothing else, we can make a libcall.
3279 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3280 if (VT == MVT::i16)
3281 LC = RTLIB::MUL_I16;
3282 else if (VT == MVT::i32)
3283 LC = RTLIB::MUL_I32;
3284 else if (VT == MVT::i64)
3285 LC = RTLIB::MUL_I64;
3286 else if (VT == MVT::i128)
3287 LC = RTLIB::MUL_I128;
3288
3289 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
3290 // We'll expand the multiplication by brute force because we have no other
3291 // options. This is a trivially-generalized version of the code from
3292 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
3293 // 4.3.1).
3294 unsigned Bits = NVT.getSizeInBits();
3295 unsigned HalfBits = Bits >> 1;
3296 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
3297 NVT);
3298 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
3299 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
3300
3301 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
3302 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
3303
3304 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3305 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
3306 // The type from TLI is too small to fit the shift amount we want.
3307 // Override it with i32. The shift will have to be legalized.
3308 ShiftAmtTy = MVT::i32;
3309 }
3310 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
3311 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
3312 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
3313 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
3314
3315 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
3316 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
3317 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
3318 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
3319
3320 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
3321 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
3322 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
3323
3324 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
3325 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
3326 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
3327 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
3328 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
3329
3330 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
3331 DAG.getNode(ISD::ADD, dl, NVT,
3332 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
3333 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
3334 return;
3335 }
3336
3337 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3338 TargetLowering::MakeLibCallOptions CallOptions;
3339 CallOptions.setSExt(true);
3340 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3341 Lo, Hi);
3342 }
3343
ExpandIntRes_READCYCLECOUNTER(SDNode * N,SDValue & Lo,SDValue & Hi)3344 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
3345 SDValue &Hi) {
3346 SDLoc DL(N);
3347 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3348 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
3349 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
3350 Lo = R.getValue(0);
3351 Hi = R.getValue(1);
3352 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3353 }
3354
ExpandIntRes_ADDSUBSAT(SDNode * N,SDValue & Lo,SDValue & Hi)3355 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
3356 SDValue &Hi) {
3357 SDValue Result = TLI.expandAddSubSat(N, DAG);
3358 SplitInteger(Result, Lo, Hi);
3359 }
3360
ExpandIntRes_SHLSAT(SDNode * N,SDValue & Lo,SDValue & Hi)3361 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
3362 SDValue &Hi) {
3363 SDValue Result = TLI.expandShlSat(N, DAG);
3364 SplitInteger(Result, Lo, Hi);
3365 }
3366
3367 /// This performs an expansion of the integer result for a fixed point
3368 /// multiplication. The default expansion performs rounding down towards
3369 /// negative infinity, though targets that do care about rounding should specify
3370 /// a target hook for rounding and provide their own expansion or lowering of
3371 /// fixed point multiplication to be consistent with rounding.
ExpandIntRes_MULFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3372 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
3373 SDValue &Hi) {
3374 SDLoc dl(N);
3375 EVT VT = N->getValueType(0);
3376 unsigned VTSize = VT.getScalarSizeInBits();
3377 SDValue LHS = N->getOperand(0);
3378 SDValue RHS = N->getOperand(1);
3379 uint64_t Scale = N->getConstantOperandVal(2);
3380 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
3381 N->getOpcode() == ISD::UMULFIXSAT);
3382 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
3383 N->getOpcode() == ISD::SMULFIXSAT);
3384
3385 // Handle special case when scale is equal to zero.
3386 if (!Scale) {
3387 SDValue Result;
3388 if (!Saturating) {
3389 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3390 } else {
3391 EVT BoolVT = getSetCCResultType(VT);
3392 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
3393 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
3394 SDValue Product = Result.getValue(0);
3395 SDValue Overflow = Result.getValue(1);
3396 if (Signed) {
3397 APInt MinVal = APInt::getSignedMinValue(VTSize);
3398 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
3399 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
3400 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3401 SDValue Zero = DAG.getConstant(0, dl, VT);
3402 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
3403 Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
3404 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
3405 } else {
3406 // For unsigned multiplication, we only need to check the max since we
3407 // can't really overflow towards zero.
3408 APInt MaxVal = APInt::getMaxValue(VTSize);
3409 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3410 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
3411 }
3412 }
3413 SplitInteger(Result, Lo, Hi);
3414 return;
3415 }
3416
3417 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
3418 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
3419 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
3420
3421 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3422 SDValue LL, LH, RL, RH;
3423 GetExpandedInteger(LHS, LL, LH);
3424 GetExpandedInteger(RHS, RL, RH);
3425 SmallVector<SDValue, 4> Result;
3426
3427 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
3428 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
3429 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3430 LL, LH, RL, RH)) {
3431 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
3432 return;
3433 }
3434
3435 unsigned NVTSize = NVT.getScalarSizeInBits();
3436 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
3437 "the size of the current value type");
3438 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3439
3440 // After getting the multiplication result in 4 parts, we need to perform a
3441 // shift right by the amount of the scale to get the result in that scale.
3442 //
3443 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
3444 // 128 bits that are cut into 4 32-bit parts:
3445 //
3446 // HH HL LH LL
3447 // |---32---|---32---|---32---|---32---|
3448 // 128 96 64 32 0
3449 //
3450 // |------VTSize-----|
3451 //
3452 // |NVTSize-|
3453 //
3454 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
3455 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
3456 // amount and get Lo and Hi using two funnel shifts. Or for the special case
3457 // when Scale is a multiple of NVTSize we can just pick the result without
3458 // shifting.
3459 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
3460 if (Scale % NVTSize) {
3461 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
3462 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
3463 ShiftAmount);
3464 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
3465 ShiftAmount);
3466 } else {
3467 Lo = Result[Part0];
3468 Hi = Result[Part0 + 1];
3469 }
3470
3471 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
3472 if (!Saturating)
3473 return;
3474
3475 // Can not overflow when there is no integer part.
3476 if (Scale == VTSize)
3477 return;
3478
3479 // To handle saturation we must check for overflow in the multiplication.
3480 //
3481 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
3482 // aren't all zeroes.
3483 //
3484 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
3485 // aren't all ones or all zeroes.
3486 //
3487 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
3488 // highest bit of HH determines saturation direction in the event of signed
3489 // saturation.
3490
3491 SDValue ResultHL = Result[2];
3492 SDValue ResultHH = Result[3];
3493
3494 SDValue SatMax, SatMin;
3495 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
3496 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3497 EVT BoolNVT = getSetCCResultType(NVT);
3498
3499 if (!Signed) {
3500 if (Scale < NVTSize) {
3501 // Overflow happened if ((HH | (HL >> Scale)) != 0).
3502 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3503 DAG.getConstant(Scale, dl, ShiftTy));
3504 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3505 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3506 } else if (Scale == NVTSize) {
3507 // Overflow happened if (HH != 0).
3508 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3509 } else if (Scale < VTSize) {
3510 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
3511 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3512 DAG.getConstant(Scale - NVTSize, dl,
3513 ShiftTy));
3514 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3515 } else
3516 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
3517 "(and saturation can't happen with Scale==VTSize).");
3518
3519 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3520 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3521 return;
3522 }
3523
3524 if (Scale < NVTSize) {
3525 // The number of overflow bits we can check are VTSize - Scale + 1 (we
3526 // include the sign bit). If these top bits are > 0, then we overflowed past
3527 // the max value. If these top bits are < -1, then we overflowed past the
3528 // min value. Otherwise, we did not overflow.
3529 unsigned OverflowBits = VTSize - Scale + 1;
3530 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3531 "Extent of overflow bits must start within HL");
3532 SDValue HLHiMask = DAG.getConstant(
3533 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3534 SDValue HLLoMask = DAG.getConstant(
3535 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3536 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3537 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3538 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3539 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3540 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3541 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3542 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3543 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3544 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3545 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3546 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3547 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3548 } else if (Scale == NVTSize) {
3549 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3550 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3551 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3552 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3553 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3554 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3555 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3556 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3557 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3558 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3559 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3560 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3561 } else if (Scale < VTSize) {
3562 // This is similar to the case when we saturate if Scale < NVTSize, but we
3563 // only need to check HH.
3564 unsigned OverflowBits = VTSize - Scale + 1;
3565 SDValue HHHiMask = DAG.getConstant(
3566 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3567 SDValue HHLoMask = DAG.getConstant(
3568 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3569 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3570 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3571 } else
3572 llvm_unreachable("Illegal scale for signed fixed point mul.");
3573
3574 // Saturate to signed maximum.
3575 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3576 APInt MaxLo = APInt::getAllOnesValue(NVTSize);
3577 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3578 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3579 // Saturate to signed minimum.
3580 APInt MinHi = APInt::getSignedMinValue(NVTSize);
3581 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3582 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3583 }
3584
ExpandIntRes_DIVFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3585 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
3586 SDValue &Hi) {
3587 SDLoc dl(N);
3588 // Try expanding in the existing type first.
3589 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
3590 N->getOperand(1),
3591 N->getConstantOperandVal(2), DAG);
3592
3593 if (!Res)
3594 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
3595 N->getConstantOperandVal(2), TLI, DAG);
3596 SplitInteger(Res, Lo, Hi);
3597 }
3598
ExpandIntRes_SADDSUBO(SDNode * Node,SDValue & Lo,SDValue & Hi)3599 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3600 SDValue &Lo, SDValue &Hi) {
3601 SDValue LHS = Node->getOperand(0);
3602 SDValue RHS = Node->getOperand(1);
3603 SDLoc dl(Node);
3604
3605 SDValue Ovf;
3606
3607 unsigned CarryOp;
3608 switch(Node->getOpcode()) {
3609 default: llvm_unreachable("Node has unexpected Opcode");
3610 case ISD::SADDO: CarryOp = ISD::SADDO_CARRY; break;
3611 case ISD::SSUBO: CarryOp = ISD::SSUBO_CARRY; break;
3612 }
3613
3614 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3615 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3616
3617 if (HasCarryOp) {
3618 // Expand the subcomponents.
3619 SDValue LHSL, LHSH, RHSL, RHSH;
3620 GetExpandedInteger(LHS, LHSL, LHSH);
3621 GetExpandedInteger(RHS, RHSL, RHSH);
3622 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
3623
3624 Lo = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3625 ISD::UADDO : ISD::USUBO, dl, VTList, { LHSL, RHSL });
3626 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3627
3628 Ovf = Hi.getValue(1);
3629 } else {
3630 // Expand the result by simply replacing it with the equivalent
3631 // non-overflow-checking operation.
3632 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3633 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3634 LHS, RHS);
3635 SplitInteger(Sum, Lo, Hi);
3636
3637 // Compute the overflow.
3638 //
3639 // LHSSign -> LHS >= 0
3640 // RHSSign -> RHS >= 0
3641 // SumSign -> Sum >= 0
3642 //
3643 // Add:
3644 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3645 // Sub:
3646 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3647 //
3648 EVT OType = Node->getValueType(1);
3649 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3650
3651 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3652 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3653 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3654 Node->getOpcode() == ISD::SADDO ?
3655 ISD::SETEQ : ISD::SETNE);
3656
3657 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3658 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3659
3660 Ovf = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3661 }
3662
3663 // Use the calculated overflow everywhere.
3664 ReplaceValueWith(SDValue(Node, 1), Ovf);
3665 }
3666
ExpandIntRes_SDIV(SDNode * N,SDValue & Lo,SDValue & Hi)3667 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3668 SDValue &Lo, SDValue &Hi) {
3669 EVT VT = N->getValueType(0);
3670 SDLoc dl(N);
3671 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3672
3673 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3674 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3675 SplitInteger(Res.getValue(0), Lo, Hi);
3676 return;
3677 }
3678
3679 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3680 if (VT == MVT::i16)
3681 LC = RTLIB::SDIV_I16;
3682 else if (VT == MVT::i32)
3683 LC = RTLIB::SDIV_I32;
3684 else if (VT == MVT::i64)
3685 LC = RTLIB::SDIV_I64;
3686 else if (VT == MVT::i128)
3687 LC = RTLIB::SDIV_I128;
3688 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3689
3690 TargetLowering::MakeLibCallOptions CallOptions;
3691 CallOptions.setSExt(true);
3692 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3693 }
3694
ExpandIntRes_Shift(SDNode * N,SDValue & Lo,SDValue & Hi)3695 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3696 SDValue &Lo, SDValue &Hi) {
3697 EVT VT = N->getValueType(0);
3698 SDLoc dl(N);
3699
3700 // If we can emit an efficient shift operation, do so now. Check to see if
3701 // the RHS is a constant.
3702 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3703 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3704
3705 // If we can determine that the high bit of the shift is zero or one, even if
3706 // the low bits are variable, emit this shift in an optimized form.
3707 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3708 return;
3709
3710 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3711 unsigned PartsOpc;
3712 if (N->getOpcode() == ISD::SHL) {
3713 PartsOpc = ISD::SHL_PARTS;
3714 } else if (N->getOpcode() == ISD::SRL) {
3715 PartsOpc = ISD::SRL_PARTS;
3716 } else {
3717 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3718 PartsOpc = ISD::SRA_PARTS;
3719 }
3720
3721 // Next check to see if the target supports this SHL_PARTS operation or if it
3722 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3723 // size, but create a libcall instead.
3724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3725 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3726 const bool LegalOrCustom =
3727 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3728 Action == TargetLowering::Custom;
3729
3730 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3731 // Expand the subcomponents.
3732 SDValue LHSL, LHSH;
3733 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3734 EVT VT = LHSL.getValueType();
3735
3736 // If the shift amount operand is coming from a vector legalization it may
3737 // have an illegal type. Fix that first by casting the operand, otherwise
3738 // the new SHL_PARTS operation would need further legalization.
3739 SDValue ShiftOp = N->getOperand(1);
3740 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3741 assert(ShiftTy.getScalarSizeInBits() >=
3742 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3743 "ShiftAmountTy is too small to cover the range of this type!");
3744 if (ShiftOp.getValueType() != ShiftTy)
3745 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3746
3747 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3748 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3749 Hi = Lo.getValue(1);
3750 return;
3751 }
3752
3753 // Otherwise, emit a libcall.
3754 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3755 bool isSigned;
3756 if (N->getOpcode() == ISD::SHL) {
3757 isSigned = false; /*sign irrelevant*/
3758 if (VT == MVT::i16)
3759 LC = RTLIB::SHL_I16;
3760 else if (VT == MVT::i32)
3761 LC = RTLIB::SHL_I32;
3762 else if (VT == MVT::i64)
3763 LC = RTLIB::SHL_I64;
3764 else if (VT == MVT::i128)
3765 LC = RTLIB::SHL_I128;
3766 } else if (N->getOpcode() == ISD::SRL) {
3767 isSigned = false;
3768 if (VT == MVT::i16)
3769 LC = RTLIB::SRL_I16;
3770 else if (VT == MVT::i32)
3771 LC = RTLIB::SRL_I32;
3772 else if (VT == MVT::i64)
3773 LC = RTLIB::SRL_I64;
3774 else if (VT == MVT::i128)
3775 LC = RTLIB::SRL_I128;
3776 } else {
3777 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3778 isSigned = true;
3779 if (VT == MVT::i16)
3780 LC = RTLIB::SRA_I16;
3781 else if (VT == MVT::i32)
3782 LC = RTLIB::SRA_I32;
3783 else if (VT == MVT::i64)
3784 LC = RTLIB::SRA_I64;
3785 else if (VT == MVT::i128)
3786 LC = RTLIB::SRA_I128;
3787 }
3788
3789 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3790 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3791 TargetLowering::MakeLibCallOptions CallOptions;
3792 CallOptions.setSExt(isSigned);
3793 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3794 return;
3795 }
3796
3797 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3798 llvm_unreachable("Unsupported shift!");
3799 }
3800
ExpandIntRes_SIGN_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)3801 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3802 SDValue &Lo, SDValue &Hi) {
3803 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3804 SDLoc dl(N);
3805 SDValue Op = N->getOperand(0);
3806 if (Op.getValueType().bitsLE(NVT)) {
3807 // The low part is sign extension of the input (degenerates to a copy).
3808 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3809 // The high part is obtained by SRA'ing all but one of the bits of low part.
3810 unsigned LoSize = NVT.getSizeInBits();
3811 Hi = DAG.getNode(
3812 ISD::SRA, dl, NVT, Lo,
3813 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3814 } else {
3815 // For example, extension of an i48 to an i64. The operand type necessarily
3816 // promotes to the result type, so will end up being expanded too.
3817 assert(getTypeAction(Op.getValueType()) ==
3818 TargetLowering::TypePromoteInteger &&
3819 "Only know how to promote this result!");
3820 SDValue Res = GetPromotedInteger(Op);
3821 assert(Res.getValueType() == N->getValueType(0) &&
3822 "Operand over promoted?");
3823 // Split the promoted operand. This will simplify when it is expanded.
3824 SplitInteger(Res, Lo, Hi);
3825 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3826 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3827 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3828 ExcessBits)));
3829 }
3830 }
3831
3832 void DAGTypeLegalizer::
ExpandIntRes_SIGN_EXTEND_INREG(SDNode * N,SDValue & Lo,SDValue & Hi)3833 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3834 SDLoc dl(N);
3835 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3836 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3837
3838 if (EVT.bitsLE(Lo.getValueType())) {
3839 // sext_inreg the low part if needed.
3840 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3841 N->getOperand(1));
3842
3843 // The high part gets the sign extension from the lo-part. This handles
3844 // things like sextinreg V:i64 from i8.
3845 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3846 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3847 TLI.getPointerTy(DAG.getDataLayout())));
3848 } else {
3849 // For example, extension of an i48 to an i64. Leave the low part alone,
3850 // sext_inreg the high part.
3851 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3852 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3853 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3854 ExcessBits)));
3855 }
3856 }
3857
ExpandIntRes_SREM(SDNode * N,SDValue & Lo,SDValue & Hi)3858 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3859 SDValue &Lo, SDValue &Hi) {
3860 EVT VT = N->getValueType(0);
3861 SDLoc dl(N);
3862 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3863
3864 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3865 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3866 SplitInteger(Res.getValue(1), Lo, Hi);
3867 return;
3868 }
3869
3870 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3871 if (VT == MVT::i16)
3872 LC = RTLIB::SREM_I16;
3873 else if (VT == MVT::i32)
3874 LC = RTLIB::SREM_I32;
3875 else if (VT == MVT::i64)
3876 LC = RTLIB::SREM_I64;
3877 else if (VT == MVT::i128)
3878 LC = RTLIB::SREM_I128;
3879 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3880
3881 TargetLowering::MakeLibCallOptions CallOptions;
3882 CallOptions.setSExt(true);
3883 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3884 }
3885
ExpandIntRes_TRUNCATE(SDNode * N,SDValue & Lo,SDValue & Hi)3886 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3887 SDValue &Lo, SDValue &Hi) {
3888 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3889 SDLoc dl(N);
3890 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3891 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3892 N->getOperand(0),
3893 DAG.getConstant(NVT.getSizeInBits(), dl,
3894 TLI.getPointerTy(DAG.getDataLayout())));
3895 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3896 }
3897
ExpandIntRes_XMULO(SDNode * N,SDValue & Lo,SDValue & Hi)3898 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3899 SDValue &Lo, SDValue &Hi) {
3900 EVT VT = N->getValueType(0);
3901 SDLoc dl(N);
3902
3903 if (N->getOpcode() == ISD::UMULO) {
3904 // This section expands the operation into the following sequence of
3905 // instructions. `iNh` here refers to a type which has half the bit width of
3906 // the type the original operation operated on.
3907 //
3908 // %0 = %LHS.HI != 0 && %RHS.HI != 0
3909 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3910 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3911 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3912 // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3913 // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3914 //
3915 // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3916 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3917 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3918 SplitInteger(LHS, LHSLow, LHSHigh);
3919 SplitInteger(RHS, RHSLow, RHSHigh);
3920 EVT HalfVT = LHSLow.getValueType()
3921 , BitVT = N->getValueType(1);
3922 SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3923 SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3924
3925 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3926 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3927 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3928 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3929
3930 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3931 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3932 SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3933 One.getValue(0));
3934
3935 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3936 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3937 SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3938 Two.getValue(0));
3939
3940 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3941 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3942 // operation recursively legalized?).
3943 //
3944 // Many backends understand this pattern and will convert into LOHI
3945 // themselves, if applicable.
3946 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3947 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3948 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3949 SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3950 SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3951 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3952 SplitInteger(Five, Lo, Hi);
3953 ReplaceValueWith(SDValue(N, 1), Overflow);
3954 return;
3955 }
3956
3957 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3958 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3959 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3960
3961 // Replace this with a libcall that will check overflow.
3962 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3963 if (VT == MVT::i32)
3964 LC = RTLIB::MULO_I32;
3965 else if (VT == MVT::i64)
3966 LC = RTLIB::MULO_I64;
3967 else if (VT == MVT::i128)
3968 LC = RTLIB::MULO_I128;
3969 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3970
3971 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3972 // Temporary for the overflow value, default it to zero.
3973 SDValue Chain =
3974 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3975 MachinePointerInfo());
3976
3977 TargetLowering::ArgListTy Args;
3978 TargetLowering::ArgListEntry Entry;
3979 for (const SDValue &Op : N->op_values()) {
3980 EVT ArgVT = Op.getValueType();
3981 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3982 Entry.Node = Op;
3983 Entry.Ty = ArgTy;
3984 Entry.IsSExt = true;
3985 Entry.IsZExt = false;
3986 Args.push_back(Entry);
3987 }
3988
3989 // Also pass the address of the overflow check.
3990 Entry.Node = Temp;
3991 Entry.Ty = PtrTy->getPointerTo();
3992 Entry.IsSExt = true;
3993 Entry.IsZExt = false;
3994 Args.push_back(Entry);
3995
3996 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3997
3998 TargetLowering::CallLoweringInfo CLI(DAG);
3999 CLI.setDebugLoc(dl)
4000 .setChain(Chain)
4001 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
4002 .setSExtResult();
4003
4004 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
4005
4006 SplitInteger(CallInfo.first, Lo, Hi);
4007 SDValue Temp2 =
4008 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
4009 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
4010 DAG.getConstant(0, dl, PtrVT),
4011 ISD::SETNE);
4012 // Use the overflow from the libcall everywhere.
4013 ReplaceValueWith(SDValue(N, 1), Ofl);
4014 }
4015
ExpandIntRes_UDIV(SDNode * N,SDValue & Lo,SDValue & Hi)4016 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
4017 SDValue &Lo, SDValue &Hi) {
4018 EVT VT = N->getValueType(0);
4019 SDLoc dl(N);
4020 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4021
4022 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4023 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4024 SplitInteger(Res.getValue(0), Lo, Hi);
4025 return;
4026 }
4027
4028 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4029 if (VT == MVT::i16)
4030 LC = RTLIB::UDIV_I16;
4031 else if (VT == MVT::i32)
4032 LC = RTLIB::UDIV_I32;
4033 else if (VT == MVT::i64)
4034 LC = RTLIB::UDIV_I64;
4035 else if (VT == MVT::i128)
4036 LC = RTLIB::UDIV_I128;
4037 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
4038
4039 TargetLowering::MakeLibCallOptions CallOptions;
4040 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4041 }
4042
ExpandIntRes_UREM(SDNode * N,SDValue & Lo,SDValue & Hi)4043 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
4044 SDValue &Lo, SDValue &Hi) {
4045 EVT VT = N->getValueType(0);
4046 SDLoc dl(N);
4047 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4048
4049 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4050 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4051 SplitInteger(Res.getValue(1), Lo, Hi);
4052 return;
4053 }
4054
4055 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4056 if (VT == MVT::i16)
4057 LC = RTLIB::UREM_I16;
4058 else if (VT == MVT::i32)
4059 LC = RTLIB::UREM_I32;
4060 else if (VT == MVT::i64)
4061 LC = RTLIB::UREM_I64;
4062 else if (VT == MVT::i128)
4063 LC = RTLIB::UREM_I128;
4064 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
4065
4066 TargetLowering::MakeLibCallOptions CallOptions;
4067 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4068 }
4069
ExpandIntRes_ZERO_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)4070 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
4071 SDValue &Lo, SDValue &Hi) {
4072 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4073 SDLoc dl(N);
4074 SDValue Op = N->getOperand(0);
4075 if (Op.getValueType().bitsLE(NVT)) {
4076 // The low part is zero extension of the input (degenerates to a copy).
4077 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
4078 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
4079 } else {
4080 // For example, extension of an i48 to an i64. The operand type necessarily
4081 // promotes to the result type, so will end up being expanded too.
4082 assert(getTypeAction(Op.getValueType()) ==
4083 TargetLowering::TypePromoteInteger &&
4084 "Only know how to promote this result!");
4085 SDValue Res = GetPromotedInteger(Op);
4086 assert(Res.getValueType() == N->getValueType(0) &&
4087 "Operand over promoted?");
4088 // Split the promoted operand. This will simplify when it is expanded.
4089 SplitInteger(Res, Lo, Hi);
4090 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4091 Hi = DAG.getZeroExtendInReg(Hi, dl,
4092 EVT::getIntegerVT(*DAG.getContext(),
4093 ExcessBits));
4094 }
4095 }
4096
ExpandIntRes_ATOMIC_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)4097 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
4098 SDValue &Lo, SDValue &Hi) {
4099 SDLoc dl(N);
4100 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
4101 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
4102 SDValue Zero = DAG.getConstant(0, dl, VT);
4103 SDValue Swap = DAG.getAtomicCmpSwap(
4104 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
4105 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
4106 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
4107
4108 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
4109 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
4110 }
4111
ExpandIntRes_VECREDUCE(SDNode * N,SDValue & Lo,SDValue & Hi)4112 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
4113 SDValue &Lo, SDValue &Hi) {
4114 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
4115 // both halves independently.
4116 SDValue Res = TLI.expandVecReduce(N, DAG);
4117 SplitInteger(Res, Lo, Hi);
4118 }
4119
ExpandIntRes_Rotate(SDNode * N,SDValue & Lo,SDValue & Hi)4120 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
4121 SDValue &Lo, SDValue &Hi) {
4122 // Lower the rotate to shifts and ORs which can be expanded.
4123 SDValue Res;
4124 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
4125 SplitInteger(Res, Lo, Hi);
4126 }
4127
ExpandIntRes_FunnelShift(SDNode * N,SDValue & Lo,SDValue & Hi)4128 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N,
4129 SDValue &Lo, SDValue &Hi) {
4130 // Lower the funnel shift to shifts and ORs which can be expanded.
4131 SDValue Res;
4132 TLI.expandFunnelShift(N, Res, DAG);
4133 SplitInteger(Res, Lo, Hi);
4134 }
4135
4136 //===----------------------------------------------------------------------===//
4137 // Integer Operand Expansion
4138 //===----------------------------------------------------------------------===//
4139
4140 /// ExpandIntegerOperand - This method is called when the specified operand of
4141 /// the specified node is found to need expansion. At this point, all of the
4142 /// result types of the node are known to be legal, but other operands of the
4143 /// node may need promotion or expansion as well as the specified one.
ExpandIntegerOperand(SDNode * N,unsigned OpNo)4144 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
4145 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
4146 dbgs() << "\n");
4147 SDValue Res = SDValue();
4148
4149 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4150 return false;
4151
4152 switch (N->getOpcode()) {
4153 default:
4154 #ifndef NDEBUG
4155 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
4156 N->dump(&DAG); dbgs() << "\n";
4157 #endif
4158 report_fatal_error("Do not know how to expand this operator's operand!");
4159
4160 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
4161 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
4162 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
4163 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
4164 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
4165 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
4166 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
4167 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
4168 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
4169 case ISD::STRICT_SINT_TO_FP:
4170 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
4171 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
4172 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
4173 case ISD::STRICT_UINT_TO_FP:
4174 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
4175
4176 case ISD::SHL:
4177 case ISD::SRA:
4178 case ISD::SRL:
4179 case ISD::ROTL:
4180 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
4181 case ISD::RETURNADDR:
4182 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
4183
4184 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
4185 }
4186
4187 // If the result is null, the sub-method took care of registering results etc.
4188 if (!Res.getNode()) return false;
4189
4190 // If the result is N, the sub-method updated N in place. Tell the legalizer
4191 // core about this.
4192 if (Res.getNode() == N)
4193 return true;
4194
4195 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4196 "Invalid operand expansion");
4197
4198 ReplaceValueWith(SDValue(N, 0), Res);
4199 return false;
4200 }
4201
4202 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
4203 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
IntegerExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl)4204 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
4205 SDValue &NewRHS,
4206 ISD::CondCode &CCCode,
4207 const SDLoc &dl) {
4208 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4209 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
4210 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
4211
4212 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
4213 if (RHSLo == RHSHi) {
4214 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
4215 if (RHSCST->isAllOnesValue()) {
4216 // Equality comparison to -1.
4217 NewLHS = DAG.getNode(ISD::AND, dl,
4218 LHSLo.getValueType(), LHSLo, LHSHi);
4219 NewRHS = RHSLo;
4220 return;
4221 }
4222 }
4223 }
4224
4225 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
4226 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
4227 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
4228 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
4229 return;
4230 }
4231
4232 // If this is a comparison of the sign bit, just look at the top part.
4233 // X > -1, x < 0
4234 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
4235 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
4236 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
4237 NewLHS = LHSHi;
4238 NewRHS = RHSHi;
4239 return;
4240 }
4241
4242 // FIXME: This generated code sucks.
4243 ISD::CondCode LowCC;
4244 switch (CCCode) {
4245 default: llvm_unreachable("Unknown integer setcc!");
4246 case ISD::SETLT:
4247 case ISD::SETULT: LowCC = ISD::SETULT; break;
4248 case ISD::SETGT:
4249 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
4250 case ISD::SETLE:
4251 case ISD::SETULE: LowCC = ISD::SETULE; break;
4252 case ISD::SETGE:
4253 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
4254 }
4255
4256 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
4257 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
4258 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
4259
4260 // NOTE: on targets without efficient SELECT of bools, we can always use
4261 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
4262 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
4263 nullptr);
4264 SDValue LoCmp, HiCmp;
4265 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
4266 TLI.isTypeLegal(RHSLo.getValueType()))
4267 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
4268 RHSLo, LowCC, false, DagCombineInfo, dl);
4269 if (!LoCmp.getNode())
4270 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
4271 RHSLo, LowCC);
4272 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
4273 TLI.isTypeLegal(RHSHi.getValueType()))
4274 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
4275 RHSHi, CCCode, false, DagCombineInfo, dl);
4276 if (!HiCmp.getNode())
4277 HiCmp =
4278 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
4279 LHSHi, RHSHi, DAG.getCondCode(CCCode));
4280
4281 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
4282 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
4283
4284 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
4285 CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
4286
4287 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
4288 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
4289 (LoCmpC && LoCmpC->isNullValue())))) {
4290 // For LE / GE, if high part is known false, ignore the low part.
4291 // For LT / GT: if low part is known false, return the high part.
4292 // if high part is known true, ignore the low part.
4293 NewLHS = HiCmp;
4294 NewRHS = SDValue();
4295 return;
4296 }
4297
4298 if (LHSHi == RHSHi) {
4299 // Comparing the low bits is enough.
4300 NewLHS = LoCmp;
4301 NewRHS = SDValue();
4302 return;
4303 }
4304
4305 // Lower with SETCCCARRY if the target supports it.
4306 EVT HiVT = LHSHi.getValueType();
4307 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
4308 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
4309
4310 // FIXME: Make all targets support this, then remove the other lowering.
4311 if (HasSETCCCARRY) {
4312 // SETCCCARRY can detect < and >= directly. For > and <=, flip
4313 // operands and condition code.
4314 bool FlipOperands = false;
4315 switch (CCCode) {
4316 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
4317 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
4318 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
4319 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
4320 default: break;
4321 }
4322 if (FlipOperands) {
4323 std::swap(LHSLo, RHSLo);
4324 std::swap(LHSHi, RHSHi);
4325 }
4326 // Perform a wide subtraction, feeding the carry from the low part into
4327 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
4328 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
4329 // zero or positive iff LHS >= RHS.
4330 EVT LoVT = LHSLo.getValueType();
4331 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
4332 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
4333 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
4334 LHSHi, RHSHi, LowCmp.getValue(1),
4335 DAG.getCondCode(CCCode));
4336 NewLHS = Res;
4337 NewRHS = SDValue();
4338 return;
4339 }
4340
4341 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
4342 false, DagCombineInfo, dl);
4343 if (!NewLHS.getNode())
4344 NewLHS =
4345 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
4346 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
4347 NewRHS = SDValue();
4348 }
4349
ExpandIntOp_BR_CC(SDNode * N)4350 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
4351 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
4352 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
4353 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4354
4355 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4356 // against zero to select between true and false values.
4357 if (!NewRHS.getNode()) {
4358 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4359 CCCode = ISD::SETNE;
4360 }
4361
4362 // Update N to have the operands specified.
4363 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
4364 DAG.getCondCode(CCCode), NewLHS, NewRHS,
4365 N->getOperand(4)), 0);
4366 }
4367
ExpandIntOp_SELECT_CC(SDNode * N)4368 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
4369 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4370 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
4371 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4372
4373 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4374 // against zero to select between true and false values.
4375 if (!NewRHS.getNode()) {
4376 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4377 CCCode = ISD::SETNE;
4378 }
4379
4380 // Update N to have the operands specified.
4381 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
4382 N->getOperand(2), N->getOperand(3),
4383 DAG.getCondCode(CCCode)), 0);
4384 }
4385
ExpandIntOp_SETCC(SDNode * N)4386 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
4387 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4388 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
4389 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4390
4391 // If ExpandSetCCOperands returned a scalar, use it.
4392 if (!NewRHS.getNode()) {
4393 assert(NewLHS.getValueType() == N->getValueType(0) &&
4394 "Unexpected setcc expansion!");
4395 return NewLHS;
4396 }
4397
4398 // Otherwise, update N to have the operands specified.
4399 return SDValue(
4400 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
4401 }
4402
ExpandIntOp_SETCCCARRY(SDNode * N)4403 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
4404 SDValue LHS = N->getOperand(0);
4405 SDValue RHS = N->getOperand(1);
4406 SDValue Carry = N->getOperand(2);
4407 SDValue Cond = N->getOperand(3);
4408 SDLoc dl = SDLoc(N);
4409
4410 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4411 GetExpandedInteger(LHS, LHSLo, LHSHi);
4412 GetExpandedInteger(RHS, RHSLo, RHSHi);
4413
4414 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
4415 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
4416 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
4417 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
4418 LowCmp.getValue(1), Cond);
4419 }
4420
ExpandIntOp_Shift(SDNode * N)4421 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
4422 // The value being shifted is legal, but the shift amount is too big.
4423 // It follows that either the result of the shift is undefined, or the
4424 // upper half of the shift amount is zero. Just use the lower half.
4425 SDValue Lo, Hi;
4426 GetExpandedInteger(N->getOperand(1), Lo, Hi);
4427 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
4428 }
4429
ExpandIntOp_RETURNADDR(SDNode * N)4430 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
4431 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
4432 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
4433 // constant to valid type.
4434 SDValue Lo, Hi;
4435 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4436 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
4437 }
4438
ExpandIntOp_SINT_TO_FP(SDNode * N)4439 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
4440 bool IsStrict = N->isStrictFPOpcode();
4441 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4442 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4443 EVT DstVT = N->getValueType(0);
4444 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
4445 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4446 "Don't know how to expand this SINT_TO_FP!");
4447 TargetLowering::MakeLibCallOptions CallOptions;
4448 CallOptions.setSExt(true);
4449 std::pair<SDValue, SDValue> Tmp =
4450 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4451
4452 if (!IsStrict)
4453 return Tmp.first;
4454
4455 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4456 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4457 return SDValue();
4458 }
4459
ExpandIntOp_STORE(StoreSDNode * N,unsigned OpNo)4460 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
4461 if (N->isAtomic()) {
4462 // It's typical to have larger CAS than atomic store instructions.
4463 SDLoc dl(N);
4464 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4465 N->getMemoryVT(),
4466 N->getOperand(0), N->getOperand(2),
4467 N->getOperand(1),
4468 N->getMemOperand());
4469 return Swap.getValue(1);
4470 }
4471 if (ISD::isNormalStore(N))
4472 return ExpandOp_NormalStore(N, OpNo);
4473
4474 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
4475 assert(OpNo == 1 && "Can only expand the stored value so far");
4476
4477 EVT VT = N->getOperand(1).getValueType();
4478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4479 SDValue Ch = N->getChain();
4480 SDValue Ptr = N->getBasePtr();
4481 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4482 AAMDNodes AAInfo = N->getAAInfo();
4483 SDLoc dl(N);
4484 SDValue Lo, Hi;
4485
4486 assert(NVT.isByteSized() && "Expanded type not byte sized!");
4487
4488 if (N->getMemoryVT().bitsLE(NVT)) {
4489 GetExpandedInteger(N->getValue(), Lo, Hi);
4490 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4491 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags,
4492 AAInfo);
4493 }
4494
4495 if (DAG.getDataLayout().isLittleEndian()) {
4496 // Little-endian - low bits are at low addresses.
4497 GetExpandedInteger(N->getValue(), Lo, Hi);
4498
4499 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4500 N->getOriginalAlign(), MMOFlags, AAInfo);
4501
4502 unsigned ExcessBits =
4503 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4504 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
4505
4506 // Increment the pointer to the other half.
4507 unsigned IncrementSize = NVT.getSizeInBits()/8;
4508 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
4509 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
4510 N->getPointerInfo().getWithOffset(IncrementSize),
4511 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo);
4512 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4513 }
4514
4515 // Big-endian - high bits are at low addresses. Favor aligned stores at
4516 // the cost of some bit-fiddling.
4517 GetExpandedInteger(N->getValue(), Lo, Hi);
4518
4519 EVT ExtVT = N->getMemoryVT();
4520 unsigned EBytes = ExtVT.getStoreSize();
4521 unsigned IncrementSize = NVT.getSizeInBits()/8;
4522 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4523 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
4524 ExtVT.getSizeInBits() - ExcessBits);
4525
4526 if (ExcessBits < NVT.getSizeInBits()) {
4527 // Transfer high bits from the top of Lo to the bottom of Hi.
4528 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
4529 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
4530 TLI.getPointerTy(DAG.getDataLayout())));
4531 Hi = DAG.getNode(
4532 ISD::OR, dl, NVT, Hi,
4533 DAG.getNode(ISD::SRL, dl, NVT, Lo,
4534 DAG.getConstant(ExcessBits, dl,
4535 TLI.getPointerTy(DAG.getDataLayout()))));
4536 }
4537
4538 // Store both the high bits and maybe some of the low bits.
4539 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
4540 N->getOriginalAlign(), MMOFlags, AAInfo);
4541
4542 // Increment the pointer to the other half.
4543 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
4544 // Store the lowest ExcessBits bits in the second half.
4545 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
4546 N->getPointerInfo().getWithOffset(IncrementSize),
4547 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
4548 N->getOriginalAlign(), MMOFlags, AAInfo);
4549 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4550 }
4551
ExpandIntOp_TRUNCATE(SDNode * N)4552 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
4553 SDValue InL, InH;
4554 GetExpandedInteger(N->getOperand(0), InL, InH);
4555 // Just truncate the low part of the source.
4556 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
4557 }
4558
ExpandIntOp_UINT_TO_FP(SDNode * N)4559 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
4560 bool IsStrict = N->isStrictFPOpcode();
4561 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4562 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4563 EVT DstVT = N->getValueType(0);
4564 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
4565 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4566 "Don't know how to expand this UINT_TO_FP!");
4567 TargetLowering::MakeLibCallOptions CallOptions;
4568 CallOptions.setSExt(true);
4569 std::pair<SDValue, SDValue> Tmp =
4570 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4571
4572 if (!IsStrict)
4573 return Tmp.first;
4574
4575 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4576 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4577 return SDValue();
4578 }
4579
ExpandIntOp_ATOMIC_STORE(SDNode * N)4580 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
4581 SDLoc dl(N);
4582 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4583 cast<AtomicSDNode>(N)->getMemoryVT(),
4584 N->getOperand(0),
4585 N->getOperand(1), N->getOperand(2),
4586 cast<AtomicSDNode>(N)->getMemOperand());
4587 return Swap.getValue(1);
4588 }
4589
4590
PromoteIntRes_EXTRACT_SUBVECTOR(SDNode * N)4591 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4592
4593 EVT OutVT = N->getValueType(0);
4594 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4595 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4596 EVT NOutVTElem = NOutVT.getVectorElementType();
4597
4598 SDLoc dl(N);
4599 SDValue BaseIdx = N->getOperand(1);
4600
4601 // TODO: We may be able to use this for types other than scalable
4602 // vectors and fix those tests that expect BUILD_VECTOR to be used
4603 if (OutVT.isScalableVector()) {
4604 SDValue InOp0 = N->getOperand(0);
4605 EVT InVT = InOp0.getValueType();
4606
4607 // Promote operands and see if this is handled by target lowering,
4608 // Otherwise, use the BUILD_VECTOR approach below
4609 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
4610 // Collect the (promoted) operands
4611 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
4612
4613 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
4614 assert(PromEltVT.bitsLE(NOutVTElem) &&
4615 "Promoted operand has an element type greater than result");
4616
4617 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
4618 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
4619 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
4620 }
4621 }
4622
4623 if (OutVT.isScalableVector())
4624 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
4625
4626 SDValue InOp0 = N->getOperand(0);
4627 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
4628 InOp0 = GetPromotedInteger(N->getOperand(0));
4629
4630 EVT InVT = InOp0.getValueType();
4631
4632 unsigned OutNumElems = OutVT.getVectorNumElements();
4633 SmallVector<SDValue, 8> Ops;
4634 Ops.reserve(OutNumElems);
4635 for (unsigned i = 0; i != OutNumElems; ++i) {
4636
4637 // Extract the element from the original vector.
4638 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4639 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4640 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4641 InVT.getVectorElementType(), N->getOperand(0), Index);
4642
4643 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4644 // Insert the converted element to the new vector.
4645 Ops.push_back(Op);
4646 }
4647
4648 return DAG.getBuildVector(NOutVT, dl, Ops);
4649 }
4650
4651
PromoteIntRes_VECTOR_SHUFFLE(SDNode * N)4652 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4653 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4654 EVT VT = N->getValueType(0);
4655 SDLoc dl(N);
4656
4657 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4658
4659 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4660 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4661 EVT OutVT = V0.getValueType();
4662
4663 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4664 }
4665
4666
PromoteIntRes_BUILD_VECTOR(SDNode * N)4667 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4668 EVT OutVT = N->getValueType(0);
4669 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4670 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4671 unsigned NumElems = N->getNumOperands();
4672 EVT NOutVTElem = NOutVT.getVectorElementType();
4673
4674 SDLoc dl(N);
4675
4676 SmallVector<SDValue, 8> Ops;
4677 Ops.reserve(NumElems);
4678 for (unsigned i = 0; i != NumElems; ++i) {
4679 SDValue Op;
4680 // BUILD_VECTOR integer operand types are allowed to be larger than the
4681 // result's element type. This may still be true after the promotion. For
4682 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4683 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4684 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4685 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4686 else
4687 Op = N->getOperand(i);
4688 Ops.push_back(Op);
4689 }
4690
4691 return DAG.getBuildVector(NOutVT, dl, Ops);
4692 }
4693
PromoteIntRes_SCALAR_TO_VECTOR(SDNode * N)4694 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4695
4696 SDLoc dl(N);
4697
4698 assert(!N->getOperand(0).getValueType().isVector() &&
4699 "Input must be a scalar");
4700
4701 EVT OutVT = N->getValueType(0);
4702 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4703 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4704 EVT NOutVTElem = NOutVT.getVectorElementType();
4705
4706 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4707
4708 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4709 }
4710
PromoteIntRes_SPLAT_VECTOR(SDNode * N)4711 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
4712 SDLoc dl(N);
4713
4714 SDValue SplatVal = N->getOperand(0);
4715
4716 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar");
4717
4718 EVT OutVT = N->getValueType(0);
4719 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4720 assert(NOutVT.isVector() && "Type must be promoted to a vector type");
4721 EVT NOutElemVT = NOutVT.getVectorElementType();
4722
4723 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
4724
4725 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
4726 }
4727
PromoteIntRes_CONCAT_VECTORS(SDNode * N)4728 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4729 SDLoc dl(N);
4730
4731 EVT OutVT = N->getValueType(0);
4732 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4733 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4734
4735 EVT OutElemTy = NOutVT.getVectorElementType();
4736
4737 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4738 unsigned NumOutElem = NOutVT.getVectorNumElements();
4739 unsigned NumOperands = N->getNumOperands();
4740 assert(NumElem * NumOperands == NumOutElem &&
4741 "Unexpected number of elements");
4742
4743 // Take the elements from the first vector.
4744 SmallVector<SDValue, 8> Ops(NumOutElem);
4745 for (unsigned i = 0; i < NumOperands; ++i) {
4746 SDValue Op = N->getOperand(i);
4747 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4748 Op = GetPromotedInteger(Op);
4749 EVT SclrTy = Op.getValueType().getVectorElementType();
4750 assert(NumElem == Op.getValueType().getVectorNumElements() &&
4751 "Unexpected number of elements");
4752
4753 for (unsigned j = 0; j < NumElem; ++j) {
4754 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4755 DAG.getVectorIdxConstant(j, dl));
4756 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4757 }
4758 }
4759
4760 return DAG.getBuildVector(NOutVT, dl, Ops);
4761 }
4762
PromoteIntRes_EXTEND_VECTOR_INREG(SDNode * N)4763 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4764 EVT VT = N->getValueType(0);
4765 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4766 assert(NVT.isVector() && "This type must be promoted to a vector type");
4767
4768 SDLoc dl(N);
4769
4770 // For operands whose TypeAction is to promote, extend the promoted node
4771 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4772 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4773 // type..
4774 if (getTypeAction(N->getOperand(0).getValueType())
4775 == TargetLowering::TypePromoteInteger) {
4776 SDValue Promoted;
4777
4778 switch(N->getOpcode()) {
4779 case ISD::SIGN_EXTEND_VECTOR_INREG:
4780 Promoted = SExtPromotedInteger(N->getOperand(0));
4781 break;
4782 case ISD::ZERO_EXTEND_VECTOR_INREG:
4783 Promoted = ZExtPromotedInteger(N->getOperand(0));
4784 break;
4785 case ISD::ANY_EXTEND_VECTOR_INREG:
4786 Promoted = GetPromotedInteger(N->getOperand(0));
4787 break;
4788 default:
4789 llvm_unreachable("Node has unexpected Opcode");
4790 }
4791 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4792 }
4793
4794 // Directly extend to the appropriate transform-to type.
4795 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4796 }
4797
PromoteIntRes_INSERT_VECTOR_ELT(SDNode * N)4798 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4799 EVT OutVT = N->getValueType(0);
4800 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4801 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4802
4803 EVT NOutVTElem = NOutVT.getVectorElementType();
4804
4805 SDLoc dl(N);
4806 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4807
4808 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4809 NOutVTElem, N->getOperand(1));
4810 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4811 V0, ConvElem, N->getOperand(2));
4812 }
4813
PromoteIntRes_VECREDUCE(SDNode * N)4814 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4815 // The VECREDUCE result size may be larger than the element size, so
4816 // we can simply change the result type.
4817 SDLoc dl(N);
4818 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4819 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4820 }
4821
PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode * N)4822 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4823 SDLoc dl(N);
4824 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4825 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4826 TLI.getVectorIdxTy(DAG.getDataLayout()));
4827 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4828 V0->getValueType(0).getScalarType(), V0, V1);
4829
4830 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4831 // element types. If this is the case then we need to expand the outgoing
4832 // value and not truncate it.
4833 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4834 }
4835
PromoteIntOp_EXTRACT_SUBVECTOR(SDNode * N)4836 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4837 SDLoc dl(N);
4838 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4839 MVT InVT = V0.getValueType().getSimpleVT();
4840 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4841 N->getValueType(0).getVectorNumElements());
4842 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4843 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4844 }
4845
PromoteIntOp_CONCAT_VECTORS(SDNode * N)4846 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4847 SDLoc dl(N);
4848
4849 EVT ResVT = N->getValueType(0);
4850 unsigned NumElems = N->getNumOperands();
4851
4852 if (ResVT.isScalableVector()) {
4853 SDValue ResVec = DAG.getUNDEF(ResVT);
4854
4855 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
4856 SDValue Op = N->getOperand(OpIdx);
4857 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
4858 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
4859 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
4860 }
4861
4862 return ResVec;
4863 }
4864
4865 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4866
4867 SmallVector<SDValue, 8> NewOps;
4868 NewOps.reserve(NumElems);
4869
4870 // For each incoming vector
4871 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
4872 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4873 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4874 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4875
4876 for (unsigned i=0; i<NumElem; ++i) {
4877 // Extract element from incoming vector
4878 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4879 DAG.getVectorIdxConstant(i, dl));
4880 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4881 NewOps.push_back(Tr);
4882 }
4883 }
4884
4885 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4886 }
4887