1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include "llvm/Support/KnownBits.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
PromoteIntegerResult(SDNode * N,unsigned ResNo)37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39 dbgs() << "\n");
40 SDValue Res = SDValue();
41
42 // See if the target wants to custom expand this node.
43 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45 return;
46 }
47
48 switch (N->getOpcode()) {
49 default:
50 #ifndef NDEBUG
51 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52 N->dump(&DAG); dbgs() << "\n";
53 #endif
54 llvm_unreachable("Do not know how to promote this operator!");
55 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
63 case ISD::CTLZ_ZERO_UNDEF:
64 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65 case ISD::PARITY:
66 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
67 case ISD::CTTZ_ZERO_UNDEF:
68 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69 case ISD::EXTRACT_VECTOR_ELT:
70 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
71 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
72 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
73 break;
74 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
75 break;
76 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
77 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
78 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
79 case ISD::STRICT_FSETCC:
80 case ISD::STRICT_FSETCCS:
81 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
82 case ISD::SMIN:
83 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
84 case ISD::UMIN:
85 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
86
87 case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
88 case ISD::SIGN_EXTEND_INREG:
89 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
90 case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
91 case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
92 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
93 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
94 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
95 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
96
97 case ISD::EXTRACT_SUBVECTOR:
98 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
99 case ISD::VECTOR_REVERSE:
100 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
101 case ISD::VECTOR_SHUFFLE:
102 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
103 case ISD::VECTOR_SPLICE:
104 Res = PromoteIntRes_VECTOR_SPLICE(N); break;
105 case ISD::INSERT_VECTOR_ELT:
106 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
107 case ISD::BUILD_VECTOR:
108 Res = PromoteIntRes_BUILD_VECTOR(N); break;
109 case ISD::SCALAR_TO_VECTOR:
110 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
111 case ISD::SPLAT_VECTOR:
112 Res = PromoteIntRes_SPLAT_VECTOR(N); break;
113 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
114 case ISD::CONCAT_VECTORS:
115 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
116
117 case ISD::ANY_EXTEND_VECTOR_INREG:
118 case ISD::SIGN_EXTEND_VECTOR_INREG:
119 case ISD::ZERO_EXTEND_VECTOR_INREG:
120 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
121
122 case ISD::SIGN_EXTEND:
123 case ISD::ZERO_EXTEND:
124 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
125
126 case ISD::STRICT_FP_TO_SINT:
127 case ISD::STRICT_FP_TO_UINT:
128 case ISD::FP_TO_SINT:
129 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
130
131 case ISD::FP_TO_SINT_SAT:
132 case ISD::FP_TO_UINT_SAT:
133 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
134
135 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
136
137 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
138
139 case ISD::AND:
140 case ISD::OR:
141 case ISD::XOR:
142 case ISD::ADD:
143 case ISD::SUB:
144 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
145
146 case ISD::SDIV:
147 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
148
149 case ISD::UDIV:
150 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
151
152 case ISD::SADDO:
153 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
154 case ISD::UADDO:
155 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
156 case ISD::SMULO:
157 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
158
159 case ISD::ADDE:
160 case ISD::SUBE:
161 case ISD::ADDCARRY:
162 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
163
164 case ISD::SADDO_CARRY:
165 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
166
167 case ISD::SADDSAT:
168 case ISD::UADDSAT:
169 case ISD::SSUBSAT:
170 case ISD::USUBSAT:
171 case ISD::SSHLSAT:
172 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break;
173
174 case ISD::SMULFIX:
175 case ISD::SMULFIXSAT:
176 case ISD::UMULFIX:
177 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
178
179 case ISD::SDIVFIX:
180 case ISD::SDIVFIXSAT:
181 case ISD::UDIVFIX:
182 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
183
184 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
185
186 case ISD::ATOMIC_LOAD:
187 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
188
189 case ISD::ATOMIC_LOAD_ADD:
190 case ISD::ATOMIC_LOAD_SUB:
191 case ISD::ATOMIC_LOAD_AND:
192 case ISD::ATOMIC_LOAD_CLR:
193 case ISD::ATOMIC_LOAD_OR:
194 case ISD::ATOMIC_LOAD_XOR:
195 case ISD::ATOMIC_LOAD_NAND:
196 case ISD::ATOMIC_LOAD_MIN:
197 case ISD::ATOMIC_LOAD_MAX:
198 case ISD::ATOMIC_LOAD_UMIN:
199 case ISD::ATOMIC_LOAD_UMAX:
200 case ISD::ATOMIC_SWAP:
201 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
202
203 case ISD::ATOMIC_CMP_SWAP:
204 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
205 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
206 break;
207
208 case ISD::VECREDUCE_ADD:
209 case ISD::VECREDUCE_MUL:
210 case ISD::VECREDUCE_AND:
211 case ISD::VECREDUCE_OR:
212 case ISD::VECREDUCE_XOR:
213 case ISD::VECREDUCE_SMAX:
214 case ISD::VECREDUCE_SMIN:
215 case ISD::VECREDUCE_UMAX:
216 case ISD::VECREDUCE_UMIN:
217 Res = PromoteIntRes_VECREDUCE(N);
218 break;
219
220 case ISD::FREEZE:
221 Res = PromoteIntRes_FREEZE(N);
222 break;
223
224 case ISD::ROTL:
225 case ISD::ROTR:
226 Res = PromoteIntRes_Rotate(N);
227 break;
228
229 case ISD::FSHL:
230 case ISD::FSHR:
231 Res = PromoteIntRes_FunnelShift(N);
232 break;
233 }
234
235 // If the result is null then the sub-method took care of registering it.
236 if (Res.getNode())
237 SetPromotedInteger(SDValue(N, ResNo), Res);
238 }
239
PromoteIntRes_MERGE_VALUES(SDNode * N,unsigned ResNo)240 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
241 unsigned ResNo) {
242 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
243 return GetPromotedInteger(Op);
244 }
245
PromoteIntRes_AssertSext(SDNode * N)246 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
247 // Sign-extend the new bits, and continue the assertion.
248 SDValue Op = SExtPromotedInteger(N->getOperand(0));
249 return DAG.getNode(ISD::AssertSext, SDLoc(N),
250 Op.getValueType(), Op, N->getOperand(1));
251 }
252
PromoteIntRes_AssertZext(SDNode * N)253 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
254 // Zero the new bits, and continue the assertion.
255 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
256 return DAG.getNode(ISD::AssertZext, SDLoc(N),
257 Op.getValueType(), Op, N->getOperand(1));
258 }
259
PromoteIntRes_Atomic0(AtomicSDNode * N)260 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
261 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
262 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
263 N->getMemoryVT(), ResVT,
264 N->getChain(), N->getBasePtr(),
265 N->getMemOperand());
266 // Legalize the chain result - switch anything that used the old chain to
267 // use the new one.
268 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
269 return Res;
270 }
271
PromoteIntRes_Atomic1(AtomicSDNode * N)272 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
273 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
274 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
275 N->getMemoryVT(),
276 N->getChain(), N->getBasePtr(),
277 Op2, N->getMemOperand());
278 // Legalize the chain result - switch anything that used the old chain to
279 // use the new one.
280 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
281 return Res;
282 }
283
PromoteIntRes_AtomicCmpSwap(AtomicSDNode * N,unsigned ResNo)284 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
285 unsigned ResNo) {
286 if (ResNo == 1) {
287 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
288 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
289 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
290
291 // Only use the result of getSetCCResultType if it is legal,
292 // otherwise just use the promoted result type (NVT).
293 if (!TLI.isTypeLegal(SVT))
294 SVT = NVT;
295
296 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
297 SDValue Res = DAG.getAtomicCmpSwap(
298 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
299 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
300 N->getMemOperand());
301 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
302 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
303 return Res.getValue(1);
304 }
305
306 // Op2 is used for the comparison and thus must be extended according to the
307 // target's atomic operations. Op3 is merely stored and so can be left alone.
308 SDValue Op2 = N->getOperand(2);
309 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
310 switch (TLI.getExtendForAtomicCmpSwapArg()) {
311 case ISD::SIGN_EXTEND:
312 Op2 = SExtPromotedInteger(Op2);
313 break;
314 case ISD::ZERO_EXTEND:
315 Op2 = ZExtPromotedInteger(Op2);
316 break;
317 case ISD::ANY_EXTEND:
318 Op2 = GetPromotedInteger(Op2);
319 break;
320 default:
321 llvm_unreachable("Invalid atomic op extension");
322 }
323
324 SDVTList VTs =
325 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
326 SDValue Res = DAG.getAtomicCmpSwap(
327 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
328 N->getBasePtr(), Op2, Op3, N->getMemOperand());
329 // Update the use to N with the newly created Res.
330 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
331 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
332 return Res;
333 }
334
PromoteIntRes_BITCAST(SDNode * N)335 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
336 SDValue InOp = N->getOperand(0);
337 EVT InVT = InOp.getValueType();
338 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
339 EVT OutVT = N->getValueType(0);
340 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
341 SDLoc dl(N);
342
343 switch (getTypeAction(InVT)) {
344 case TargetLowering::TypeLegal:
345 break;
346 case TargetLowering::TypePromoteInteger:
347 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
348 // The input promotes to the same size. Convert the promoted value.
349 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
350 break;
351 case TargetLowering::TypeSoftenFloat:
352 // Promote the integer operand by hand.
353 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
354 case TargetLowering::TypeSoftPromoteHalf:
355 // Promote the integer operand by hand.
356 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
357 case TargetLowering::TypePromoteFloat: {
358 // Convert the promoted float by hand.
359 if (!NOutVT.isVector())
360 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
361 break;
362 }
363 case TargetLowering::TypeExpandInteger:
364 case TargetLowering::TypeExpandFloat:
365 break;
366 case TargetLowering::TypeScalarizeVector:
367 // Convert the element to an integer and promote it by hand.
368 if (!NOutVT.isVector())
369 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
370 BitConvertToInteger(GetScalarizedVector(InOp)));
371 break;
372 case TargetLowering::TypeScalarizeScalableVector:
373 report_fatal_error("Scalarization of scalable vectors is not supported.");
374 case TargetLowering::TypeSplitVector: {
375 if (!NOutVT.isVector()) {
376 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
377 // pieces of the input into integers and reassemble in the final type.
378 SDValue Lo, Hi;
379 GetSplitVector(N->getOperand(0), Lo, Hi);
380 Lo = BitConvertToInteger(Lo);
381 Hi = BitConvertToInteger(Hi);
382
383 if (DAG.getDataLayout().isBigEndian())
384 std::swap(Lo, Hi);
385
386 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
387 EVT::getIntegerVT(*DAG.getContext(),
388 NOutVT.getSizeInBits()),
389 JoinIntegers(Lo, Hi));
390 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
391 }
392 break;
393 }
394 case TargetLowering::TypeWidenVector:
395 // The input is widened to the same size. Convert to the widened value.
396 // Make sure that the outgoing value is not a vector, because this would
397 // make us bitcast between two vectors which are legalized in different ways.
398 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
399 SDValue Res =
400 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
401
402 // For big endian targets we need to shift the casted value or the
403 // interesting bits will end up at the wrong place.
404 if (DAG.getDataLayout().isBigEndian()) {
405 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
406 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout());
407 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
408 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
409 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
410 }
411 return Res;
412 }
413 // If the output type is also a vector and widening it to the same size
414 // as the widened input type would be a legal type, we can widen the bitcast
415 // and handle the promotion after.
416 if (NOutVT.isVector()) {
417 unsigned WidenInSize = NInVT.getSizeInBits();
418 unsigned OutSize = OutVT.getSizeInBits();
419 if (WidenInSize % OutSize == 0) {
420 unsigned Scale = WidenInSize / OutSize;
421 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
422 OutVT.getVectorElementType(),
423 OutVT.getVectorNumElements() * Scale);
424 if (isTypeLegal(WideOutVT)) {
425 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
426 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
427 DAG.getVectorIdxConstant(0, dl));
428 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
429 }
430 }
431 }
432 }
433
434 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
435 CreateStackStoreLoad(InOp, OutVT));
436 }
437
438 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount
439 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
getShiftAmountTyForConstant(EVT VT,const TargetLowering & TLI,SelectionDAG & DAG)440 static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI,
441 SelectionDAG &DAG) {
442 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
443 // If any possible shift value won't fit in the prefered type, just use
444 // something safe. It will be legalized when the shift is expanded.
445 if (!ShiftVT.isVector() &&
446 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits()))
447 ShiftVT = MVT::i32;
448 return ShiftVT;
449 }
450
PromoteIntRes_FREEZE(SDNode * N)451 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
452 SDValue V = GetPromotedInteger(N->getOperand(0));
453 return DAG.getNode(ISD::FREEZE, SDLoc(N),
454 V.getValueType(), V);
455 }
456
PromoteIntRes_BSWAP(SDNode * N)457 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
458 SDValue Op = GetPromotedInteger(N->getOperand(0));
459 EVT OVT = N->getValueType(0);
460 EVT NVT = Op.getValueType();
461 SDLoc dl(N);
462
463 // If the larger BSWAP isn't supported by the target, try to expand now.
464 // If we expand later we'll end up with more operations since we lost the
465 // original type. We only do this for scalars since we have a shuffle
466 // based lowering for vectors in LegalizeVectorOps.
467 if (!OVT.isVector() && !TLI.isOperationLegalOrCustom(ISD::BSWAP, NVT)) {
468 if (SDValue Res = TLI.expandBSWAP(N, DAG))
469 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
470 }
471
472 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
473 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
474 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
475 DAG.getConstant(DiffBits, dl, ShiftVT));
476 }
477
PromoteIntRes_BITREVERSE(SDNode * N)478 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
479 SDValue Op = GetPromotedInteger(N->getOperand(0));
480 EVT OVT = N->getValueType(0);
481 EVT NVT = Op.getValueType();
482 SDLoc dl(N);
483
484 // If the larger BITREVERSE isn't supported by the target, try to expand now.
485 // If we expand later we'll end up with more operations since we lost the
486 // original type. We only do this for scalars since we have a shuffle
487 // based lowering for vectors in LegalizeVectorOps.
488 if (!OVT.isVector() && OVT.isSimple() &&
489 !TLI.isOperationLegalOrCustom(ISD::BITREVERSE, NVT)) {
490 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
491 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
492 }
493
494 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
495 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
496 return DAG.getNode(ISD::SRL, dl, NVT,
497 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
498 DAG.getConstant(DiffBits, dl, ShiftVT));
499 }
500
PromoteIntRes_BUILD_PAIR(SDNode * N)501 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
502 // The pair element type may be legal, or may not promote to the same type as
503 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
504 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
505 TLI.getTypeToTransformTo(*DAG.getContext(),
506 N->getValueType(0)), JoinIntegers(N->getOperand(0),
507 N->getOperand(1)));
508 }
509
PromoteIntRes_Constant(SDNode * N)510 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
511 EVT VT = N->getValueType(0);
512 // FIXME there is no actual debug info here
513 SDLoc dl(N);
514 // Zero extend things like i1, sign extend everything else. It shouldn't
515 // matter in theory which one we pick, but this tends to give better code?
516 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
517 SDValue Result = DAG.getNode(Opc, dl,
518 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
519 SDValue(N, 0));
520 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
521 return Result;
522 }
523
PromoteIntRes_CTLZ(SDNode * N)524 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
525 // Zero extend to the promoted type and do the count there.
526 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
527 SDLoc dl(N);
528 EVT OVT = N->getValueType(0);
529 EVT NVT = Op.getValueType();
530 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
531 // Subtract off the extra leading bits in the bigger type.
532 return DAG.getNode(
533 ISD::SUB, dl, NVT, Op,
534 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
535 NVT));
536 }
537
PromoteIntRes_CTPOP_PARITY(SDNode * N)538 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
539 // Zero extend to the promoted type and do the count or parity there.
540 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
541 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
542 }
543
PromoteIntRes_CTTZ(SDNode * N)544 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
545 SDValue Op = GetPromotedInteger(N->getOperand(0));
546 EVT OVT = N->getValueType(0);
547 EVT NVT = Op.getValueType();
548 SDLoc dl(N);
549 if (N->getOpcode() == ISD::CTTZ) {
550 // The count is the same in the promoted type except if the original
551 // value was zero. This can be handled by setting the bit just off
552 // the top of the original type.
553 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
554 OVT.getScalarSizeInBits());
555 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
556 }
557 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
558 }
559
PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode * N)560 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
561 SDLoc dl(N);
562 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
563
564 SDValue Op0 = N->getOperand(0);
565 SDValue Op1 = N->getOperand(1);
566
567 // If the input also needs to be promoted, do that first so we can get a
568 // get a good idea for the output type.
569 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
570 == TargetLowering::TypePromoteInteger) {
571 SDValue In = GetPromotedInteger(Op0);
572
573 // If the new type is larger than NVT, use it. We probably won't need to
574 // promote it again.
575 EVT SVT = In.getValueType().getScalarType();
576 if (SVT.bitsGE(NVT)) {
577 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
578 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
579 }
580 }
581
582 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
583 }
584
PromoteIntRes_FP_TO_XINT(SDNode * N)585 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
587 unsigned NewOpc = N->getOpcode();
588 SDLoc dl(N);
589
590 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
591 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
592 // and SINT conversions are Custom, there is no way to tell which is
593 // preferable. We choose SINT because that's the right thing on PPC.)
594 if (N->getOpcode() == ISD::FP_TO_UINT &&
595 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
596 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
597 NewOpc = ISD::FP_TO_SINT;
598
599 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
600 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
601 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
602 NewOpc = ISD::STRICT_FP_TO_SINT;
603
604 SDValue Res;
605 if (N->isStrictFPOpcode()) {
606 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
607 {N->getOperand(0), N->getOperand(1)});
608 // Legalize the chain result - switch anything that used the old chain to
609 // use the new one.
610 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
611 } else
612 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
613
614 // Assert that the converted value fits in the original type. If it doesn't
615 // (eg: because the value being converted is too big), then the result of the
616 // original operation was undefined anyway, so the assert is still correct.
617 //
618 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
619 // before legalization: fp-to-uint16, 65534. -> 0xfffe
620 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
621 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
622 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ?
623 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
624 DAG.getValueType(N->getValueType(0).getScalarType()));
625 }
626
PromoteIntRes_FP_TO_XINT_SAT(SDNode * N)627 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
628 // Promote the result type, while keeping the original width in Op1.
629 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
630 SDLoc dl(N);
631 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
632 N->getOperand(1));
633 }
634
PromoteIntRes_FP_TO_FP16(SDNode * N)635 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
636 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
637 SDLoc dl(N);
638
639 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
640 }
641
PromoteIntRes_FLT_ROUNDS(SDNode * N)642 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
643 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
644 SDLoc dl(N);
645
646 SDValue Res =
647 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
648
649 // Legalize the chain result - switch anything that used the old chain to
650 // use the new one.
651 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
652 return Res;
653 }
654
PromoteIntRes_INT_EXTEND(SDNode * N)655 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
656 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
657 SDLoc dl(N);
658
659 if (getTypeAction(N->getOperand(0).getValueType())
660 == TargetLowering::TypePromoteInteger) {
661 SDValue Res = GetPromotedInteger(N->getOperand(0));
662 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
663
664 // If the result and operand types are the same after promotion, simplify
665 // to an in-register extension.
666 if (NVT == Res.getValueType()) {
667 // The high bits are not guaranteed to be anything. Insert an extend.
668 if (N->getOpcode() == ISD::SIGN_EXTEND)
669 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
670 DAG.getValueType(N->getOperand(0).getValueType()));
671 if (N->getOpcode() == ISD::ZERO_EXTEND)
672 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
673 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
674 return Res;
675 }
676 }
677
678 // Otherwise, just extend the original operand all the way to the larger type.
679 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
680 }
681
PromoteIntRes_LOAD(LoadSDNode * N)682 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
683 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
684 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
685 ISD::LoadExtType ExtType =
686 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
687 SDLoc dl(N);
688 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
689 N->getMemoryVT(), N->getMemOperand());
690
691 // Legalize the chain result - switch anything that used the old chain to
692 // use the new one.
693 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
694 return Res;
695 }
696
PromoteIntRes_MLOAD(MaskedLoadSDNode * N)697 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
698 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
699 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
700
701 SDLoc dl(N);
702 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
703 N->getOffset(), N->getMask(), ExtPassThru,
704 N->getMemoryVT(), N->getMemOperand(),
705 N->getAddressingMode(), ISD::EXTLOAD);
706 // Legalize the chain result - switch anything that used the old chain to
707 // use the new one.
708 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
709 return Res;
710 }
711
PromoteIntRes_MGATHER(MaskedGatherSDNode * N)712 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
713 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
714 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
715 assert(NVT == ExtPassThru.getValueType() &&
716 "Gather result type and the passThru argument type should be the same");
717
718 ISD::LoadExtType ExtType = N->getExtensionType();
719 if (ExtType == ISD::NON_EXTLOAD)
720 ExtType = ISD::EXTLOAD;
721
722 SDLoc dl(N);
723 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
724 N->getIndex(), N->getScale() };
725 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
726 N->getMemoryVT(), dl, Ops,
727 N->getMemOperand(), N->getIndexType(),
728 ExtType);
729 // Legalize the chain result - switch anything that used the old chain to
730 // use the new one.
731 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
732 return Res;
733 }
734
735 /// Promote the overflow flag of an overflowing arithmetic node.
PromoteIntRes_Overflow(SDNode * N)736 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
737 // Change the return type of the boolean result while obeying
738 // getSetCCResultType.
739 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
740 EVT VT = N->getValueType(0);
741 EVT SVT = getSetCCResultType(VT);
742 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
743 unsigned NumOps = N->getNumOperands();
744 assert(NumOps <= 3 && "Too many operands");
745 if (NumOps == 3)
746 Ops[2] = N->getOperand(2);
747
748 SDLoc dl(N);
749 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
750 makeArrayRef(Ops, NumOps));
751
752 // Modified the sum result - switch anything that used the old sum to use
753 // the new one.
754 ReplaceValueWith(SDValue(N, 0), Res);
755
756 // Convert to the expected type.
757 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
758 }
759
PromoteIntRes_ADDSUBSHLSAT(SDNode * N)760 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
761 // If the promoted type is legal, we can convert this to:
762 // 1. ANY_EXTEND iN to iM
763 // 2. SHL by M-N
764 // 3. [US][ADD|SUB|SHL]SAT
765 // 4. L/ASHR by M-N
766 // Else it is more efficient to convert this to a min and a max
767 // operation in the higher precision arithmetic.
768 SDLoc dl(N);
769 SDValue Op1 = N->getOperand(0);
770 SDValue Op2 = N->getOperand(1);
771 unsigned OldBits = Op1.getScalarValueSizeInBits();
772
773 unsigned Opcode = N->getOpcode();
774 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
775
776 SDValue Op1Promoted, Op2Promoted;
777 if (IsShift) {
778 Op1Promoted = GetPromotedInteger(Op1);
779 Op2Promoted = ZExtPromotedInteger(Op2);
780 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
781 Op1Promoted = ZExtPromotedInteger(Op1);
782 Op2Promoted = ZExtPromotedInteger(Op2);
783 } else {
784 Op1Promoted = SExtPromotedInteger(Op1);
785 Op2Promoted = SExtPromotedInteger(Op2);
786 }
787 EVT PromotedType = Op1Promoted.getValueType();
788 unsigned NewBits = PromotedType.getScalarSizeInBits();
789
790 if (Opcode == ISD::UADDSAT) {
791 APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits);
792 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
793 SDValue Add =
794 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
795 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
796 }
797
798 // USUBSAT can always be promoted as long as we have zero-extended the args.
799 if (Opcode == ISD::USUBSAT)
800 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
801 Op2Promoted);
802
803 // Shift cannot use a min/max expansion, we can't detect overflow if all of
804 // the bits have been shifted out.
805 if (IsShift || TLI.isOperationLegalOrCustom(Opcode, PromotedType)) {
806 unsigned ShiftOp;
807 switch (Opcode) {
808 case ISD::SADDSAT:
809 case ISD::SSUBSAT:
810 case ISD::SSHLSAT:
811 ShiftOp = ISD::SRA;
812 break;
813 case ISD::USHLSAT:
814 ShiftOp = ISD::SRL;
815 break;
816 default:
817 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
818 "addition, subtraction or left shift");
819 }
820
821 unsigned SHLAmount = NewBits - OldBits;
822 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
823 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
824 Op1Promoted =
825 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
826 if (!IsShift)
827 Op2Promoted =
828 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
829
830 SDValue Result =
831 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
832 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
833 }
834
835 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
836 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
837 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
838 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
839 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
840 SDValue Result =
841 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
842 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
843 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
844 return Result;
845 }
846
PromoteIntRes_MULFIX(SDNode * N)847 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
848 // Can just promote the operands then continue with operation.
849 SDLoc dl(N);
850 SDValue Op1Promoted, Op2Promoted;
851 bool Signed =
852 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
853 bool Saturating =
854 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
855 if (Signed) {
856 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
857 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
858 } else {
859 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
860 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
861 }
862 EVT OldType = N->getOperand(0).getValueType();
863 EVT PromotedType = Op1Promoted.getValueType();
864 unsigned DiffSize =
865 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
866
867 if (Saturating) {
868 // Promoting the operand and result values changes the saturation width,
869 // which is extends the values that we clamp to on saturation. This could be
870 // resolved by shifting one of the operands the same amount, which would
871 // also shift the result we compare against, then shifting back.
872 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
873 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
874 DAG.getConstant(DiffSize, dl, ShiftTy));
875 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
876 Op2Promoted, N->getOperand(2));
877 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
878 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
879 DAG.getConstant(DiffSize, dl, ShiftTy));
880 }
881 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
882 N->getOperand(2));
883 }
884
SaturateWidenedDIVFIX(SDValue V,SDLoc & dl,unsigned SatW,bool Signed,const TargetLowering & TLI,SelectionDAG & DAG)885 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
886 unsigned SatW, bool Signed,
887 const TargetLowering &TLI,
888 SelectionDAG &DAG) {
889 EVT VT = V.getValueType();
890 unsigned VTW = VT.getScalarSizeInBits();
891
892 if (!Signed) {
893 // Saturate to the unsigned maximum by getting the minimum of V and the
894 // maximum.
895 return DAG.getNode(ISD::UMIN, dl, VT, V,
896 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
897 dl, VT));
898 }
899
900 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
901 // signed minimum of it and V.
902 V = DAG.getNode(ISD::SMIN, dl, VT, V,
903 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
904 dl, VT));
905 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
906 // signed maximum of it and V.
907 V = DAG.getNode(ISD::SMAX, dl, VT, V,
908 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
909 dl, VT));
910 return V;
911 }
912
earlyExpandDIVFIX(SDNode * N,SDValue LHS,SDValue RHS,unsigned Scale,const TargetLowering & TLI,SelectionDAG & DAG,unsigned SatW=0)913 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
914 unsigned Scale, const TargetLowering &TLI,
915 SelectionDAG &DAG, unsigned SatW = 0) {
916 EVT VT = LHS.getValueType();
917 unsigned VTSize = VT.getScalarSizeInBits();
918 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
919 N->getOpcode() == ISD::SDIVFIXSAT;
920 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
921 N->getOpcode() == ISD::UDIVFIXSAT;
922
923 SDLoc dl(N);
924 // Widen the types by a factor of two. This is guaranteed to expand, since it
925 // will always have enough high bits in the LHS to shift into.
926 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
927 if (VT.isVector())
928 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
929 VT.getVectorElementCount());
930 if (Signed) {
931 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
932 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
933 } else {
934 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
935 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
936 }
937
938 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
939 DAG);
940 assert(Res && "Expanding DIVFIX with wide type failed?");
941 if (Saturating) {
942 // If the caller has told us to saturate at something less, use that width
943 // instead of the type before doubling. However, it cannot be more than
944 // what we just widened!
945 assert(SatW <= VTSize &&
946 "Tried to saturate to more than the original type?");
947 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
948 TLI, DAG);
949 }
950 return DAG.getZExtOrTrunc(Res, dl, VT);
951 }
952
PromoteIntRes_DIVFIX(SDNode * N)953 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
954 SDLoc dl(N);
955 SDValue Op1Promoted, Op2Promoted;
956 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
957 N->getOpcode() == ISD::SDIVFIXSAT;
958 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
959 N->getOpcode() == ISD::UDIVFIXSAT;
960 if (Signed) {
961 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
962 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
963 } else {
964 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
965 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
966 }
967 EVT PromotedType = Op1Promoted.getValueType();
968 unsigned Scale = N->getConstantOperandVal(2);
969
970 // If the type is already legal and the operation is legal in that type, we
971 // should not early expand.
972 if (TLI.isTypeLegal(PromotedType)) {
973 TargetLowering::LegalizeAction Action =
974 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
975 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
976 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
977 unsigned Diff = PromotedType.getScalarSizeInBits() -
978 N->getValueType(0).getScalarSizeInBits();
979 if (Saturating)
980 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
981 DAG.getConstant(Diff, dl, ShiftTy));
982 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
983 Op2Promoted, N->getOperand(2));
984 if (Saturating)
985 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
986 DAG.getConstant(Diff, dl, ShiftTy));
987 return Res;
988 }
989 }
990
991 // See if we can perform the division in this type without expanding.
992 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
993 Op2Promoted, Scale, DAG)) {
994 if (Saturating)
995 Res = SaturateWidenedDIVFIX(Res, dl,
996 N->getValueType(0).getScalarSizeInBits(),
997 Signed, TLI, DAG);
998 return Res;
999 }
1000 // If we cannot, expand it to twice the type width. If we are saturating, give
1001 // it the original width as a saturating width so we don't need to emit
1002 // two saturations.
1003 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1004 N->getValueType(0).getScalarSizeInBits());
1005 }
1006
PromoteIntRes_SADDSUBO(SDNode * N,unsigned ResNo)1007 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1008 if (ResNo == 1)
1009 return PromoteIntRes_Overflow(N);
1010
1011 // The operation overflowed iff the result in the larger type is not the
1012 // sign extension of its truncation to the original type.
1013 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1014 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1015 EVT OVT = N->getOperand(0).getValueType();
1016 EVT NVT = LHS.getValueType();
1017 SDLoc dl(N);
1018
1019 // Do the arithmetic in the larger type.
1020 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1021 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1022
1023 // Calculate the overflow flag: sign extend the arithmetic result from
1024 // the original type.
1025 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1026 DAG.getValueType(OVT));
1027 // Overflowed if and only if this is not equal to Res.
1028 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1029
1030 // Use the calculated overflow everywhere.
1031 ReplaceValueWith(SDValue(N, 1), Ofl);
1032
1033 return Res;
1034 }
1035
PromoteIntRes_SELECT(SDNode * N)1036 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
1037 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1038 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1039 return DAG.getSelect(SDLoc(N),
1040 LHS.getValueType(), N->getOperand(0), LHS, RHS);
1041 }
1042
PromoteIntRes_VSELECT(SDNode * N)1043 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
1044 SDValue Mask = N->getOperand(0);
1045
1046 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1047 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1048 return DAG.getNode(ISD::VSELECT, SDLoc(N),
1049 LHS.getValueType(), Mask, LHS, RHS);
1050 }
1051
PromoteIntRes_SELECT_CC(SDNode * N)1052 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1053 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1054 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1055 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1056 LHS.getValueType(), N->getOperand(0),
1057 N->getOperand(1), LHS, RHS, N->getOperand(4));
1058 }
1059
PromoteIntRes_SETCC(SDNode * N)1060 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1061 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1062 EVT InVT = N->getOperand(OpNo).getValueType();
1063 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1064
1065 EVT SVT = getSetCCResultType(InVT);
1066
1067 // If we got back a type that needs to be promoted, this likely means the
1068 // the input type also needs to be promoted. So get the promoted type for
1069 // the input and try the query again.
1070 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1071 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1072 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1073 SVT = getSetCCResultType(InVT);
1074 } else {
1075 // Input type isn't promoted, just use the default promoted type.
1076 SVT = NVT;
1077 }
1078 }
1079
1080 SDLoc dl(N);
1081 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1082 "Vector compare must return a vector result!");
1083
1084 // Get the SETCC result using the canonical SETCC type.
1085 SDValue SetCC;
1086 if (N->isStrictFPOpcode()) {
1087 EVT VTs[] = {SVT, MVT::Other};
1088 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1089 N->getOperand(2), N->getOperand(3)};
1090 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
1091 // Legalize the chain result - switch anything that used the old chain to
1092 // use the new one.
1093 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1094 } else
1095 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1096 N->getOperand(1), N->getOperand(2));
1097
1098 // Convert to the expected type.
1099 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1100 }
1101
PromoteIntRes_SHL(SDNode * N)1102 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1103 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1104 SDValue RHS = N->getOperand(1);
1105 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1106 RHS = ZExtPromotedInteger(RHS);
1107 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
1108 }
1109
PromoteIntRes_SIGN_EXTEND_INREG(SDNode * N)1110 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1111 SDValue Op = GetPromotedInteger(N->getOperand(0));
1112 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1113 Op.getValueType(), Op, N->getOperand(1));
1114 }
1115
PromoteIntRes_SimpleIntBinOp(SDNode * N)1116 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1117 // The input may have strange things in the top bits of the registers, but
1118 // these operations don't care. They may have weird bits going out, but
1119 // that too is okay if they are integer operations.
1120 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1121 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1122 return DAG.getNode(N->getOpcode(), SDLoc(N),
1123 LHS.getValueType(), LHS, RHS);
1124 }
1125
PromoteIntRes_SExtIntBinOp(SDNode * N)1126 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1127 // Sign extend the input.
1128 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1129 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1130 return DAG.getNode(N->getOpcode(), SDLoc(N),
1131 LHS.getValueType(), LHS, RHS);
1132 }
1133
PromoteIntRes_ZExtIntBinOp(SDNode * N)1134 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1135 // Zero extend the input.
1136 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1137 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1138 return DAG.getNode(N->getOpcode(), SDLoc(N),
1139 LHS.getValueType(), LHS, RHS);
1140 }
1141
PromoteIntRes_UMINUMAX(SDNode * N)1142 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1143 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1144 // whatever is best for the target.
1145 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0));
1146 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1));
1147 return DAG.getNode(N->getOpcode(), SDLoc(N),
1148 LHS.getValueType(), LHS, RHS);
1149 }
1150
PromoteIntRes_SRA(SDNode * N)1151 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1152 // The input value must be properly sign extended.
1153 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1154 SDValue RHS = N->getOperand(1);
1155 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1156 RHS = ZExtPromotedInteger(RHS);
1157 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
1158 }
1159
PromoteIntRes_SRL(SDNode * N)1160 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1161 // The input value must be properly zero extended.
1162 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1163 SDValue RHS = N->getOperand(1);
1164 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1165 RHS = ZExtPromotedInteger(RHS);
1166 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
1167 }
1168
PromoteIntRes_Rotate(SDNode * N)1169 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1170 // Lower the rotate to shifts and ORs which can be promoted.
1171 SDValue Res;
1172 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
1173 ReplaceValueWith(SDValue(N, 0), Res);
1174 return SDValue();
1175 }
1176
PromoteIntRes_FunnelShift(SDNode * N)1177 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1178 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1179 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1180 SDValue Amount = GetPromotedInteger(N->getOperand(2));
1181
1182 SDLoc DL(N);
1183 EVT OldVT = N->getOperand(0).getValueType();
1184 EVT VT = Lo.getValueType();
1185 unsigned Opcode = N->getOpcode();
1186 bool IsFSHR = Opcode == ISD::FSHR;
1187 unsigned OldBits = OldVT.getScalarSizeInBits();
1188 unsigned NewBits = VT.getScalarSizeInBits();
1189
1190 // Amount has to be interpreted modulo the old bit width.
1191 Amount =
1192 DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT));
1193
1194 // If the promoted type is twice the size (or more), then we use the
1195 // traditional funnel 'double' shift codegen. This isn't necessary if the
1196 // shift amount is constant.
1197 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1198 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1199 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) &&
1200 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1201 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1202 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1203 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1204 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1205 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount);
1206 if (!IsFSHR)
1207 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1208 return Res;
1209 }
1210
1211 // Shift Lo up to occupy the upper bits of the promoted type.
1212 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT);
1213 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1214
1215 // Increase Amount to shift the result into the lower bits of the promoted
1216 // type.
1217 if (IsFSHR)
1218 Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset);
1219
1220 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount);
1221 }
1222
PromoteIntRes_TRUNCATE(SDNode * N)1223 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1224 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1225 SDValue Res;
1226 SDValue InOp = N->getOperand(0);
1227 SDLoc dl(N);
1228
1229 switch (getTypeAction(InOp.getValueType())) {
1230 default: llvm_unreachable("Unknown type action!");
1231 case TargetLowering::TypeLegal:
1232 case TargetLowering::TypeExpandInteger:
1233 Res = InOp;
1234 break;
1235 case TargetLowering::TypePromoteInteger:
1236 Res = GetPromotedInteger(InOp);
1237 break;
1238 case TargetLowering::TypeSplitVector: {
1239 EVT InVT = InOp.getValueType();
1240 assert(InVT.isVector() && "Cannot split scalar types");
1241 unsigned NumElts = InVT.getVectorNumElements();
1242 assert(NumElts == NVT.getVectorNumElements() &&
1243 "Dst and Src must have the same number of elements");
1244 assert(isPowerOf2_32(NumElts) &&
1245 "Promoted vector type must be a power of two");
1246
1247 SDValue EOp1, EOp2;
1248 GetSplitVector(InOp, EOp1, EOp2);
1249
1250 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1251 NumElts/2);
1252 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1253 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1254
1255 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1256 }
1257 case TargetLowering::TypeWidenVector: {
1258 SDValue WideInOp = GetWidenedVector(InOp);
1259
1260 // Truncate widened InOp.
1261 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1262 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1263 N->getValueType(0).getScalarType(), NumElem);
1264 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1265
1266 // Zero extend so that the elements are of same type as those of NVT
1267 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1268 NumElem);
1269 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1270
1271 // Extract the low NVT subvector.
1272 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1273 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1274 }
1275 }
1276
1277 // Truncate to NVT instead of VT
1278 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1279 }
1280
PromoteIntRes_UADDSUBO(SDNode * N,unsigned ResNo)1281 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1282 if (ResNo == 1)
1283 return PromoteIntRes_Overflow(N);
1284
1285 // The operation overflowed iff the result in the larger type is not the
1286 // zero extension of its truncation to the original type.
1287 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1288 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1289 EVT OVT = N->getOperand(0).getValueType();
1290 EVT NVT = LHS.getValueType();
1291 SDLoc dl(N);
1292
1293 // Do the arithmetic in the larger type.
1294 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1295 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1296
1297 // Calculate the overflow flag: zero extend the arithmetic result from
1298 // the original type.
1299 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1300 // Overflowed if and only if this is not equal to Res.
1301 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1302
1303 // Use the calculated overflow everywhere.
1304 ReplaceValueWith(SDValue(N, 1), Ofl);
1305
1306 return Res;
1307 }
1308
1309 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1310 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
1311 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
PromoteIntRes_ADDSUBCARRY(SDNode * N,unsigned ResNo)1312 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1313 if (ResNo == 1)
1314 return PromoteIntRes_Overflow(N);
1315
1316 // We need to sign-extend the operands so the carry value computed by the
1317 // wide operation will be equivalent to the carry value computed by the
1318 // narrow operation.
1319 // An ADDCARRY can generate carry only if any of the operands has its
1320 // most significant bit set. Sign extension propagates the most significant
1321 // bit into the higher bits which means the extra bit that the narrow
1322 // addition would need (i.e. the carry) will be propagated through the higher
1323 // bits of the wide addition.
1324 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1325 // preserved by sign extension.
1326 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1327 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1328
1329 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1330
1331 // Do the arithmetic in the wide type.
1332 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1333 LHS, RHS, N->getOperand(2));
1334
1335 // Update the users of the original carry/borrow value.
1336 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1337
1338 return SDValue(Res.getNode(), 0);
1339 }
1340
PromoteIntRes_SADDSUBO_CARRY(SDNode * N,unsigned ResNo)1341 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1342 unsigned ResNo) {
1343 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1344 return PromoteIntRes_Overflow(N);
1345 }
1346
PromoteIntRes_ABS(SDNode * N)1347 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1348 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1349 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1350 }
1351
PromoteIntRes_XMULO(SDNode * N,unsigned ResNo)1352 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1353 // Promote the overflow bit trivially.
1354 if (ResNo == 1)
1355 return PromoteIntRes_Overflow(N);
1356
1357 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1358 SDLoc DL(N);
1359 EVT SmallVT = LHS.getValueType();
1360
1361 // To determine if the result overflowed in a larger type, we extend the
1362 // input to the larger type, do the multiply (checking if it overflows),
1363 // then also check the high bits of the result to see if overflow happened
1364 // there.
1365 if (N->getOpcode() == ISD::SMULO) {
1366 LHS = SExtPromotedInteger(LHS);
1367 RHS = SExtPromotedInteger(RHS);
1368 } else {
1369 LHS = ZExtPromotedInteger(LHS);
1370 RHS = ZExtPromotedInteger(RHS);
1371 }
1372 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1373 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1374
1375 // Overflow occurred if it occurred in the larger type, or if the high part
1376 // of the result does not zero/sign-extend the low part. Check this second
1377 // possibility first.
1378 SDValue Overflow;
1379 if (N->getOpcode() == ISD::UMULO) {
1380 // Unsigned overflow occurred if the high part is non-zero.
1381 unsigned Shift = SmallVT.getScalarSizeInBits();
1382 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG);
1383 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1384 DAG.getConstant(Shift, DL, ShiftTy));
1385 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1386 DAG.getConstant(0, DL, Hi.getValueType()),
1387 ISD::SETNE);
1388 } else {
1389 // Signed overflow occurred if the high part does not sign extend the low.
1390 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1391 Mul, DAG.getValueType(SmallVT));
1392 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1393 }
1394
1395 // The only other way for overflow to occur is if the multiplication in the
1396 // larger type itself overflowed.
1397 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1398 SDValue(Mul.getNode(), 1));
1399
1400 // Use the calculated overflow everywhere.
1401 ReplaceValueWith(SDValue(N, 1), Overflow);
1402 return Mul;
1403 }
1404
PromoteIntRes_UNDEF(SDNode * N)1405 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1406 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1407 N->getValueType(0)));
1408 }
1409
PromoteIntRes_VSCALE(SDNode * N)1410 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1411 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1412
1413 APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
1414 return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1415 }
1416
PromoteIntRes_VAARG(SDNode * N)1417 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1418 SDValue Chain = N->getOperand(0); // Get the chain.
1419 SDValue Ptr = N->getOperand(1); // Get the pointer.
1420 EVT VT = N->getValueType(0);
1421 SDLoc dl(N);
1422
1423 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1424 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1425 // The argument is passed as NumRegs registers of type RegVT.
1426
1427 SmallVector<SDValue, 8> Parts(NumRegs);
1428 for (unsigned i = 0; i < NumRegs; ++i) {
1429 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1430 N->getConstantOperandVal(3));
1431 Chain = Parts[i].getValue(1);
1432 }
1433
1434 // Handle endianness of the load.
1435 if (DAG.getDataLayout().isBigEndian())
1436 std::reverse(Parts.begin(), Parts.end());
1437
1438 // Assemble the parts in the promoted type.
1439 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1440 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1441 for (unsigned i = 1; i < NumRegs; ++i) {
1442 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1443 // Shift it to the right position and "or" it in.
1444 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1445 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1446 TLI.getPointerTy(DAG.getDataLayout())));
1447 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1448 }
1449
1450 // Modified the chain result - switch anything that used the old chain to
1451 // use the new one.
1452 ReplaceValueWith(SDValue(N, 1), Chain);
1453
1454 return Res;
1455 }
1456
1457 //===----------------------------------------------------------------------===//
1458 // Integer Operand Promotion
1459 //===----------------------------------------------------------------------===//
1460
1461 /// PromoteIntegerOperand - This method is called when the specified operand of
1462 /// the specified node is found to need promotion. At this point, all of the
1463 /// result types of the node are known to be legal, but other operands of the
1464 /// node may need promotion or expansion as well as the specified one.
PromoteIntegerOperand(SDNode * N,unsigned OpNo)1465 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1466 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1467 dbgs() << "\n");
1468 SDValue Res = SDValue();
1469 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1470 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1471 return false;
1472 }
1473
1474 switch (N->getOpcode()) {
1475 default:
1476 #ifndef NDEBUG
1477 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1478 N->dump(&DAG); dbgs() << "\n";
1479 #endif
1480 llvm_unreachable("Do not know how to promote this operator's operand!");
1481
1482 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1483 case ISD::ATOMIC_STORE:
1484 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1485 break;
1486 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1487 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1488 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1489 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1490 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1491 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1492 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1493 case ISD::INSERT_VECTOR_ELT:
1494 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1495 case ISD::SCALAR_TO_VECTOR:
1496 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1497 case ISD::SPLAT_VECTOR:
1498 Res = PromoteIntOp_SPLAT_VECTOR(N); break;
1499 case ISD::VSELECT:
1500 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1501 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1502 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1503 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1504 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1505 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1506 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1507 OpNo); break;
1508 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1509 OpNo); break;
1510 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1511 OpNo); break;
1512 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1513 OpNo); break;
1514 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1515 OpNo); break;
1516 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1517 case ISD::FP16_TO_FP:
1518 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1519 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1520 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1521 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1522
1523 case ISD::SHL:
1524 case ISD::SRA:
1525 case ISD::SRL:
1526 case ISD::ROTL:
1527 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1528
1529 case ISD::SADDO_CARRY:
1530 case ISD::SSUBO_CARRY:
1531 case ISD::ADDCARRY:
1532 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1533
1534 case ISD::FRAMEADDR:
1535 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1536
1537 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1538
1539 case ISD::SMULFIX:
1540 case ISD::SMULFIXSAT:
1541 case ISD::UMULFIX:
1542 case ISD::UMULFIXSAT:
1543 case ISD::SDIVFIX:
1544 case ISD::SDIVFIXSAT:
1545 case ISD::UDIVFIX:
1546 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1547
1548 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1549
1550 case ISD::VECREDUCE_ADD:
1551 case ISD::VECREDUCE_MUL:
1552 case ISD::VECREDUCE_AND:
1553 case ISD::VECREDUCE_OR:
1554 case ISD::VECREDUCE_XOR:
1555 case ISD::VECREDUCE_SMAX:
1556 case ISD::VECREDUCE_SMIN:
1557 case ISD::VECREDUCE_UMAX:
1558 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1559
1560 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
1561 }
1562
1563 // If the result is null, the sub-method took care of registering results etc.
1564 if (!Res.getNode()) return false;
1565
1566 // If the result is N, the sub-method updated N in place. Tell the legalizer
1567 // core about this.
1568 if (Res.getNode() == N)
1569 return true;
1570
1571 const bool IsStrictFp = N->isStrictFPOpcode();
1572 assert(Res.getValueType() == N->getValueType(0) &&
1573 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1574 "Invalid operand expansion");
1575 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1576 Res.dump());
1577
1578 ReplaceValueWith(SDValue(N, 0), Res);
1579 if (IsStrictFp)
1580 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1581
1582 return false;
1583 }
1584
1585 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1586 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
PromoteSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode CCCode)1587 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1588 ISD::CondCode CCCode) {
1589 // We have to insert explicit sign or zero extends. Note that we could
1590 // insert sign extends for ALL conditions. For those operations where either
1591 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1592 // which will choose the cheapest for the target.
1593 switch (CCCode) {
1594 default: llvm_unreachable("Unknown integer comparison!");
1595 case ISD::SETEQ:
1596 case ISD::SETNE: {
1597 SDValue OpL = GetPromotedInteger(NewLHS);
1598 SDValue OpR = GetPromotedInteger(NewRHS);
1599
1600 // We would prefer to promote the comparison operand with sign extension.
1601 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1602 // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1603 // instruction, which is redundant eventually.
1604 unsigned OpLEffectiveBits =
1605 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1606 unsigned OpREffectiveBits =
1607 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1608 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1609 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1610 NewLHS = OpL;
1611 NewRHS = OpR;
1612 } else {
1613 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1614 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1615 }
1616 break;
1617 }
1618 case ISD::SETUGE:
1619 case ISD::SETUGT:
1620 case ISD::SETULE:
1621 case ISD::SETULT:
1622 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1623 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1624 break;
1625 case ISD::SETGE:
1626 case ISD::SETGT:
1627 case ISD::SETLT:
1628 case ISD::SETLE:
1629 NewLHS = SExtPromotedInteger(NewLHS);
1630 NewRHS = SExtPromotedInteger(NewRHS);
1631 break;
1632 }
1633 }
1634
PromoteIntOp_ANY_EXTEND(SDNode * N)1635 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1636 SDValue Op = GetPromotedInteger(N->getOperand(0));
1637 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1638 }
1639
PromoteIntOp_ATOMIC_STORE(AtomicSDNode * N)1640 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1641 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1642 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1643 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1644 }
1645
PromoteIntOp_BITCAST(SDNode * N)1646 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1647 // This should only occur in unusual situations like bitcasting to an
1648 // x86_fp80, so just turn it into a store+load
1649 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1650 }
1651
PromoteIntOp_BR_CC(SDNode * N,unsigned OpNo)1652 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1653 assert(OpNo == 2 && "Don't know how to promote this operand!");
1654
1655 SDValue LHS = N->getOperand(2);
1656 SDValue RHS = N->getOperand(3);
1657 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1658
1659 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1660 // legal types.
1661 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1662 N->getOperand(1), LHS, RHS, N->getOperand(4)),
1663 0);
1664 }
1665
PromoteIntOp_BRCOND(SDNode * N,unsigned OpNo)1666 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1667 assert(OpNo == 1 && "only know how to promote condition");
1668
1669 // Promote all the way up to the canonical SetCC type.
1670 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1671
1672 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1673 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1674 N->getOperand(2)), 0);
1675 }
1676
PromoteIntOp_BUILD_PAIR(SDNode * N)1677 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1678 // Since the result type is legal, the operands must promote to it.
1679 EVT OVT = N->getOperand(0).getValueType();
1680 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1681 SDValue Hi = GetPromotedInteger(N->getOperand(1));
1682 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1683 SDLoc dl(N);
1684
1685 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1686 DAG.getConstant(OVT.getSizeInBits(), dl,
1687 TLI.getPointerTy(DAG.getDataLayout())));
1688 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1689 }
1690
PromoteIntOp_BUILD_VECTOR(SDNode * N)1691 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1692 // The vector type is legal but the element type is not. This implies
1693 // that the vector is a power-of-two in length and that the element
1694 // type does not have a strange size (eg: it is not i1).
1695 EVT VecVT = N->getValueType(0);
1696 unsigned NumElts = VecVT.getVectorNumElements();
1697 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1698 "Legal vector of one illegal element?");
1699
1700 // Promote the inserted value. The type does not need to match the
1701 // vector element type. Check that any extra bits introduced will be
1702 // truncated away.
1703 assert(N->getOperand(0).getValueSizeInBits() >=
1704 N->getValueType(0).getScalarSizeInBits() &&
1705 "Type of inserted value narrower than vector element type!");
1706
1707 SmallVector<SDValue, 16> NewOps;
1708 for (unsigned i = 0; i < NumElts; ++i)
1709 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1710
1711 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1712 }
1713
PromoteIntOp_INSERT_VECTOR_ELT(SDNode * N,unsigned OpNo)1714 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1715 unsigned OpNo) {
1716 if (OpNo == 1) {
1717 // Promote the inserted value. This is valid because the type does not
1718 // have to match the vector element type.
1719
1720 // Check that any extra bits introduced will be truncated away.
1721 assert(N->getOperand(1).getValueSizeInBits() >=
1722 N->getValueType(0).getScalarSizeInBits() &&
1723 "Type of inserted value narrower than vector element type!");
1724 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1725 GetPromotedInteger(N->getOperand(1)),
1726 N->getOperand(2)),
1727 0);
1728 }
1729
1730 assert(OpNo == 2 && "Different operand and result vector types?");
1731
1732 // Promote the index.
1733 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1734 TLI.getVectorIdxTy(DAG.getDataLayout()));
1735 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1736 N->getOperand(1), Idx), 0);
1737 }
1738
PromoteIntOp_SCALAR_TO_VECTOR(SDNode * N)1739 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1740 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1741 // the operand in place.
1742 return SDValue(DAG.UpdateNodeOperands(N,
1743 GetPromotedInteger(N->getOperand(0))), 0);
1744 }
1745
PromoteIntOp_SPLAT_VECTOR(SDNode * N)1746 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1747 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
1748 // operand in place.
1749 return SDValue(
1750 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1751 }
1752
PromoteIntOp_SELECT(SDNode * N,unsigned OpNo)1753 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1754 assert(OpNo == 0 && "Only know how to promote the condition!");
1755 SDValue Cond = N->getOperand(0);
1756 EVT OpTy = N->getOperand(1).getValueType();
1757
1758 if (N->getOpcode() == ISD::VSELECT)
1759 if (SDValue Res = WidenVSELECTMask(N))
1760 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1761 Res, N->getOperand(1), N->getOperand(2));
1762
1763 // Promote all the way up to the canonical SetCC type.
1764 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1765 Cond = PromoteTargetBoolean(Cond, OpVT);
1766
1767 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1768 N->getOperand(2)), 0);
1769 }
1770
PromoteIntOp_SELECT_CC(SDNode * N,unsigned OpNo)1771 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1772 assert(OpNo == 0 && "Don't know how to promote this operand!");
1773
1774 SDValue LHS = N->getOperand(0);
1775 SDValue RHS = N->getOperand(1);
1776 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1777
1778 // The CC (#4) and the possible return values (#2 and #3) have legal types.
1779 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1780 N->getOperand(3), N->getOperand(4)), 0);
1781 }
1782
PromoteIntOp_SETCC(SDNode * N,unsigned OpNo)1783 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1784 assert(OpNo == 0 && "Don't know how to promote this operand!");
1785
1786 SDValue LHS = N->getOperand(0);
1787 SDValue RHS = N->getOperand(1);
1788 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1789
1790 // The CC (#2) is always legal.
1791 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1792 }
1793
PromoteIntOp_Shift(SDNode * N)1794 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1795 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1796 ZExtPromotedInteger(N->getOperand(1))), 0);
1797 }
1798
PromoteIntOp_SIGN_EXTEND(SDNode * N)1799 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1800 SDValue Op = GetPromotedInteger(N->getOperand(0));
1801 SDLoc dl(N);
1802 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1803 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1804 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1805 }
1806
PromoteIntOp_SINT_TO_FP(SDNode * N)1807 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1808 return SDValue(DAG.UpdateNodeOperands(N,
1809 SExtPromotedInteger(N->getOperand(0))), 0);
1810 }
1811
PromoteIntOp_STRICT_SINT_TO_FP(SDNode * N)1812 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
1813 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1814 SExtPromotedInteger(N->getOperand(1))), 0);
1815 }
1816
PromoteIntOp_STORE(StoreSDNode * N,unsigned OpNo)1817 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1818 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1819 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1820 SDLoc dl(N);
1821
1822 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1823
1824 // Truncate the value and store the result.
1825 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1826 N->getMemoryVT(), N->getMemOperand());
1827 }
1828
PromoteIntOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)1829 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1830 unsigned OpNo) {
1831
1832 SDValue DataOp = N->getValue();
1833 EVT DataVT = DataOp.getValueType();
1834 SDValue Mask = N->getMask();
1835 SDLoc dl(N);
1836
1837 bool TruncateStore = false;
1838 if (OpNo == 4) {
1839 Mask = PromoteTargetBoolean(Mask, DataVT);
1840 // Update in place.
1841 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1842 NewOps[4] = Mask;
1843 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1844 } else { // Data operand
1845 assert(OpNo == 1 && "Unexpected operand for promotion");
1846 DataOp = GetPromotedInteger(DataOp);
1847 TruncateStore = true;
1848 }
1849
1850 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(),
1851 N->getOffset(), Mask, N->getMemoryVT(),
1852 N->getMemOperand(), N->getAddressingMode(),
1853 TruncateStore, N->isCompressingStore());
1854 }
1855
PromoteIntOp_MLOAD(MaskedLoadSDNode * N,unsigned OpNo)1856 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1857 unsigned OpNo) {
1858 assert(OpNo == 3 && "Only know how to promote the mask!");
1859 EVT DataVT = N->getValueType(0);
1860 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1861 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1862 NewOps[OpNo] = Mask;
1863 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1864 if (Res == N)
1865 return SDValue(Res, 0);
1866
1867 // Update triggered CSE, do our own replacement since caller can't.
1868 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1869 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1870 return SDValue();
1871 }
1872
PromoteIntOp_MGATHER(MaskedGatherSDNode * N,unsigned OpNo)1873 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1874 unsigned OpNo) {
1875
1876 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1877 if (OpNo == 2) {
1878 // The Mask
1879 EVT DataVT = N->getValueType(0);
1880 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1881 } else if (OpNo == 4) {
1882 // The Index
1883 if (N->isIndexSigned())
1884 // Need to sign extend the index since the bits will likely be used.
1885 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1886 else
1887 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1888 } else
1889 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1890
1891 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1892 if (Res == N)
1893 return SDValue(Res, 0);
1894
1895 // Update triggered CSE, do our own replacement since caller can't.
1896 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1897 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1898 return SDValue();
1899 }
1900
PromoteIntOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)1901 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1902 unsigned OpNo) {
1903 bool TruncateStore = N->isTruncatingStore();
1904 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1905 if (OpNo == 2) {
1906 // The Mask
1907 EVT DataVT = N->getValue().getValueType();
1908 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1909 } else if (OpNo == 4) {
1910 // The Index
1911 if (N->isIndexSigned())
1912 // Need to sign extend the index since the bits will likely be used.
1913 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1914 else
1915 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1916
1917 N->setIndexType(TLI.getCanonicalIndexType(N->getIndexType(),
1918 N->getMemoryVT(), NewOps[OpNo]));
1919 } else {
1920 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1921 TruncateStore = true;
1922 }
1923
1924 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
1925 SDLoc(N), NewOps, N->getMemOperand(),
1926 N->getIndexType(), TruncateStore);
1927 }
1928
PromoteIntOp_TRUNCATE(SDNode * N)1929 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1930 SDValue Op = GetPromotedInteger(N->getOperand(0));
1931 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1932 }
1933
PromoteIntOp_UINT_TO_FP(SDNode * N)1934 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1935 return SDValue(DAG.UpdateNodeOperands(N,
1936 ZExtPromotedInteger(N->getOperand(0))), 0);
1937 }
1938
PromoteIntOp_STRICT_UINT_TO_FP(SDNode * N)1939 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
1940 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1941 ZExtPromotedInteger(N->getOperand(1))), 0);
1942 }
1943
PromoteIntOp_ZERO_EXTEND(SDNode * N)1944 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1945 SDLoc dl(N);
1946 SDValue Op = GetPromotedInteger(N->getOperand(0));
1947 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1948 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
1949 }
1950
PromoteIntOp_ADDSUBCARRY(SDNode * N,unsigned OpNo)1951 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1952 assert(OpNo == 2 && "Don't know how to promote this operand!");
1953
1954 SDValue LHS = N->getOperand(0);
1955 SDValue RHS = N->getOperand(1);
1956 SDValue Carry = N->getOperand(2);
1957 SDLoc DL(N);
1958
1959 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1960
1961 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1962 }
1963
PromoteIntOp_FIX(SDNode * N)1964 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
1965 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1966 return SDValue(
1967 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1968 }
1969
PromoteIntOp_FRAMERETURNADDR(SDNode * N)1970 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1971 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1972 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1973 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1974 }
1975
PromoteIntOp_PREFETCH(SDNode * N,unsigned OpNo)1976 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1977 assert(OpNo > 1 && "Don't know how to promote this operand!");
1978 // Promote the rw, locality, and cache type arguments to a supported integer
1979 // width.
1980 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1981 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1982 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1983 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1984 Op2, Op3, Op4),
1985 0);
1986 }
1987
PromoteIntOp_FPOWI(SDNode * N)1988 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1989 SDValue Op = SExtPromotedInteger(N->getOperand(1));
1990 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1991 }
1992
PromoteIntOp_VECREDUCE(SDNode * N)1993 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1994 SDLoc dl(N);
1995 SDValue Op;
1996 switch (N->getOpcode()) {
1997 default: llvm_unreachable("Expected integer vector reduction");
1998 case ISD::VECREDUCE_ADD:
1999 case ISD::VECREDUCE_MUL:
2000 case ISD::VECREDUCE_AND:
2001 case ISD::VECREDUCE_OR:
2002 case ISD::VECREDUCE_XOR:
2003 Op = GetPromotedInteger(N->getOperand(0));
2004 break;
2005 case ISD::VECREDUCE_SMAX:
2006 case ISD::VECREDUCE_SMIN:
2007 Op = SExtPromotedInteger(N->getOperand(0));
2008 break;
2009 case ISD::VECREDUCE_UMAX:
2010 case ISD::VECREDUCE_UMIN:
2011 Op = ZExtPromotedInteger(N->getOperand(0));
2012 break;
2013 }
2014
2015 EVT EltVT = Op.getValueType().getVectorElementType();
2016 EVT VT = N->getValueType(0);
2017 if (VT.bitsGE(EltVT))
2018 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
2019
2020 // Result size must be >= element size. If this is not the case after
2021 // promotion, also promote the result type and then truncate.
2022 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
2023 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
2024 }
2025
PromoteIntOp_SET_ROUNDING(SDNode * N)2026 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2027 SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2028 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2029 }
2030
2031 //===----------------------------------------------------------------------===//
2032 // Integer Result Expansion
2033 //===----------------------------------------------------------------------===//
2034
2035 /// ExpandIntegerResult - This method is called when the specified result of the
2036 /// specified node is found to need expansion. At this point, the node may also
2037 /// have invalid operands or may have other results that need promotion, we just
2038 /// know that (at least) one result needs expansion.
ExpandIntegerResult(SDNode * N,unsigned ResNo)2039 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2040 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
2041 dbgs() << "\n");
2042 SDValue Lo, Hi;
2043 Lo = Hi = SDValue();
2044
2045 // See if the target wants to custom expand this node.
2046 if (CustomLowerNode(N, N->getValueType(ResNo), true))
2047 return;
2048
2049 switch (N->getOpcode()) {
2050 default:
2051 #ifndef NDEBUG
2052 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2053 N->dump(&DAG); dbgs() << "\n";
2054 #endif
2055 report_fatal_error("Do not know how to expand the result of this "
2056 "operator!");
2057
2058 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2059 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
2060 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2061 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2062 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2063
2064 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2065 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2066 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2067 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2068 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2069
2070 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2071 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2072 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2073 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2074 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2075 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2076 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2077 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2078 case ISD::CTLZ_ZERO_UNDEF:
2079 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2080 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2081 case ISD::CTTZ_ZERO_UNDEF:
2082 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2083 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
2084 case ISD::STRICT_FP_TO_SINT:
2085 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
2086 case ISD::STRICT_FP_TO_UINT:
2087 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
2088 case ISD::FP_TO_SINT_SAT:
2089 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2090 case ISD::STRICT_LLROUND:
2091 case ISD::STRICT_LLRINT:
2092 case ISD::LLROUND:
2093 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
2094 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2095 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2096 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
2097 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2098 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2099 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2100 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2101 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2102 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2103 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2104 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2105 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2106
2107 case ISD::ATOMIC_LOAD_ADD:
2108 case ISD::ATOMIC_LOAD_SUB:
2109 case ISD::ATOMIC_LOAD_AND:
2110 case ISD::ATOMIC_LOAD_CLR:
2111 case ISD::ATOMIC_LOAD_OR:
2112 case ISD::ATOMIC_LOAD_XOR:
2113 case ISD::ATOMIC_LOAD_NAND:
2114 case ISD::ATOMIC_LOAD_MIN:
2115 case ISD::ATOMIC_LOAD_MAX:
2116 case ISD::ATOMIC_LOAD_UMIN:
2117 case ISD::ATOMIC_LOAD_UMAX:
2118 case ISD::ATOMIC_SWAP:
2119 case ISD::ATOMIC_CMP_SWAP: {
2120 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2121 SplitInteger(Tmp.first, Lo, Hi);
2122 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2123 break;
2124 }
2125 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2126 AtomicSDNode *AN = cast<AtomicSDNode>(N);
2127 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2128 SDValue Tmp = DAG.getAtomicCmpSwap(
2129 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
2130 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2131 AN->getMemOperand());
2132
2133 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2134 // success simply by comparing the loaded value against the ingoing
2135 // comparison.
2136 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2137 N->getOperand(2), ISD::SETEQ);
2138
2139 SplitInteger(Tmp, Lo, Hi);
2140 ReplaceValueWith(SDValue(N, 1), Success);
2141 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2142 break;
2143 }
2144
2145 case ISD::AND:
2146 case ISD::OR:
2147 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2148
2149 case ISD::UMAX:
2150 case ISD::SMAX:
2151 case ISD::UMIN:
2152 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2153
2154 case ISD::ADD:
2155 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2156
2157 case ISD::ADDC:
2158 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2159
2160 case ISD::ADDE:
2161 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2162
2163 case ISD::ADDCARRY:
2164 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
2165
2166 case ISD::SADDO_CARRY:
2167 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2168
2169 case ISD::SHL:
2170 case ISD::SRA:
2171 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2172
2173 case ISD::SADDO:
2174 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2175 case ISD::UADDO:
2176 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2177 case ISD::UMULO:
2178 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2179
2180 case ISD::SADDSAT:
2181 case ISD::UADDSAT:
2182 case ISD::SSUBSAT:
2183 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2184
2185 case ISD::SSHLSAT:
2186 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2187
2188 case ISD::SMULFIX:
2189 case ISD::SMULFIXSAT:
2190 case ISD::UMULFIX:
2191 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2192
2193 case ISD::SDIVFIX:
2194 case ISD::SDIVFIXSAT:
2195 case ISD::UDIVFIX:
2196 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2197
2198 case ISD::VECREDUCE_ADD:
2199 case ISD::VECREDUCE_MUL:
2200 case ISD::VECREDUCE_AND:
2201 case ISD::VECREDUCE_OR:
2202 case ISD::VECREDUCE_XOR:
2203 case ISD::VECREDUCE_SMAX:
2204 case ISD::VECREDUCE_SMIN:
2205 case ISD::VECREDUCE_UMAX:
2206 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2207
2208 case ISD::ROTL:
2209 case ISD::ROTR:
2210 ExpandIntRes_Rotate(N, Lo, Hi);
2211 break;
2212
2213 case ISD::FSHL:
2214 case ISD::FSHR:
2215 ExpandIntRes_FunnelShift(N, Lo, Hi);
2216 break;
2217 }
2218
2219 // If Lo/Hi is null, the sub-method took care of registering results etc.
2220 if (Lo.getNode())
2221 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2222 }
2223
2224 /// Lower an atomic node to the appropriate builtin call.
ExpandAtomic(SDNode * Node)2225 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2226 unsigned Opc = Node->getOpcode();
2227 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2228 AtomicOrdering order = cast<AtomicSDNode>(Node)->getOrdering();
2229 // Lower to outline atomic libcall if outline atomics enabled,
2230 // or to sync libcall otherwise
2231 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2232 EVT RetVT = Node->getValueType(0);
2233 TargetLowering::MakeLibCallOptions CallOptions;
2234 SmallVector<SDValue, 4> Ops;
2235 if (TLI.getLibcallName(LC)) {
2236 Ops.append(Node->op_begin() + 2, Node->op_end());
2237 Ops.push_back(Node->getOperand(1));
2238 } else {
2239 LC = RTLIB::getSYNC(Opc, VT);
2240 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2241 "Unexpected atomic op or value type!");
2242 Ops.append(Node->op_begin() + 1, Node->op_end());
2243 }
2244 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2245 Node->getOperand(0));
2246 }
2247
2248 /// N is a shift by a value that needs to be expanded,
2249 /// and the shift amount is a constant 'Amt'. Expand the operation.
ExpandShiftByConstant(SDNode * N,const APInt & Amt,SDValue & Lo,SDValue & Hi)2250 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2251 SDValue &Lo, SDValue &Hi) {
2252 SDLoc DL(N);
2253 // Expand the incoming operand to be shifted, so that we have its parts
2254 SDValue InL, InH;
2255 GetExpandedInteger(N->getOperand(0), InL, InH);
2256
2257 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2258 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2259 if (!Amt) {
2260 Lo = InL;
2261 Hi = InH;
2262 return;
2263 }
2264
2265 EVT NVT = InL.getValueType();
2266 unsigned VTBits = N->getValueType(0).getSizeInBits();
2267 unsigned NVTBits = NVT.getSizeInBits();
2268 EVT ShTy = N->getOperand(1).getValueType();
2269
2270 if (N->getOpcode() == ISD::SHL) {
2271 if (Amt.ugt(VTBits)) {
2272 Lo = Hi = DAG.getConstant(0, DL, NVT);
2273 } else if (Amt.ugt(NVTBits)) {
2274 Lo = DAG.getConstant(0, DL, NVT);
2275 Hi = DAG.getNode(ISD::SHL, DL,
2276 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2277 } else if (Amt == NVTBits) {
2278 Lo = DAG.getConstant(0, DL, NVT);
2279 Hi = InL;
2280 } else {
2281 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
2282 Hi = DAG.getNode(ISD::OR, DL, NVT,
2283 DAG.getNode(ISD::SHL, DL, NVT, InH,
2284 DAG.getConstant(Amt, DL, ShTy)),
2285 DAG.getNode(ISD::SRL, DL, NVT, InL,
2286 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2287 }
2288 return;
2289 }
2290
2291 if (N->getOpcode() == ISD::SRL) {
2292 if (Amt.ugt(VTBits)) {
2293 Lo = Hi = DAG.getConstant(0, DL, NVT);
2294 } else if (Amt.ugt(NVTBits)) {
2295 Lo = DAG.getNode(ISD::SRL, DL,
2296 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2297 Hi = DAG.getConstant(0, DL, NVT);
2298 } else if (Amt == NVTBits) {
2299 Lo = InH;
2300 Hi = DAG.getConstant(0, DL, NVT);
2301 } else {
2302 Lo = DAG.getNode(ISD::OR, DL, NVT,
2303 DAG.getNode(ISD::SRL, DL, NVT, InL,
2304 DAG.getConstant(Amt, DL, ShTy)),
2305 DAG.getNode(ISD::SHL, DL, NVT, InH,
2306 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2307 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2308 }
2309 return;
2310 }
2311
2312 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2313 if (Amt.ugt(VTBits)) {
2314 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2315 DAG.getConstant(NVTBits - 1, DL, ShTy));
2316 } else if (Amt.ugt(NVTBits)) {
2317 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2318 DAG.getConstant(Amt - NVTBits, DL, ShTy));
2319 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2320 DAG.getConstant(NVTBits - 1, DL, ShTy));
2321 } else if (Amt == NVTBits) {
2322 Lo = InH;
2323 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2324 DAG.getConstant(NVTBits - 1, DL, ShTy));
2325 } else {
2326 Lo = DAG.getNode(ISD::OR, DL, NVT,
2327 DAG.getNode(ISD::SRL, DL, NVT, InL,
2328 DAG.getConstant(Amt, DL, ShTy)),
2329 DAG.getNode(ISD::SHL, DL, NVT, InH,
2330 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2331 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2332 }
2333 }
2334
2335 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2336 /// this shift based on knowledge of the high bit of the shift amount. If we
2337 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2338 /// shift amount.
2339 bool DAGTypeLegalizer::
ExpandShiftWithKnownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2340 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2341 SDValue Amt = N->getOperand(1);
2342 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2343 EVT ShTy = Amt.getValueType();
2344 unsigned ShBits = ShTy.getScalarSizeInBits();
2345 unsigned NVTBits = NVT.getScalarSizeInBits();
2346 assert(isPowerOf2_32(NVTBits) &&
2347 "Expanded integer type size not a power of two!");
2348 SDLoc dl(N);
2349
2350 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2351 KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2352
2353 // If we don't know anything about the high bits, exit.
2354 if (((Known.Zero|Known.One) & HighBitMask) == 0)
2355 return false;
2356
2357 // Get the incoming operand to be shifted.
2358 SDValue InL, InH;
2359 GetExpandedInteger(N->getOperand(0), InL, InH);
2360
2361 // If we know that any of the high bits of the shift amount are one, then we
2362 // can do this as a couple of simple shifts.
2363 if (Known.One.intersects(HighBitMask)) {
2364 // Mask out the high bit, which we know is set.
2365 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2366 DAG.getConstant(~HighBitMask, dl, ShTy));
2367
2368 switch (N->getOpcode()) {
2369 default: llvm_unreachable("Unknown shift");
2370 case ISD::SHL:
2371 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2372 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2373 return true;
2374 case ISD::SRL:
2375 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2376 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2377 return true;
2378 case ISD::SRA:
2379 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2380 DAG.getConstant(NVTBits - 1, dl, ShTy));
2381 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2382 return true;
2383 }
2384 }
2385
2386 // If we know that all of the high bits of the shift amount are zero, then we
2387 // can do this as a couple of simple shifts.
2388 if (HighBitMask.isSubsetOf(Known.Zero)) {
2389 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2390 // shift if x is zero. We can use XOR here because x is known to be smaller
2391 // than 32.
2392 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2393 DAG.getConstant(NVTBits - 1, dl, ShTy));
2394
2395 unsigned Op1, Op2;
2396 switch (N->getOpcode()) {
2397 default: llvm_unreachable("Unknown shift");
2398 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2399 case ISD::SRL:
2400 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2401 }
2402
2403 // When shifting right the arithmetic for Lo and Hi is swapped.
2404 if (N->getOpcode() != ISD::SHL)
2405 std::swap(InL, InH);
2406
2407 // Use a little trick to get the bits that move from Lo to Hi. First
2408 // shift by one bit.
2409 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2410 // Then compute the remaining shift with amount-1.
2411 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2412
2413 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2414 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2415
2416 if (N->getOpcode() != ISD::SHL)
2417 std::swap(Hi, Lo);
2418 return true;
2419 }
2420
2421 return false;
2422 }
2423
2424 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2425 /// of any size.
2426 bool DAGTypeLegalizer::
ExpandShiftWithUnknownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2427 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2428 SDValue Amt = N->getOperand(1);
2429 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2430 EVT ShTy = Amt.getValueType();
2431 unsigned NVTBits = NVT.getSizeInBits();
2432 assert(isPowerOf2_32(NVTBits) &&
2433 "Expanded integer type size not a power of two!");
2434 SDLoc dl(N);
2435
2436 // Get the incoming operand to be shifted.
2437 SDValue InL, InH;
2438 GetExpandedInteger(N->getOperand(0), InL, InH);
2439
2440 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2441 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2442 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2443 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2444 Amt, NVBitsNode, ISD::SETULT);
2445 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2446 Amt, DAG.getConstant(0, dl, ShTy),
2447 ISD::SETEQ);
2448
2449 SDValue LoS, HiS, LoL, HiL;
2450 switch (N->getOpcode()) {
2451 default: llvm_unreachable("Unknown shift");
2452 case ISD::SHL:
2453 // Short: ShAmt < NVTBits
2454 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2455 HiS = DAG.getNode(ISD::OR, dl, NVT,
2456 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2457 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2458
2459 // Long: ShAmt >= NVTBits
2460 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2461 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2462
2463 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2464 Hi = DAG.getSelect(dl, NVT, isZero, InH,
2465 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2466 return true;
2467 case ISD::SRL:
2468 // Short: ShAmt < NVTBits
2469 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2470 LoS = DAG.getNode(ISD::OR, dl, NVT,
2471 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2472 // FIXME: If Amt is zero, the following shift generates an undefined result
2473 // on some architectures.
2474 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2475
2476 // Long: ShAmt >= NVTBits
2477 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2478 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2479
2480 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2481 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2482 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2483 return true;
2484 case ISD::SRA:
2485 // Short: ShAmt < NVTBits
2486 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2487 LoS = DAG.getNode(ISD::OR, dl, NVT,
2488 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2489 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2490
2491 // Long: ShAmt >= NVTBits
2492 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2493 DAG.getConstant(NVTBits - 1, dl, ShTy));
2494 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2495
2496 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2497 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2498 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2499 return true;
2500 }
2501 }
2502
getExpandedMinMaxOps(int Op)2503 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2504
2505 switch (Op) {
2506 default: llvm_unreachable("invalid min/max opcode");
2507 case ISD::SMAX:
2508 return std::make_pair(ISD::SETGT, ISD::UMAX);
2509 case ISD::UMAX:
2510 return std::make_pair(ISD::SETUGT, ISD::UMAX);
2511 case ISD::SMIN:
2512 return std::make_pair(ISD::SETLT, ISD::UMIN);
2513 case ISD::UMIN:
2514 return std::make_pair(ISD::SETULT, ISD::UMIN);
2515 }
2516 }
2517
ExpandIntRes_MINMAX(SDNode * N,SDValue & Lo,SDValue & Hi)2518 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2519 SDValue &Lo, SDValue &Hi) {
2520 SDLoc DL(N);
2521 ISD::NodeType LoOpc;
2522 ISD::CondCode CondC;
2523 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2524
2525 // Expand the subcomponents.
2526 SDValue LHSL, LHSH, RHSL, RHSH;
2527 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2528 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2529
2530 // Value types
2531 EVT NVT = LHSL.getValueType();
2532 EVT CCT = getSetCCResultType(NVT);
2533
2534 // Hi part is always the same op
2535 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2536
2537 // We need to know whether to select Lo part that corresponds to 'winning'
2538 // Hi part or if Hi parts are equal.
2539 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2540 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2541
2542 // Lo part corresponding to the 'winning' Hi part
2543 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2544
2545 // Recursed Lo part if Hi parts are equal, this uses unsigned version
2546 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2547
2548 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2549 }
2550
ExpandIntRes_ADDSUB(SDNode * N,SDValue & Lo,SDValue & Hi)2551 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2552 SDValue &Lo, SDValue &Hi) {
2553 SDLoc dl(N);
2554 // Expand the subcomponents.
2555 SDValue LHSL, LHSH, RHSL, RHSH;
2556 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2557 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2558
2559 EVT NVT = LHSL.getValueType();
2560 SDValue LoOps[2] = { LHSL, RHSL };
2561 SDValue HiOps[3] = { LHSH, RHSH };
2562
2563 bool HasOpCarry = TLI.isOperationLegalOrCustom(
2564 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2565 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2566 if (HasOpCarry) {
2567 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2568 if (N->getOpcode() == ISD::ADD) {
2569 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2570 HiOps[2] = Lo.getValue(1);
2571 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2572 } else {
2573 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2574 HiOps[2] = Lo.getValue(1);
2575 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2576 }
2577 return;
2578 }
2579
2580 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2581 // them. TODO: Teach operation legalization how to expand unsupported
2582 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2583 // a carry of type MVT::Glue, but there doesn't seem to be any way to
2584 // generate a value of this type in the expanded code sequence.
2585 bool hasCarry =
2586 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2587 ISD::ADDC : ISD::SUBC,
2588 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2589
2590 if (hasCarry) {
2591 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2592 if (N->getOpcode() == ISD::ADD) {
2593 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2594 HiOps[2] = Lo.getValue(1);
2595 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2596 } else {
2597 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2598 HiOps[2] = Lo.getValue(1);
2599 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2600 }
2601 return;
2602 }
2603
2604 bool hasOVF =
2605 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2606 ISD::UADDO : ISD::USUBO,
2607 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2608 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
2609
2610 if (hasOVF) {
2611 EVT OvfVT = getSetCCResultType(NVT);
2612 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2613 int RevOpc;
2614 if (N->getOpcode() == ISD::ADD) {
2615 RevOpc = ISD::SUB;
2616 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2617 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2618 } else {
2619 RevOpc = ISD::ADD;
2620 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2621 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2622 }
2623 SDValue OVF = Lo.getValue(1);
2624
2625 switch (BoolType) {
2626 case TargetLoweringBase::UndefinedBooleanContent:
2627 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2628 LLVM_FALLTHROUGH;
2629 case TargetLoweringBase::ZeroOrOneBooleanContent:
2630 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2631 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2632 break;
2633 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2634 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2635 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2636 }
2637 return;
2638 }
2639
2640 if (N->getOpcode() == ISD::ADD) {
2641 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2642 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2643 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2644 ISD::SETULT);
2645
2646 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
2647 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2648 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2649 return;
2650 }
2651
2652 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2653 DAG.getConstant(1, dl, NVT),
2654 DAG.getConstant(0, dl, NVT));
2655 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2656 ISD::SETULT);
2657 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2658 DAG.getConstant(1, dl, NVT), Carry1);
2659 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2660 } else {
2661 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2662 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2663 SDValue Cmp =
2664 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2665 LoOps[0], LoOps[1], ISD::SETULT);
2666
2667 SDValue Borrow;
2668 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
2669 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2670 else
2671 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2672 DAG.getConstant(0, dl, NVT));
2673
2674 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2675 }
2676 }
2677
ExpandIntRes_ADDSUBC(SDNode * N,SDValue & Lo,SDValue & Hi)2678 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2679 SDValue &Lo, SDValue &Hi) {
2680 // Expand the subcomponents.
2681 SDValue LHSL, LHSH, RHSL, RHSH;
2682 SDLoc dl(N);
2683 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2684 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2685 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2686 SDValue LoOps[2] = { LHSL, RHSL };
2687 SDValue HiOps[3] = { LHSH, RHSH };
2688
2689 if (N->getOpcode() == ISD::ADDC) {
2690 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2691 HiOps[2] = Lo.getValue(1);
2692 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2693 } else {
2694 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2695 HiOps[2] = Lo.getValue(1);
2696 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2697 }
2698
2699 // Legalized the flag result - switch anything that used the old flag to
2700 // use the new one.
2701 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2702 }
2703
ExpandIntRes_ADDSUBE(SDNode * N,SDValue & Lo,SDValue & Hi)2704 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2705 SDValue &Lo, SDValue &Hi) {
2706 // Expand the subcomponents.
2707 SDValue LHSL, LHSH, RHSL, RHSH;
2708 SDLoc dl(N);
2709 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2710 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2711 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2712 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2713 SDValue HiOps[3] = { LHSH, RHSH };
2714
2715 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2716 HiOps[2] = Lo.getValue(1);
2717 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2718
2719 // Legalized the flag result - switch anything that used the old flag to
2720 // use the new one.
2721 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2722 }
2723
ExpandIntRes_UADDSUBO(SDNode * N,SDValue & Lo,SDValue & Hi)2724 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2725 SDValue &Lo, SDValue &Hi) {
2726 SDValue LHS = N->getOperand(0);
2727 SDValue RHS = N->getOperand(1);
2728 SDLoc dl(N);
2729
2730 SDValue Ovf;
2731
2732 unsigned CarryOp, NoCarryOp;
2733 ISD::CondCode Cond;
2734 switch(N->getOpcode()) {
2735 case ISD::UADDO:
2736 CarryOp = ISD::ADDCARRY;
2737 NoCarryOp = ISD::ADD;
2738 Cond = ISD::SETULT;
2739 break;
2740 case ISD::USUBO:
2741 CarryOp = ISD::SUBCARRY;
2742 NoCarryOp = ISD::SUB;
2743 Cond = ISD::SETUGT;
2744 break;
2745 default:
2746 llvm_unreachable("Node has unexpected Opcode");
2747 }
2748
2749 bool HasCarryOp = TLI.isOperationLegalOrCustom(
2750 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2751
2752 if (HasCarryOp) {
2753 // Expand the subcomponents.
2754 SDValue LHSL, LHSH, RHSL, RHSH;
2755 GetExpandedInteger(LHS, LHSL, LHSH);
2756 GetExpandedInteger(RHS, RHSL, RHSH);
2757 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2758 SDValue LoOps[2] = { LHSL, RHSL };
2759 SDValue HiOps[3] = { LHSH, RHSH };
2760
2761 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2762 HiOps[2] = Lo.getValue(1);
2763 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
2764
2765 Ovf = Hi.getValue(1);
2766 } else {
2767 // Expand the result by simply replacing it with the equivalent
2768 // non-overflow-checking operation.
2769 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2770 SplitInteger(Sum, Lo, Hi);
2771
2772 // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2773 // overflows iff a - b > a.
2774 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2775 }
2776
2777 // Legalized the flag result - switch anything that used the old flag to
2778 // use the new one.
2779 ReplaceValueWith(SDValue(N, 1), Ovf);
2780 }
2781
ExpandIntRes_ADDSUBCARRY(SDNode * N,SDValue & Lo,SDValue & Hi)2782 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2783 SDValue &Lo, SDValue &Hi) {
2784 // Expand the subcomponents.
2785 SDValue LHSL, LHSH, RHSL, RHSH;
2786 SDLoc dl(N);
2787 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2788 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2789 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2790 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2791 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2792
2793 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2794 HiOps[2] = Lo.getValue(1);
2795 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2796
2797 // Legalized the flag result - switch anything that used the old flag to
2798 // use the new one.
2799 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2800 }
2801
ExpandIntRes_SADDSUBO_CARRY(SDNode * N,SDValue & Lo,SDValue & Hi)2802 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
2803 SDValue &Lo, SDValue &Hi) {
2804 // Expand the subcomponents.
2805 SDValue LHSL, LHSH, RHSL, RHSH;
2806 SDLoc dl(N);
2807 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2808 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2809 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2810
2811 // We need to use an unsigned carry op for the lo part.
2812 unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY
2813 : ISD::SUBCARRY;
2814 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
2815 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
2816
2817 // Legalized the flag result - switch anything that used the old flag to
2818 // use the new one.
2819 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2820 }
2821
ExpandIntRes_ANY_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)2822 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2823 SDValue &Lo, SDValue &Hi) {
2824 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2825 SDLoc dl(N);
2826 SDValue Op = N->getOperand(0);
2827 if (Op.getValueType().bitsLE(NVT)) {
2828 // The low part is any extension of the input (which degenerates to a copy).
2829 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2830 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2831 } else {
2832 // For example, extension of an i48 to an i64. The operand type necessarily
2833 // promotes to the result type, so will end up being expanded too.
2834 assert(getTypeAction(Op.getValueType()) ==
2835 TargetLowering::TypePromoteInteger &&
2836 "Only know how to promote this result!");
2837 SDValue Res = GetPromotedInteger(Op);
2838 assert(Res.getValueType() == N->getValueType(0) &&
2839 "Operand over promoted?");
2840 // Split the promoted operand. This will simplify when it is expanded.
2841 SplitInteger(Res, Lo, Hi);
2842 }
2843 }
2844
ExpandIntRes_AssertSext(SDNode * N,SDValue & Lo,SDValue & Hi)2845 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2846 SDValue &Lo, SDValue &Hi) {
2847 SDLoc dl(N);
2848 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2849 EVT NVT = Lo.getValueType();
2850 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2851 unsigned NVTBits = NVT.getSizeInBits();
2852 unsigned EVTBits = EVT.getSizeInBits();
2853
2854 if (NVTBits < EVTBits) {
2855 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2856 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2857 EVTBits - NVTBits)));
2858 } else {
2859 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2860 // The high part replicates the sign bit of Lo, make it explicit.
2861 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2862 DAG.getConstant(NVTBits - 1, dl,
2863 TLI.getPointerTy(DAG.getDataLayout())));
2864 }
2865 }
2866
ExpandIntRes_AssertZext(SDNode * N,SDValue & Lo,SDValue & Hi)2867 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2868 SDValue &Lo, SDValue &Hi) {
2869 SDLoc dl(N);
2870 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2871 EVT NVT = Lo.getValueType();
2872 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2873 unsigned NVTBits = NVT.getSizeInBits();
2874 unsigned EVTBits = EVT.getSizeInBits();
2875
2876 if (NVTBits < EVTBits) {
2877 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2878 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2879 EVTBits - NVTBits)));
2880 } else {
2881 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2882 // The high part must be zero, make it explicit.
2883 Hi = DAG.getConstant(0, dl, NVT);
2884 }
2885 }
2886
ExpandIntRes_BITREVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)2887 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2888 SDValue &Lo, SDValue &Hi) {
2889 SDLoc dl(N);
2890 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2891 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2892 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2893 }
2894
ExpandIntRes_BSWAP(SDNode * N,SDValue & Lo,SDValue & Hi)2895 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2896 SDValue &Lo, SDValue &Hi) {
2897 SDLoc dl(N);
2898 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2899 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2900 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2901 }
2902
ExpandIntRes_PARITY(SDNode * N,SDValue & Lo,SDValue & Hi)2903 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
2904 SDValue &Hi) {
2905 SDLoc dl(N);
2906 // parity(HiLo) -> parity(Lo^Hi)
2907 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2908 EVT NVT = Lo.getValueType();
2909 Lo =
2910 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
2911 Hi = DAG.getConstant(0, dl, NVT);
2912 }
2913
ExpandIntRes_Constant(SDNode * N,SDValue & Lo,SDValue & Hi)2914 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2915 SDValue &Lo, SDValue &Hi) {
2916 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2917 unsigned NBitWidth = NVT.getSizeInBits();
2918 auto Constant = cast<ConstantSDNode>(N);
2919 const APInt &Cst = Constant->getAPIntValue();
2920 bool IsTarget = Constant->isTargetOpcode();
2921 bool IsOpaque = Constant->isOpaque();
2922 SDLoc dl(N);
2923 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2924 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2925 IsOpaque);
2926 }
2927
ExpandIntRes_ABS(SDNode * N,SDValue & Lo,SDValue & Hi)2928 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2929 SDLoc dl(N);
2930
2931 SDValue N0 = N->getOperand(0);
2932 GetExpandedInteger(N0, Lo, Hi);
2933 EVT NVT = Lo.getValueType();
2934
2935 // If we have ADDCARRY, use the expanded form of the sra+add+xor sequence we
2936 // use in LegalizeDAG. The ADD part of the expansion is based on
2937 // ExpandIntRes_ADDSUB which also uses ADDCARRY/UADDO after checking that
2938 // ADDCARRY is LegalOrCustom. Each of the pieces here can be further expanded
2939 // if needed. Shift expansion has a special case for filling with sign bits
2940 // so that we will only end up with one SRA.
2941 bool HasAddCarry = TLI.isOperationLegalOrCustom(
2942 ISD::ADDCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2943 if (HasAddCarry) {
2944 EVT ShiftAmtTy = getShiftAmountTyForConstant(NVT, TLI, DAG);
2945 SDValue Sign =
2946 DAG.getNode(ISD::SRA, dl, NVT, Hi,
2947 DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy));
2948 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2949 Lo = DAG.getNode(ISD::UADDO, dl, VTList, Lo, Sign);
2950 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
2951 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
2952 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
2953 return;
2954 }
2955
2956 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2957 EVT VT = N->getValueType(0);
2958 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2959 DAG.getConstant(0, dl, VT), N0);
2960 SDValue NegLo, NegHi;
2961 SplitInteger(Neg, NegLo, NegHi);
2962
2963 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2964 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2965 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2966 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2967 }
2968
ExpandIntRes_CTLZ(SDNode * N,SDValue & Lo,SDValue & Hi)2969 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2970 SDValue &Lo, SDValue &Hi) {
2971 SDLoc dl(N);
2972 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2973 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2974 EVT NVT = Lo.getValueType();
2975
2976 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2977 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2978
2979 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2980 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2981
2982 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2983 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2984 DAG.getConstant(NVT.getSizeInBits(), dl,
2985 NVT)));
2986 Hi = DAG.getConstant(0, dl, NVT);
2987 }
2988
ExpandIntRes_CTPOP(SDNode * N,SDValue & Lo,SDValue & Hi)2989 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2990 SDValue &Lo, SDValue &Hi) {
2991 SDLoc dl(N);
2992 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2993 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2994 EVT NVT = Lo.getValueType();
2995 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2996 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2997 Hi = DAG.getConstant(0, dl, NVT);
2998 }
2999
ExpandIntRes_CTTZ(SDNode * N,SDValue & Lo,SDValue & Hi)3000 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
3001 SDValue &Lo, SDValue &Hi) {
3002 SDLoc dl(N);
3003 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
3004 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3005 EVT NVT = Lo.getValueType();
3006
3007 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3008 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3009
3010 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
3011 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
3012
3013 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
3014 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
3015 DAG.getConstant(NVT.getSizeInBits(), dl,
3016 NVT)));
3017 Hi = DAG.getConstant(0, dl, NVT);
3018 }
3019
ExpandIntRes_FLT_ROUNDS(SDNode * N,SDValue & Lo,SDValue & Hi)3020 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
3021 SDValue &Hi) {
3022 SDLoc dl(N);
3023 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3024 unsigned NBitWidth = NVT.getSizeInBits();
3025
3026 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3027 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0));
3028 SDValue Chain = Lo.getValue(1);
3029 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
3030 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3031 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
3032
3033 // Legalize the chain result - switch anything that used the old chain to
3034 // use the new one.
3035 ReplaceValueWith(SDValue(N, 1), Chain);
3036 }
3037
ExpandIntRes_FP_TO_SINT(SDNode * N,SDValue & Lo,SDValue & Hi)3038 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
3039 SDValue &Hi) {
3040 SDLoc dl(N);
3041 EVT VT = N->getValueType(0);
3042
3043 bool IsStrict = N->isStrictFPOpcode();
3044 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3045 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3046 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3047 Op = GetPromotedFloat(Op);
3048
3049 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3050 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3051 Op = GetSoftPromotedHalf(Op);
3052 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3053 }
3054
3055 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
3056 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
3057 TargetLowering::MakeLibCallOptions CallOptions;
3058 CallOptions.setSExt(true);
3059 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3060 CallOptions, dl, Chain);
3061 SplitInteger(Tmp.first, Lo, Hi);
3062
3063 if (IsStrict)
3064 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3065 }
3066
ExpandIntRes_FP_TO_UINT(SDNode * N,SDValue & Lo,SDValue & Hi)3067 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
3068 SDValue &Hi) {
3069 SDLoc dl(N);
3070 EVT VT = N->getValueType(0);
3071
3072 bool IsStrict = N->isStrictFPOpcode();
3073 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3074 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3075 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3076 Op = GetPromotedFloat(Op);
3077
3078 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3079 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3080 Op = GetSoftPromotedHalf(Op);
3081 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3082 }
3083
3084 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
3085 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
3086 TargetLowering::MakeLibCallOptions CallOptions;
3087 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3088 CallOptions, dl, Chain);
3089 SplitInteger(Tmp.first, Lo, Hi);
3090
3091 if (IsStrict)
3092 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3093 }
3094
ExpandIntRes_FP_TO_XINT_SAT(SDNode * N,SDValue & Lo,SDValue & Hi)3095 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3096 SDValue &Hi) {
3097 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3098 SplitInteger(Res, Lo, Hi);
3099 }
3100
ExpandIntRes_LLROUND_LLRINT(SDNode * N,SDValue & Lo,SDValue & Hi)3101 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
3102 SDValue &Hi) {
3103 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
3104
3105 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
3106 "Input type needs to be promoted!");
3107
3108 EVT VT = Op.getValueType();
3109
3110 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3111 if (N->getOpcode() == ISD::LLROUND ||
3112 N->getOpcode() == ISD::STRICT_LLROUND) {
3113 if (VT == MVT::f32)
3114 LC = RTLIB::LLROUND_F32;
3115 else if (VT == MVT::f64)
3116 LC = RTLIB::LLROUND_F64;
3117 else if (VT == MVT::f80)
3118 LC = RTLIB::LLROUND_F80;
3119 else if (VT == MVT::f128)
3120 LC = RTLIB::LLROUND_F128;
3121 else if (VT == MVT::ppcf128)
3122 LC = RTLIB::LLROUND_PPCF128;
3123 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
3124 } else if (N->getOpcode() == ISD::LLRINT ||
3125 N->getOpcode() == ISD::STRICT_LLRINT) {
3126 if (VT == MVT::f32)
3127 LC = RTLIB::LLRINT_F32;
3128 else if (VT == MVT::f64)
3129 LC = RTLIB::LLRINT_F64;
3130 else if (VT == MVT::f80)
3131 LC = RTLIB::LLRINT_F80;
3132 else if (VT == MVT::f128)
3133 LC = RTLIB::LLRINT_F128;
3134 else if (VT == MVT::ppcf128)
3135 LC = RTLIB::LLRINT_PPCF128;
3136 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
3137 } else
3138 llvm_unreachable("Unexpected opcode!");
3139
3140 SDLoc dl(N);
3141 EVT RetVT = N->getValueType(0);
3142 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();
3143
3144 TargetLowering::MakeLibCallOptions CallOptions;
3145 CallOptions.setSExt(true);
3146 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3147 Op, CallOptions, dl,
3148 Chain);
3149 SplitInteger(Tmp.first, Lo, Hi);
3150
3151 if (N->isStrictFPOpcode())
3152 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3153 }
3154
ExpandIntRes_LOAD(LoadSDNode * N,SDValue & Lo,SDValue & Hi)3155 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
3156 SDValue &Lo, SDValue &Hi) {
3157 if (N->isAtomic()) {
3158 // It's typical to have larger CAS than atomic load instructions.
3159 SDLoc dl(N);
3160 EVT VT = N->getMemoryVT();
3161 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3162 SDValue Zero = DAG.getConstant(0, dl, VT);
3163 SDValue Swap = DAG.getAtomicCmpSwap(
3164 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
3165 VT, VTs, N->getOperand(0),
3166 N->getOperand(1), Zero, Zero, N->getMemOperand());
3167 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3168 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3169 return;
3170 }
3171
3172 if (ISD::isNormalLoad(N)) {
3173 ExpandRes_NormalLoad(N, Lo, Hi);
3174 return;
3175 }
3176
3177 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
3178
3179 EVT VT = N->getValueType(0);
3180 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3181 SDValue Ch = N->getChain();
3182 SDValue Ptr = N->getBasePtr();
3183 ISD::LoadExtType ExtType = N->getExtensionType();
3184 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3185 AAMDNodes AAInfo = N->getAAInfo();
3186 SDLoc dl(N);
3187
3188 assert(NVT.isByteSized() && "Expanded type not byte sized!");
3189
3190 if (N->getMemoryVT().bitsLE(NVT)) {
3191 EVT MemVT = N->getMemoryVT();
3192
3193 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
3194 N->getOriginalAlign(), MMOFlags, AAInfo);
3195
3196 // Remember the chain.
3197 Ch = Lo.getValue(1);
3198
3199 if (ExtType == ISD::SEXTLOAD) {
3200 // The high part is obtained by SRA'ing all but one of the bits of the
3201 // lo part.
3202 unsigned LoSize = Lo.getValueSizeInBits();
3203 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3204 DAG.getConstant(LoSize - 1, dl,
3205 TLI.getPointerTy(DAG.getDataLayout())));
3206 } else if (ExtType == ISD::ZEXTLOAD) {
3207 // The high part is just a zero.
3208 Hi = DAG.getConstant(0, dl, NVT);
3209 } else {
3210 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
3211 // The high part is undefined.
3212 Hi = DAG.getUNDEF(NVT);
3213 }
3214 } else if (DAG.getDataLayout().isLittleEndian()) {
3215 // Little-endian - low bits are at low addresses.
3216 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
3217 N->getOriginalAlign(), MMOFlags, AAInfo);
3218
3219 unsigned ExcessBits =
3220 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3221 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3222
3223 // Increment the pointer to the other half.
3224 unsigned IncrementSize = NVT.getSizeInBits()/8;
3225 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3226 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3227 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3228 N->getOriginalAlign(), MMOFlags, AAInfo);
3229
3230 // Build a factor node to remember that this load is independent of the
3231 // other one.
3232 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3233 Hi.getValue(1));
3234 } else {
3235 // Big-endian - high bits are at low addresses. Favor aligned loads at
3236 // the cost of some bit-fiddling.
3237 EVT MemVT = N->getMemoryVT();
3238 unsigned EBytes = MemVT.getStoreSize();
3239 unsigned IncrementSize = NVT.getSizeInBits()/8;
3240 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3241
3242 // Load both the high bits and maybe some of the low bits.
3243 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3244 EVT::getIntegerVT(*DAG.getContext(),
3245 MemVT.getSizeInBits() - ExcessBits),
3246 N->getOriginalAlign(), MMOFlags, AAInfo);
3247
3248 // Increment the pointer to the other half.
3249 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3250 // Load the rest of the low bits.
3251 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3252 N->getPointerInfo().getWithOffset(IncrementSize),
3253 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3254 N->getOriginalAlign(), MMOFlags, AAInfo);
3255
3256 // Build a factor node to remember that this load is independent of the
3257 // other one.
3258 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3259 Hi.getValue(1));
3260
3261 if (ExcessBits < NVT.getSizeInBits()) {
3262 // Transfer low bits from the bottom of Hi to the top of Lo.
3263 Lo = DAG.getNode(
3264 ISD::OR, dl, NVT, Lo,
3265 DAG.getNode(ISD::SHL, dl, NVT, Hi,
3266 DAG.getConstant(ExcessBits, dl,
3267 TLI.getPointerTy(DAG.getDataLayout()))));
3268 // Move high bits to the right position in Hi.
3269 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
3270 Hi,
3271 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3272 TLI.getPointerTy(DAG.getDataLayout())));
3273 }
3274 }
3275
3276 // Legalize the chain result - switch anything that used the old chain to
3277 // use the new one.
3278 ReplaceValueWith(SDValue(N, 1), Ch);
3279 }
3280
ExpandIntRes_Logical(SDNode * N,SDValue & Lo,SDValue & Hi)3281 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
3282 SDValue &Lo, SDValue &Hi) {
3283 SDLoc dl(N);
3284 SDValue LL, LH, RL, RH;
3285 GetExpandedInteger(N->getOperand(0), LL, LH);
3286 GetExpandedInteger(N->getOperand(1), RL, RH);
3287 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
3288 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3289 }
3290
ExpandIntRes_MUL(SDNode * N,SDValue & Lo,SDValue & Hi)3291 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
3292 SDValue &Lo, SDValue &Hi) {
3293 EVT VT = N->getValueType(0);
3294 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3295 SDLoc dl(N);
3296
3297 SDValue LL, LH, RL, RH;
3298 GetExpandedInteger(N->getOperand(0), LL, LH);
3299 GetExpandedInteger(N->getOperand(1), RL, RH);
3300
3301 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
3302 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3303 LL, LH, RL, RH))
3304 return;
3305
3306 // If nothing else, we can make a libcall.
3307 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3308 if (VT == MVT::i16)
3309 LC = RTLIB::MUL_I16;
3310 else if (VT == MVT::i32)
3311 LC = RTLIB::MUL_I32;
3312 else if (VT == MVT::i64)
3313 LC = RTLIB::MUL_I64;
3314 else if (VT == MVT::i128)
3315 LC = RTLIB::MUL_I128;
3316
3317 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
3318 // We'll expand the multiplication by brute force because we have no other
3319 // options. This is a trivially-generalized version of the code from
3320 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
3321 // 4.3.1).
3322 unsigned Bits = NVT.getSizeInBits();
3323 unsigned HalfBits = Bits >> 1;
3324 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
3325 NVT);
3326 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
3327 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
3328
3329 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
3330 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
3331
3332 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3333 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
3334 // The type from TLI is too small to fit the shift amount we want.
3335 // Override it with i32. The shift will have to be legalized.
3336 ShiftAmtTy = MVT::i32;
3337 }
3338 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
3339 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
3340 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
3341 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
3342
3343 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
3344 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
3345 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
3346 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
3347
3348 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
3349 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
3350 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
3351
3352 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
3353 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
3354 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
3355 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
3356 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
3357
3358 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
3359 DAG.getNode(ISD::ADD, dl, NVT,
3360 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
3361 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
3362 return;
3363 }
3364
3365 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3366 TargetLowering::MakeLibCallOptions CallOptions;
3367 CallOptions.setSExt(true);
3368 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3369 Lo, Hi);
3370 }
3371
ExpandIntRes_READCYCLECOUNTER(SDNode * N,SDValue & Lo,SDValue & Hi)3372 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
3373 SDValue &Hi) {
3374 SDLoc DL(N);
3375 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3376 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
3377 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
3378 Lo = R.getValue(0);
3379 Hi = R.getValue(1);
3380 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3381 }
3382
ExpandIntRes_ADDSUBSAT(SDNode * N,SDValue & Lo,SDValue & Hi)3383 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
3384 SDValue &Hi) {
3385 SDValue Result = TLI.expandAddSubSat(N, DAG);
3386 SplitInteger(Result, Lo, Hi);
3387 }
3388
ExpandIntRes_SHLSAT(SDNode * N,SDValue & Lo,SDValue & Hi)3389 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
3390 SDValue &Hi) {
3391 SDValue Result = TLI.expandShlSat(N, DAG);
3392 SplitInteger(Result, Lo, Hi);
3393 }
3394
3395 /// This performs an expansion of the integer result for a fixed point
3396 /// multiplication. The default expansion performs rounding down towards
3397 /// negative infinity, though targets that do care about rounding should specify
3398 /// a target hook for rounding and provide their own expansion or lowering of
3399 /// fixed point multiplication to be consistent with rounding.
ExpandIntRes_MULFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3400 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
3401 SDValue &Hi) {
3402 SDLoc dl(N);
3403 EVT VT = N->getValueType(0);
3404 unsigned VTSize = VT.getScalarSizeInBits();
3405 SDValue LHS = N->getOperand(0);
3406 SDValue RHS = N->getOperand(1);
3407 uint64_t Scale = N->getConstantOperandVal(2);
3408 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
3409 N->getOpcode() == ISD::UMULFIXSAT);
3410 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
3411 N->getOpcode() == ISD::SMULFIXSAT);
3412
3413 // Handle special case when scale is equal to zero.
3414 if (!Scale) {
3415 SDValue Result;
3416 if (!Saturating) {
3417 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3418 } else {
3419 EVT BoolVT = getSetCCResultType(VT);
3420 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
3421 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
3422 SDValue Product = Result.getValue(0);
3423 SDValue Overflow = Result.getValue(1);
3424 if (Signed) {
3425 APInt MinVal = APInt::getSignedMinValue(VTSize);
3426 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
3427 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
3428 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3429 SDValue Zero = DAG.getConstant(0, dl, VT);
3430 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
3431 Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
3432 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
3433 } else {
3434 // For unsigned multiplication, we only need to check the max since we
3435 // can't really overflow towards zero.
3436 APInt MaxVal = APInt::getMaxValue(VTSize);
3437 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3438 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
3439 }
3440 }
3441 SplitInteger(Result, Lo, Hi);
3442 return;
3443 }
3444
3445 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
3446 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
3447 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
3448
3449 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3450 SDValue LL, LH, RL, RH;
3451 GetExpandedInteger(LHS, LL, LH);
3452 GetExpandedInteger(RHS, RL, RH);
3453 SmallVector<SDValue, 4> Result;
3454
3455 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
3456 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
3457 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3458 LL, LH, RL, RH)) {
3459 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
3460 return;
3461 }
3462
3463 unsigned NVTSize = NVT.getScalarSizeInBits();
3464 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
3465 "the size of the current value type");
3466 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3467
3468 // After getting the multiplication result in 4 parts, we need to perform a
3469 // shift right by the amount of the scale to get the result in that scale.
3470 //
3471 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
3472 // 128 bits that are cut into 4 32-bit parts:
3473 //
3474 // HH HL LH LL
3475 // |---32---|---32---|---32---|---32---|
3476 // 128 96 64 32 0
3477 //
3478 // |------VTSize-----|
3479 //
3480 // |NVTSize-|
3481 //
3482 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
3483 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
3484 // amount and get Lo and Hi using two funnel shifts. Or for the special case
3485 // when Scale is a multiple of NVTSize we can just pick the result without
3486 // shifting.
3487 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
3488 if (Scale % NVTSize) {
3489 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
3490 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
3491 ShiftAmount);
3492 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
3493 ShiftAmount);
3494 } else {
3495 Lo = Result[Part0];
3496 Hi = Result[Part0 + 1];
3497 }
3498
3499 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
3500 if (!Saturating)
3501 return;
3502
3503 // Can not overflow when there is no integer part.
3504 if (Scale == VTSize)
3505 return;
3506
3507 // To handle saturation we must check for overflow in the multiplication.
3508 //
3509 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
3510 // aren't all zeroes.
3511 //
3512 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
3513 // aren't all ones or all zeroes.
3514 //
3515 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
3516 // highest bit of HH determines saturation direction in the event of signed
3517 // saturation.
3518
3519 SDValue ResultHL = Result[2];
3520 SDValue ResultHH = Result[3];
3521
3522 SDValue SatMax, SatMin;
3523 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
3524 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3525 EVT BoolNVT = getSetCCResultType(NVT);
3526
3527 if (!Signed) {
3528 if (Scale < NVTSize) {
3529 // Overflow happened if ((HH | (HL >> Scale)) != 0).
3530 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3531 DAG.getConstant(Scale, dl, ShiftTy));
3532 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3533 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3534 } else if (Scale == NVTSize) {
3535 // Overflow happened if (HH != 0).
3536 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3537 } else if (Scale < VTSize) {
3538 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
3539 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3540 DAG.getConstant(Scale - NVTSize, dl,
3541 ShiftTy));
3542 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3543 } else
3544 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
3545 "(and saturation can't happen with Scale==VTSize).");
3546
3547 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3548 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3549 return;
3550 }
3551
3552 if (Scale < NVTSize) {
3553 // The number of overflow bits we can check are VTSize - Scale + 1 (we
3554 // include the sign bit). If these top bits are > 0, then we overflowed past
3555 // the max value. If these top bits are < -1, then we overflowed past the
3556 // min value. Otherwise, we did not overflow.
3557 unsigned OverflowBits = VTSize - Scale + 1;
3558 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3559 "Extent of overflow bits must start within HL");
3560 SDValue HLHiMask = DAG.getConstant(
3561 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3562 SDValue HLLoMask = DAG.getConstant(
3563 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3564 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3565 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3566 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3567 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3568 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3569 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3570 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3571 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3572 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3573 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3574 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3575 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3576 } else if (Scale == NVTSize) {
3577 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3578 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3579 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3580 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3581 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3582 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3583 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3584 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3585 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3586 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3587 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3588 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3589 } else if (Scale < VTSize) {
3590 // This is similar to the case when we saturate if Scale < NVTSize, but we
3591 // only need to check HH.
3592 unsigned OverflowBits = VTSize - Scale + 1;
3593 SDValue HHHiMask = DAG.getConstant(
3594 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3595 SDValue HHLoMask = DAG.getConstant(
3596 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3597 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3598 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3599 } else
3600 llvm_unreachable("Illegal scale for signed fixed point mul.");
3601
3602 // Saturate to signed maximum.
3603 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3604 APInt MaxLo = APInt::getAllOnesValue(NVTSize);
3605 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3606 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3607 // Saturate to signed minimum.
3608 APInt MinHi = APInt::getSignedMinValue(NVTSize);
3609 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3610 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3611 }
3612
ExpandIntRes_DIVFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3613 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
3614 SDValue &Hi) {
3615 SDLoc dl(N);
3616 // Try expanding in the existing type first.
3617 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
3618 N->getOperand(1),
3619 N->getConstantOperandVal(2), DAG);
3620
3621 if (!Res)
3622 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
3623 N->getConstantOperandVal(2), TLI, DAG);
3624 SplitInteger(Res, Lo, Hi);
3625 }
3626
ExpandIntRes_SADDSUBO(SDNode * Node,SDValue & Lo,SDValue & Hi)3627 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3628 SDValue &Lo, SDValue &Hi) {
3629 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
3630 "Node has unexpected Opcode");
3631 SDValue LHS = Node->getOperand(0);
3632 SDValue RHS = Node->getOperand(1);
3633 SDLoc dl(Node);
3634
3635 SDValue Ovf;
3636
3637 bool IsAdd = Node->getOpcode() == ISD::SADDO;
3638 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
3639
3640 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3641 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3642
3643 if (HasCarryOp) {
3644 // Expand the subcomponents.
3645 SDValue LHSL, LHSH, RHSL, RHSH;
3646 GetExpandedInteger(LHS, LHSL, LHSH);
3647 GetExpandedInteger(RHS, RHSL, RHSH);
3648 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
3649
3650 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
3651 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3652
3653 Ovf = Hi.getValue(1);
3654 } else {
3655 // Expand the result by simply replacing it with the equivalent
3656 // non-overflow-checking operation.
3657 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3658 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3659 LHS, RHS);
3660 SplitInteger(Sum, Lo, Hi);
3661
3662 // Compute the overflow.
3663 //
3664 // LHSSign -> LHS < 0
3665 // RHSSign -> RHS < 0
3666 // SumSign -> Sum < 0
3667 //
3668 // Add:
3669 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3670 // Sub:
3671 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3672 //
3673 // To get better codegen we can rewrite this by doing bitwise math on
3674 // the integers and extract the final sign bit at the end. So the
3675 // above becomes:
3676 //
3677 // Add:
3678 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
3679 // Sub:
3680 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
3681 //
3682 // NOTE: This is different than the expansion we do in expandSADDSUBO
3683 // because it is more costly to determine the RHS is > 0 for SSUBO with the
3684 // integers split.
3685 EVT VT = LHS.getValueType();
3686 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
3687 if (IsAdd)
3688 SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
3689
3690 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
3691 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
3692 EVT OType = Node->getValueType(1);
3693 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
3694 }
3695
3696 // Use the calculated overflow everywhere.
3697 ReplaceValueWith(SDValue(Node, 1), Ovf);
3698 }
3699
ExpandIntRes_SDIV(SDNode * N,SDValue & Lo,SDValue & Hi)3700 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3701 SDValue &Lo, SDValue &Hi) {
3702 EVT VT = N->getValueType(0);
3703 SDLoc dl(N);
3704 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3705
3706 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3707 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3708 SplitInteger(Res.getValue(0), Lo, Hi);
3709 return;
3710 }
3711
3712 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3713 if (VT == MVT::i16)
3714 LC = RTLIB::SDIV_I16;
3715 else if (VT == MVT::i32)
3716 LC = RTLIB::SDIV_I32;
3717 else if (VT == MVT::i64)
3718 LC = RTLIB::SDIV_I64;
3719 else if (VT == MVT::i128)
3720 LC = RTLIB::SDIV_I128;
3721 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3722
3723 TargetLowering::MakeLibCallOptions CallOptions;
3724 CallOptions.setSExt(true);
3725 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3726 }
3727
ExpandIntRes_Shift(SDNode * N,SDValue & Lo,SDValue & Hi)3728 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3729 SDValue &Lo, SDValue &Hi) {
3730 EVT VT = N->getValueType(0);
3731 SDLoc dl(N);
3732
3733 // If we can emit an efficient shift operation, do so now. Check to see if
3734 // the RHS is a constant.
3735 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3736 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3737
3738 // If we can determine that the high bit of the shift is zero or one, even if
3739 // the low bits are variable, emit this shift in an optimized form.
3740 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3741 return;
3742
3743 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3744 unsigned PartsOpc;
3745 if (N->getOpcode() == ISD::SHL) {
3746 PartsOpc = ISD::SHL_PARTS;
3747 } else if (N->getOpcode() == ISD::SRL) {
3748 PartsOpc = ISD::SRL_PARTS;
3749 } else {
3750 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3751 PartsOpc = ISD::SRA_PARTS;
3752 }
3753
3754 // Next check to see if the target supports this SHL_PARTS operation or if it
3755 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3756 // size, but create a libcall instead.
3757 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3758 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3759 const bool LegalOrCustom =
3760 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3761 Action == TargetLowering::Custom;
3762
3763 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3764 // Expand the subcomponents.
3765 SDValue LHSL, LHSH;
3766 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3767 EVT VT = LHSL.getValueType();
3768
3769 // If the shift amount operand is coming from a vector legalization it may
3770 // have an illegal type. Fix that first by casting the operand, otherwise
3771 // the new SHL_PARTS operation would need further legalization.
3772 SDValue ShiftOp = N->getOperand(1);
3773 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3774 assert(ShiftTy.getScalarSizeInBits() >=
3775 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3776 "ShiftAmountTy is too small to cover the range of this type!");
3777 if (ShiftOp.getValueType() != ShiftTy)
3778 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3779
3780 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3781 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3782 Hi = Lo.getValue(1);
3783 return;
3784 }
3785
3786 // Otherwise, emit a libcall.
3787 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3788 bool isSigned;
3789 if (N->getOpcode() == ISD::SHL) {
3790 isSigned = false; /*sign irrelevant*/
3791 if (VT == MVT::i16)
3792 LC = RTLIB::SHL_I16;
3793 else if (VT == MVT::i32)
3794 LC = RTLIB::SHL_I32;
3795 else if (VT == MVT::i64)
3796 LC = RTLIB::SHL_I64;
3797 else if (VT == MVT::i128)
3798 LC = RTLIB::SHL_I128;
3799 } else if (N->getOpcode() == ISD::SRL) {
3800 isSigned = false;
3801 if (VT == MVT::i16)
3802 LC = RTLIB::SRL_I16;
3803 else if (VT == MVT::i32)
3804 LC = RTLIB::SRL_I32;
3805 else if (VT == MVT::i64)
3806 LC = RTLIB::SRL_I64;
3807 else if (VT == MVT::i128)
3808 LC = RTLIB::SRL_I128;
3809 } else {
3810 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3811 isSigned = true;
3812 if (VT == MVT::i16)
3813 LC = RTLIB::SRA_I16;
3814 else if (VT == MVT::i32)
3815 LC = RTLIB::SRA_I32;
3816 else if (VT == MVT::i64)
3817 LC = RTLIB::SRA_I64;
3818 else if (VT == MVT::i128)
3819 LC = RTLIB::SRA_I128;
3820 }
3821
3822 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3823 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3824 TargetLowering::MakeLibCallOptions CallOptions;
3825 CallOptions.setSExt(isSigned);
3826 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3827 return;
3828 }
3829
3830 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3831 llvm_unreachable("Unsupported shift!");
3832 }
3833
ExpandIntRes_SIGN_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)3834 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3835 SDValue &Lo, SDValue &Hi) {
3836 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3837 SDLoc dl(N);
3838 SDValue Op = N->getOperand(0);
3839 if (Op.getValueType().bitsLE(NVT)) {
3840 // The low part is sign extension of the input (degenerates to a copy).
3841 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3842 // The high part is obtained by SRA'ing all but one of the bits of low part.
3843 unsigned LoSize = NVT.getSizeInBits();
3844 Hi = DAG.getNode(
3845 ISD::SRA, dl, NVT, Lo,
3846 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3847 } else {
3848 // For example, extension of an i48 to an i64. The operand type necessarily
3849 // promotes to the result type, so will end up being expanded too.
3850 assert(getTypeAction(Op.getValueType()) ==
3851 TargetLowering::TypePromoteInteger &&
3852 "Only know how to promote this result!");
3853 SDValue Res = GetPromotedInteger(Op);
3854 assert(Res.getValueType() == N->getValueType(0) &&
3855 "Operand over promoted?");
3856 // Split the promoted operand. This will simplify when it is expanded.
3857 SplitInteger(Res, Lo, Hi);
3858 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3859 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3860 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3861 ExcessBits)));
3862 }
3863 }
3864
3865 void DAGTypeLegalizer::
ExpandIntRes_SIGN_EXTEND_INREG(SDNode * N,SDValue & Lo,SDValue & Hi)3866 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3867 SDLoc dl(N);
3868 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3869 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3870
3871 if (EVT.bitsLE(Lo.getValueType())) {
3872 // sext_inreg the low part if needed.
3873 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3874 N->getOperand(1));
3875
3876 // The high part gets the sign extension from the lo-part. This handles
3877 // things like sextinreg V:i64 from i8.
3878 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3879 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3880 TLI.getPointerTy(DAG.getDataLayout())));
3881 } else {
3882 // For example, extension of an i48 to an i64. Leave the low part alone,
3883 // sext_inreg the high part.
3884 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3885 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3886 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3887 ExcessBits)));
3888 }
3889 }
3890
ExpandIntRes_SREM(SDNode * N,SDValue & Lo,SDValue & Hi)3891 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3892 SDValue &Lo, SDValue &Hi) {
3893 EVT VT = N->getValueType(0);
3894 SDLoc dl(N);
3895 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3896
3897 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3898 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3899 SplitInteger(Res.getValue(1), Lo, Hi);
3900 return;
3901 }
3902
3903 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3904 if (VT == MVT::i16)
3905 LC = RTLIB::SREM_I16;
3906 else if (VT == MVT::i32)
3907 LC = RTLIB::SREM_I32;
3908 else if (VT == MVT::i64)
3909 LC = RTLIB::SREM_I64;
3910 else if (VT == MVT::i128)
3911 LC = RTLIB::SREM_I128;
3912 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3913
3914 TargetLowering::MakeLibCallOptions CallOptions;
3915 CallOptions.setSExt(true);
3916 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3917 }
3918
ExpandIntRes_TRUNCATE(SDNode * N,SDValue & Lo,SDValue & Hi)3919 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3920 SDValue &Lo, SDValue &Hi) {
3921 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3922 SDLoc dl(N);
3923 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3924 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3925 N->getOperand(0),
3926 DAG.getConstant(NVT.getSizeInBits(), dl,
3927 TLI.getPointerTy(DAG.getDataLayout())));
3928 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3929 }
3930
ExpandIntRes_XMULO(SDNode * N,SDValue & Lo,SDValue & Hi)3931 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3932 SDValue &Lo, SDValue &Hi) {
3933 EVT VT = N->getValueType(0);
3934 SDLoc dl(N);
3935
3936 if (N->getOpcode() == ISD::UMULO) {
3937 // This section expands the operation into the following sequence of
3938 // instructions. `iNh` here refers to a type which has half the bit width of
3939 // the type the original operation operated on.
3940 //
3941 // %0 = %LHS.HI != 0 && %RHS.HI != 0
3942 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3943 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3944 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3945 // %4 = add iNh %1.0, %2.0 as iN
3946 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
3947 //
3948 // %lo = %3.LO
3949 // %hi = %5.0
3950 // %ovf = %0 || %1.1 || %2.1 || %5.1
3951 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3952 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3953 GetExpandedInteger(LHS, LHSLow, LHSHigh);
3954 GetExpandedInteger(RHS, RHSLow, RHSHigh);
3955 EVT HalfVT = LHSLow.getValueType();
3956 EVT BitVT = N->getValueType(1);
3957 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
3958
3959 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3960 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3961 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3962 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3963
3964 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
3965 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3966
3967 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
3968 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3969
3970 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
3971
3972 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3973 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3974 // operation recursively legalized?).
3975 //
3976 // Many backends understand this pattern and will convert into LOHI
3977 // themselves, if applicable.
3978 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3979 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3980 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3981 SplitInteger(Three, Lo, Hi);
3982
3983 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
3984 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
3985 ReplaceValueWith(SDValue(N, 1), Overflow);
3986 return;
3987 }
3988
3989 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3990 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3991 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3992
3993 // Replace this with a libcall that will check overflow.
3994 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3995 if (VT == MVT::i32)
3996 LC = RTLIB::MULO_I32;
3997 else if (VT == MVT::i64)
3998 LC = RTLIB::MULO_I64;
3999 else if (VT == MVT::i128)
4000 LC = RTLIB::MULO_I128;
4001 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
4002
4003 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
4004 // Temporary for the overflow value, default it to zero.
4005 SDValue Chain =
4006 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
4007 MachinePointerInfo());
4008
4009 TargetLowering::ArgListTy Args;
4010 TargetLowering::ArgListEntry Entry;
4011 for (const SDValue &Op : N->op_values()) {
4012 EVT ArgVT = Op.getValueType();
4013 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
4014 Entry.Node = Op;
4015 Entry.Ty = ArgTy;
4016 Entry.IsSExt = true;
4017 Entry.IsZExt = false;
4018 Args.push_back(Entry);
4019 }
4020
4021 // Also pass the address of the overflow check.
4022 Entry.Node = Temp;
4023 Entry.Ty = PtrTy->getPointerTo();
4024 Entry.IsSExt = true;
4025 Entry.IsZExt = false;
4026 Args.push_back(Entry);
4027
4028 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
4029
4030 TargetLowering::CallLoweringInfo CLI(DAG);
4031 CLI.setDebugLoc(dl)
4032 .setChain(Chain)
4033 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
4034 .setSExtResult();
4035
4036 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
4037
4038 SplitInteger(CallInfo.first, Lo, Hi);
4039 SDValue Temp2 =
4040 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
4041 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
4042 DAG.getConstant(0, dl, PtrVT),
4043 ISD::SETNE);
4044 // Use the overflow from the libcall everywhere.
4045 ReplaceValueWith(SDValue(N, 1), Ofl);
4046 }
4047
ExpandIntRes_UDIV(SDNode * N,SDValue & Lo,SDValue & Hi)4048 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
4049 SDValue &Lo, SDValue &Hi) {
4050 EVT VT = N->getValueType(0);
4051 SDLoc dl(N);
4052 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4053
4054 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4055 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4056 SplitInteger(Res.getValue(0), Lo, Hi);
4057 return;
4058 }
4059
4060 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4061 if (VT == MVT::i16)
4062 LC = RTLIB::UDIV_I16;
4063 else if (VT == MVT::i32)
4064 LC = RTLIB::UDIV_I32;
4065 else if (VT == MVT::i64)
4066 LC = RTLIB::UDIV_I64;
4067 else if (VT == MVT::i128)
4068 LC = RTLIB::UDIV_I128;
4069 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
4070
4071 TargetLowering::MakeLibCallOptions CallOptions;
4072 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4073 }
4074
ExpandIntRes_UREM(SDNode * N,SDValue & Lo,SDValue & Hi)4075 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
4076 SDValue &Lo, SDValue &Hi) {
4077 EVT VT = N->getValueType(0);
4078 SDLoc dl(N);
4079 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4080
4081 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4082 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4083 SplitInteger(Res.getValue(1), Lo, Hi);
4084 return;
4085 }
4086
4087 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4088 if (VT == MVT::i16)
4089 LC = RTLIB::UREM_I16;
4090 else if (VT == MVT::i32)
4091 LC = RTLIB::UREM_I32;
4092 else if (VT == MVT::i64)
4093 LC = RTLIB::UREM_I64;
4094 else if (VT == MVT::i128)
4095 LC = RTLIB::UREM_I128;
4096 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
4097
4098 TargetLowering::MakeLibCallOptions CallOptions;
4099 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4100 }
4101
ExpandIntRes_ZERO_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)4102 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
4103 SDValue &Lo, SDValue &Hi) {
4104 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4105 SDLoc dl(N);
4106 SDValue Op = N->getOperand(0);
4107 if (Op.getValueType().bitsLE(NVT)) {
4108 // The low part is zero extension of the input (degenerates to a copy).
4109 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
4110 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
4111 } else {
4112 // For example, extension of an i48 to an i64. The operand type necessarily
4113 // promotes to the result type, so will end up being expanded too.
4114 assert(getTypeAction(Op.getValueType()) ==
4115 TargetLowering::TypePromoteInteger &&
4116 "Only know how to promote this result!");
4117 SDValue Res = GetPromotedInteger(Op);
4118 assert(Res.getValueType() == N->getValueType(0) &&
4119 "Operand over promoted?");
4120 // Split the promoted operand. This will simplify when it is expanded.
4121 SplitInteger(Res, Lo, Hi);
4122 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4123 Hi = DAG.getZeroExtendInReg(Hi, dl,
4124 EVT::getIntegerVT(*DAG.getContext(),
4125 ExcessBits));
4126 }
4127 }
4128
ExpandIntRes_ATOMIC_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)4129 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
4130 SDValue &Lo, SDValue &Hi) {
4131 SDLoc dl(N);
4132 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
4133 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
4134 SDValue Zero = DAG.getConstant(0, dl, VT);
4135 SDValue Swap = DAG.getAtomicCmpSwap(
4136 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
4137 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
4138 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
4139
4140 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
4141 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
4142 }
4143
ExpandIntRes_VECREDUCE(SDNode * N,SDValue & Lo,SDValue & Hi)4144 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
4145 SDValue &Lo, SDValue &Hi) {
4146 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
4147 // both halves independently.
4148 SDValue Res = TLI.expandVecReduce(N, DAG);
4149 SplitInteger(Res, Lo, Hi);
4150 }
4151
ExpandIntRes_Rotate(SDNode * N,SDValue & Lo,SDValue & Hi)4152 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
4153 SDValue &Lo, SDValue &Hi) {
4154 // Lower the rotate to shifts and ORs which can be expanded.
4155 SDValue Res;
4156 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
4157 SplitInteger(Res, Lo, Hi);
4158 }
4159
ExpandIntRes_FunnelShift(SDNode * N,SDValue & Lo,SDValue & Hi)4160 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N,
4161 SDValue &Lo, SDValue &Hi) {
4162 // Lower the funnel shift to shifts and ORs which can be expanded.
4163 SDValue Res;
4164 TLI.expandFunnelShift(N, Res, DAG);
4165 SplitInteger(Res, Lo, Hi);
4166 }
4167
4168 //===----------------------------------------------------------------------===//
4169 // Integer Operand Expansion
4170 //===----------------------------------------------------------------------===//
4171
4172 /// ExpandIntegerOperand - This method is called when the specified operand of
4173 /// the specified node is found to need expansion. At this point, all of the
4174 /// result types of the node are known to be legal, but other operands of the
4175 /// node may need promotion or expansion as well as the specified one.
ExpandIntegerOperand(SDNode * N,unsigned OpNo)4176 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
4177 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
4178 dbgs() << "\n");
4179 SDValue Res = SDValue();
4180
4181 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4182 return false;
4183
4184 switch (N->getOpcode()) {
4185 default:
4186 #ifndef NDEBUG
4187 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
4188 N->dump(&DAG); dbgs() << "\n";
4189 #endif
4190 report_fatal_error("Do not know how to expand this operator's operand!");
4191
4192 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
4193 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
4194 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
4195 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
4196 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
4197 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
4198 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
4199 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
4200 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
4201 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
4202 case ISD::STRICT_SINT_TO_FP:
4203 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
4204 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
4205 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
4206 case ISD::STRICT_UINT_TO_FP:
4207 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
4208
4209 case ISD::SHL:
4210 case ISD::SRA:
4211 case ISD::SRL:
4212 case ISD::ROTL:
4213 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
4214 case ISD::RETURNADDR:
4215 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
4216
4217 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
4218 }
4219
4220 // If the result is null, the sub-method took care of registering results etc.
4221 if (!Res.getNode()) return false;
4222
4223 // If the result is N, the sub-method updated N in place. Tell the legalizer
4224 // core about this.
4225 if (Res.getNode() == N)
4226 return true;
4227
4228 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4229 "Invalid operand expansion");
4230
4231 ReplaceValueWith(SDValue(N, 0), Res);
4232 return false;
4233 }
4234
4235 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
4236 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
IntegerExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl)4237 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
4238 SDValue &NewRHS,
4239 ISD::CondCode &CCCode,
4240 const SDLoc &dl) {
4241 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4242 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
4243 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
4244
4245 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
4246 if (RHSLo == RHSHi) {
4247 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
4248 if (RHSCST->isAllOnesValue()) {
4249 // Equality comparison to -1.
4250 NewLHS = DAG.getNode(ISD::AND, dl,
4251 LHSLo.getValueType(), LHSLo, LHSHi);
4252 NewRHS = RHSLo;
4253 return;
4254 }
4255 }
4256 }
4257
4258 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
4259 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
4260 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
4261 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
4262 return;
4263 }
4264
4265 // If this is a comparison of the sign bit, just look at the top part.
4266 // X > -1, x < 0
4267 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
4268 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
4269 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
4270 NewLHS = LHSHi;
4271 NewRHS = RHSHi;
4272 return;
4273 }
4274
4275 // FIXME: This generated code sucks.
4276 ISD::CondCode LowCC;
4277 switch (CCCode) {
4278 default: llvm_unreachable("Unknown integer setcc!");
4279 case ISD::SETLT:
4280 case ISD::SETULT: LowCC = ISD::SETULT; break;
4281 case ISD::SETGT:
4282 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
4283 case ISD::SETLE:
4284 case ISD::SETULE: LowCC = ISD::SETULE; break;
4285 case ISD::SETGE:
4286 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
4287 }
4288
4289 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
4290 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
4291 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
4292
4293 // NOTE: on targets without efficient SELECT of bools, we can always use
4294 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
4295 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
4296 nullptr);
4297 SDValue LoCmp, HiCmp;
4298 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
4299 TLI.isTypeLegal(RHSLo.getValueType()))
4300 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
4301 RHSLo, LowCC, false, DagCombineInfo, dl);
4302 if (!LoCmp.getNode())
4303 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
4304 RHSLo, LowCC);
4305 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
4306 TLI.isTypeLegal(RHSHi.getValueType()))
4307 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
4308 RHSHi, CCCode, false, DagCombineInfo, dl);
4309 if (!HiCmp.getNode())
4310 HiCmp =
4311 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
4312 LHSHi, RHSHi, DAG.getCondCode(CCCode));
4313
4314 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
4315 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
4316
4317 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
4318 CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
4319
4320 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
4321 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
4322 (LoCmpC && LoCmpC->isNullValue())))) {
4323 // For LE / GE, if high part is known false, ignore the low part.
4324 // For LT / GT: if low part is known false, return the high part.
4325 // if high part is known true, ignore the low part.
4326 NewLHS = HiCmp;
4327 NewRHS = SDValue();
4328 return;
4329 }
4330
4331 if (LHSHi == RHSHi) {
4332 // Comparing the low bits is enough.
4333 NewLHS = LoCmp;
4334 NewRHS = SDValue();
4335 return;
4336 }
4337
4338 // Lower with SETCCCARRY if the target supports it.
4339 EVT HiVT = LHSHi.getValueType();
4340 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
4341 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
4342
4343 // FIXME: Make all targets support this, then remove the other lowering.
4344 if (HasSETCCCARRY) {
4345 // SETCCCARRY can detect < and >= directly. For > and <=, flip
4346 // operands and condition code.
4347 bool FlipOperands = false;
4348 switch (CCCode) {
4349 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
4350 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
4351 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
4352 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
4353 default: break;
4354 }
4355 if (FlipOperands) {
4356 std::swap(LHSLo, RHSLo);
4357 std::swap(LHSHi, RHSHi);
4358 }
4359 // Perform a wide subtraction, feeding the carry from the low part into
4360 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
4361 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
4362 // zero or positive iff LHS >= RHS.
4363 EVT LoVT = LHSLo.getValueType();
4364 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
4365 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
4366 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
4367 LHSHi, RHSHi, LowCmp.getValue(1),
4368 DAG.getCondCode(CCCode));
4369 NewLHS = Res;
4370 NewRHS = SDValue();
4371 return;
4372 }
4373
4374 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
4375 false, DagCombineInfo, dl);
4376 if (!NewLHS.getNode())
4377 NewLHS =
4378 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
4379 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
4380 NewRHS = SDValue();
4381 }
4382
ExpandIntOp_BR_CC(SDNode * N)4383 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
4384 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
4385 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
4386 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4387
4388 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4389 // against zero to select between true and false values.
4390 if (!NewRHS.getNode()) {
4391 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4392 CCCode = ISD::SETNE;
4393 }
4394
4395 // Update N to have the operands specified.
4396 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
4397 DAG.getCondCode(CCCode), NewLHS, NewRHS,
4398 N->getOperand(4)), 0);
4399 }
4400
ExpandIntOp_SELECT_CC(SDNode * N)4401 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
4402 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4403 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
4404 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4405
4406 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4407 // against zero to select between true and false values.
4408 if (!NewRHS.getNode()) {
4409 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4410 CCCode = ISD::SETNE;
4411 }
4412
4413 // Update N to have the operands specified.
4414 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
4415 N->getOperand(2), N->getOperand(3),
4416 DAG.getCondCode(CCCode)), 0);
4417 }
4418
ExpandIntOp_SETCC(SDNode * N)4419 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
4420 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4421 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
4422 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4423
4424 // If ExpandSetCCOperands returned a scalar, use it.
4425 if (!NewRHS.getNode()) {
4426 assert(NewLHS.getValueType() == N->getValueType(0) &&
4427 "Unexpected setcc expansion!");
4428 return NewLHS;
4429 }
4430
4431 // Otherwise, update N to have the operands specified.
4432 return SDValue(
4433 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
4434 }
4435
ExpandIntOp_SETCCCARRY(SDNode * N)4436 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
4437 SDValue LHS = N->getOperand(0);
4438 SDValue RHS = N->getOperand(1);
4439 SDValue Carry = N->getOperand(2);
4440 SDValue Cond = N->getOperand(3);
4441 SDLoc dl = SDLoc(N);
4442
4443 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4444 GetExpandedInteger(LHS, LHSLo, LHSHi);
4445 GetExpandedInteger(RHS, RHSLo, RHSHi);
4446
4447 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
4448 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
4449 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
4450 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
4451 LowCmp.getValue(1), Cond);
4452 }
4453
ExpandIntOp_SPLAT_VECTOR(SDNode * N)4454 SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
4455 // Split the operand and replace with SPLAT_VECTOR_PARTS.
4456 SDValue Lo, Hi;
4457 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4458 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo,
4459 Hi);
4460 }
4461
ExpandIntOp_Shift(SDNode * N)4462 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
4463 // The value being shifted is legal, but the shift amount is too big.
4464 // It follows that either the result of the shift is undefined, or the
4465 // upper half of the shift amount is zero. Just use the lower half.
4466 SDValue Lo, Hi;
4467 GetExpandedInteger(N->getOperand(1), Lo, Hi);
4468 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
4469 }
4470
ExpandIntOp_RETURNADDR(SDNode * N)4471 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
4472 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
4473 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
4474 // constant to valid type.
4475 SDValue Lo, Hi;
4476 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4477 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
4478 }
4479
ExpandIntOp_SINT_TO_FP(SDNode * N)4480 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
4481 bool IsStrict = N->isStrictFPOpcode();
4482 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4483 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4484 EVT DstVT = N->getValueType(0);
4485 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
4486 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4487 "Don't know how to expand this SINT_TO_FP!");
4488 TargetLowering::MakeLibCallOptions CallOptions;
4489 CallOptions.setSExt(true);
4490 std::pair<SDValue, SDValue> Tmp =
4491 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4492
4493 if (!IsStrict)
4494 return Tmp.first;
4495
4496 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4497 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4498 return SDValue();
4499 }
4500
ExpandIntOp_STORE(StoreSDNode * N,unsigned OpNo)4501 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
4502 if (N->isAtomic()) {
4503 // It's typical to have larger CAS than atomic store instructions.
4504 SDLoc dl(N);
4505 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4506 N->getMemoryVT(),
4507 N->getOperand(0), N->getOperand(2),
4508 N->getOperand(1),
4509 N->getMemOperand());
4510 return Swap.getValue(1);
4511 }
4512 if (ISD::isNormalStore(N))
4513 return ExpandOp_NormalStore(N, OpNo);
4514
4515 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
4516 assert(OpNo == 1 && "Can only expand the stored value so far");
4517
4518 EVT VT = N->getOperand(1).getValueType();
4519 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4520 SDValue Ch = N->getChain();
4521 SDValue Ptr = N->getBasePtr();
4522 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4523 AAMDNodes AAInfo = N->getAAInfo();
4524 SDLoc dl(N);
4525 SDValue Lo, Hi;
4526
4527 assert(NVT.isByteSized() && "Expanded type not byte sized!");
4528
4529 if (N->getMemoryVT().bitsLE(NVT)) {
4530 GetExpandedInteger(N->getValue(), Lo, Hi);
4531 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4532 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags,
4533 AAInfo);
4534 }
4535
4536 if (DAG.getDataLayout().isLittleEndian()) {
4537 // Little-endian - low bits are at low addresses.
4538 GetExpandedInteger(N->getValue(), Lo, Hi);
4539
4540 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4541 N->getOriginalAlign(), MMOFlags, AAInfo);
4542
4543 unsigned ExcessBits =
4544 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4545 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
4546
4547 // Increment the pointer to the other half.
4548 unsigned IncrementSize = NVT.getSizeInBits()/8;
4549 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
4550 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
4551 N->getPointerInfo().getWithOffset(IncrementSize),
4552 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo);
4553 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4554 }
4555
4556 // Big-endian - high bits are at low addresses. Favor aligned stores at
4557 // the cost of some bit-fiddling.
4558 GetExpandedInteger(N->getValue(), Lo, Hi);
4559
4560 EVT ExtVT = N->getMemoryVT();
4561 unsigned EBytes = ExtVT.getStoreSize();
4562 unsigned IncrementSize = NVT.getSizeInBits()/8;
4563 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4564 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
4565 ExtVT.getSizeInBits() - ExcessBits);
4566
4567 if (ExcessBits < NVT.getSizeInBits()) {
4568 // Transfer high bits from the top of Lo to the bottom of Hi.
4569 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
4570 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
4571 TLI.getPointerTy(DAG.getDataLayout())));
4572 Hi = DAG.getNode(
4573 ISD::OR, dl, NVT, Hi,
4574 DAG.getNode(ISD::SRL, dl, NVT, Lo,
4575 DAG.getConstant(ExcessBits, dl,
4576 TLI.getPointerTy(DAG.getDataLayout()))));
4577 }
4578
4579 // Store both the high bits and maybe some of the low bits.
4580 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
4581 N->getOriginalAlign(), MMOFlags, AAInfo);
4582
4583 // Increment the pointer to the other half.
4584 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
4585 // Store the lowest ExcessBits bits in the second half.
4586 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
4587 N->getPointerInfo().getWithOffset(IncrementSize),
4588 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
4589 N->getOriginalAlign(), MMOFlags, AAInfo);
4590 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4591 }
4592
ExpandIntOp_TRUNCATE(SDNode * N)4593 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
4594 SDValue InL, InH;
4595 GetExpandedInteger(N->getOperand(0), InL, InH);
4596 // Just truncate the low part of the source.
4597 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
4598 }
4599
ExpandIntOp_UINT_TO_FP(SDNode * N)4600 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
4601 bool IsStrict = N->isStrictFPOpcode();
4602 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4603 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4604 EVT DstVT = N->getValueType(0);
4605 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
4606 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4607 "Don't know how to expand this UINT_TO_FP!");
4608 TargetLowering::MakeLibCallOptions CallOptions;
4609 CallOptions.setSExt(true);
4610 std::pair<SDValue, SDValue> Tmp =
4611 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4612
4613 if (!IsStrict)
4614 return Tmp.first;
4615
4616 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4617 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4618 return SDValue();
4619 }
4620
ExpandIntOp_ATOMIC_STORE(SDNode * N)4621 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
4622 SDLoc dl(N);
4623 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4624 cast<AtomicSDNode>(N)->getMemoryVT(),
4625 N->getOperand(0),
4626 N->getOperand(1), N->getOperand(2),
4627 cast<AtomicSDNode>(N)->getMemOperand());
4628 return Swap.getValue(1);
4629 }
4630
PromoteIntRes_VECTOR_SPLICE(SDNode * N)4631 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
4632 SDLoc dl(N);
4633
4634 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4635 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4636 EVT OutVT = V0.getValueType();
4637
4638 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2));
4639 }
4640
PromoteIntRes_EXTRACT_SUBVECTOR(SDNode * N)4641 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4642
4643 EVT OutVT = N->getValueType(0);
4644 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4645 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4646 EVT NOutVTElem = NOutVT.getVectorElementType();
4647
4648 SDLoc dl(N);
4649 SDValue BaseIdx = N->getOperand(1);
4650
4651 // TODO: We may be able to use this for types other than scalable
4652 // vectors and fix those tests that expect BUILD_VECTOR to be used
4653 if (OutVT.isScalableVector()) {
4654 SDValue InOp0 = N->getOperand(0);
4655 EVT InVT = InOp0.getValueType();
4656
4657 // Promote operands and see if this is handled by target lowering,
4658 // Otherwise, use the BUILD_VECTOR approach below
4659 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
4660 // Collect the (promoted) operands
4661 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
4662
4663 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
4664 assert(PromEltVT.bitsLE(NOutVTElem) &&
4665 "Promoted operand has an element type greater than result");
4666
4667 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
4668 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
4669 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
4670 }
4671 }
4672
4673 if (OutVT.isScalableVector())
4674 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
4675
4676 SDValue InOp0 = N->getOperand(0);
4677 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
4678 InOp0 = GetPromotedInteger(N->getOperand(0));
4679
4680 EVT InVT = InOp0.getValueType();
4681
4682 unsigned OutNumElems = OutVT.getVectorNumElements();
4683 SmallVector<SDValue, 8> Ops;
4684 Ops.reserve(OutNumElems);
4685 for (unsigned i = 0; i != OutNumElems; ++i) {
4686
4687 // Extract the element from the original vector.
4688 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4689 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4690 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4691 InVT.getVectorElementType(), N->getOperand(0), Index);
4692
4693 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4694 // Insert the converted element to the new vector.
4695 Ops.push_back(Op);
4696 }
4697
4698 return DAG.getBuildVector(NOutVT, dl, Ops);
4699 }
4700
PromoteIntRes_VECTOR_REVERSE(SDNode * N)4701 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
4702 SDLoc dl(N);
4703
4704 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4705 EVT OutVT = V0.getValueType();
4706
4707 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0);
4708 }
4709
PromoteIntRes_VECTOR_SHUFFLE(SDNode * N)4710 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4711 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4712 EVT VT = N->getValueType(0);
4713 SDLoc dl(N);
4714
4715 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4716
4717 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4718 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4719 EVT OutVT = V0.getValueType();
4720
4721 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4722 }
4723
4724
PromoteIntRes_BUILD_VECTOR(SDNode * N)4725 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4726 EVT OutVT = N->getValueType(0);
4727 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4728 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4729 unsigned NumElems = N->getNumOperands();
4730 EVT NOutVTElem = NOutVT.getVectorElementType();
4731
4732 SDLoc dl(N);
4733
4734 SmallVector<SDValue, 8> Ops;
4735 Ops.reserve(NumElems);
4736 for (unsigned i = 0; i != NumElems; ++i) {
4737 SDValue Op;
4738 // BUILD_VECTOR integer operand types are allowed to be larger than the
4739 // result's element type. This may still be true after the promotion. For
4740 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4741 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4742 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4743 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4744 else
4745 Op = N->getOperand(i);
4746 Ops.push_back(Op);
4747 }
4748
4749 return DAG.getBuildVector(NOutVT, dl, Ops);
4750 }
4751
PromoteIntRes_SCALAR_TO_VECTOR(SDNode * N)4752 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4753
4754 SDLoc dl(N);
4755
4756 assert(!N->getOperand(0).getValueType().isVector() &&
4757 "Input must be a scalar");
4758
4759 EVT OutVT = N->getValueType(0);
4760 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4761 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4762 EVT NOutVTElem = NOutVT.getVectorElementType();
4763
4764 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4765
4766 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4767 }
4768
PromoteIntRes_SPLAT_VECTOR(SDNode * N)4769 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
4770 SDLoc dl(N);
4771
4772 SDValue SplatVal = N->getOperand(0);
4773
4774 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar");
4775
4776 EVT OutVT = N->getValueType(0);
4777 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4778 assert(NOutVT.isVector() && "Type must be promoted to a vector type");
4779 EVT NOutElemVT = NOutVT.getVectorElementType();
4780
4781 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
4782
4783 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
4784 }
4785
PromoteIntRes_STEP_VECTOR(SDNode * N)4786 SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
4787 SDLoc dl(N);
4788 EVT OutVT = N->getValueType(0);
4789 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4790 assert(NOutVT.isVector() && "Type must be promoted to a vector type");
4791 EVT NOutElemVT = TLI.getTypeToTransformTo(*DAG.getContext(),
4792 NOutVT.getVectorElementType());
4793 APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
4794 SDValue Step = DAG.getConstant(StepVal.getSExtValue(), dl, NOutElemVT);
4795 return DAG.getStepVector(dl, NOutVT, Step);
4796 }
4797
PromoteIntRes_CONCAT_VECTORS(SDNode * N)4798 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4799 SDLoc dl(N);
4800
4801 EVT OutVT = N->getValueType(0);
4802 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4803 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4804
4805 EVT OutElemTy = NOutVT.getVectorElementType();
4806
4807 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4808 unsigned NumOutElem = NOutVT.getVectorNumElements();
4809 unsigned NumOperands = N->getNumOperands();
4810 assert(NumElem * NumOperands == NumOutElem &&
4811 "Unexpected number of elements");
4812
4813 // Take the elements from the first vector.
4814 SmallVector<SDValue, 8> Ops(NumOutElem);
4815 for (unsigned i = 0; i < NumOperands; ++i) {
4816 SDValue Op = N->getOperand(i);
4817 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4818 Op = GetPromotedInteger(Op);
4819 EVT SclrTy = Op.getValueType().getVectorElementType();
4820 assert(NumElem == Op.getValueType().getVectorNumElements() &&
4821 "Unexpected number of elements");
4822
4823 for (unsigned j = 0; j < NumElem; ++j) {
4824 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4825 DAG.getVectorIdxConstant(j, dl));
4826 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4827 }
4828 }
4829
4830 return DAG.getBuildVector(NOutVT, dl, Ops);
4831 }
4832
PromoteIntRes_EXTEND_VECTOR_INREG(SDNode * N)4833 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4834 EVT VT = N->getValueType(0);
4835 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4836 assert(NVT.isVector() && "This type must be promoted to a vector type");
4837
4838 SDLoc dl(N);
4839
4840 // For operands whose TypeAction is to promote, extend the promoted node
4841 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4842 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4843 // type..
4844 if (getTypeAction(N->getOperand(0).getValueType())
4845 == TargetLowering::TypePromoteInteger) {
4846 SDValue Promoted;
4847
4848 switch(N->getOpcode()) {
4849 case ISD::SIGN_EXTEND_VECTOR_INREG:
4850 Promoted = SExtPromotedInteger(N->getOperand(0));
4851 break;
4852 case ISD::ZERO_EXTEND_VECTOR_INREG:
4853 Promoted = ZExtPromotedInteger(N->getOperand(0));
4854 break;
4855 case ISD::ANY_EXTEND_VECTOR_INREG:
4856 Promoted = GetPromotedInteger(N->getOperand(0));
4857 break;
4858 default:
4859 llvm_unreachable("Node has unexpected Opcode");
4860 }
4861 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4862 }
4863
4864 // Directly extend to the appropriate transform-to type.
4865 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4866 }
4867
PromoteIntRes_INSERT_VECTOR_ELT(SDNode * N)4868 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4869 EVT OutVT = N->getValueType(0);
4870 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4871 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4872
4873 EVT NOutVTElem = NOutVT.getVectorElementType();
4874
4875 SDLoc dl(N);
4876 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4877
4878 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4879 NOutVTElem, N->getOperand(1));
4880 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4881 V0, ConvElem, N->getOperand(2));
4882 }
4883
PromoteIntRes_VECREDUCE(SDNode * N)4884 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4885 // The VECREDUCE result size may be larger than the element size, so
4886 // we can simply change the result type.
4887 SDLoc dl(N);
4888 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4889 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4890 }
4891
PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode * N)4892 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4893 SDLoc dl(N);
4894 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4895 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4896 TLI.getVectorIdxTy(DAG.getDataLayout()));
4897 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4898 V0->getValueType(0).getScalarType(), V0, V1);
4899
4900 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4901 // element types. If this is the case then we need to expand the outgoing
4902 // value and not truncate it.
4903 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4904 }
4905
PromoteIntOp_EXTRACT_SUBVECTOR(SDNode * N)4906 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4907 SDLoc dl(N);
4908 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4909 MVT InVT = V0.getValueType().getSimpleVT();
4910 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4911 N->getValueType(0).getVectorNumElements());
4912 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4913 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4914 }
4915
PromoteIntOp_CONCAT_VECTORS(SDNode * N)4916 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4917 SDLoc dl(N);
4918
4919 EVT ResVT = N->getValueType(0);
4920 unsigned NumElems = N->getNumOperands();
4921
4922 if (ResVT.isScalableVector()) {
4923 SDValue ResVec = DAG.getUNDEF(ResVT);
4924
4925 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
4926 SDValue Op = N->getOperand(OpIdx);
4927 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
4928 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
4929 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
4930 }
4931
4932 return ResVec;
4933 }
4934
4935 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4936
4937 SmallVector<SDValue, 8> NewOps;
4938 NewOps.reserve(NumElems);
4939
4940 // For each incoming vector
4941 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
4942 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4943 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4944 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4945
4946 for (unsigned i=0; i<NumElem; ++i) {
4947 // Extract element from incoming vector
4948 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4949 DAG.getVectorIdxConstant(i, dl));
4950 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4951 NewOps.push_back(Tr);
4952 }
4953 }
4954
4955 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4956 }
4957