1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include "llvm/Support/KnownBits.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
PromoteIntegerResult(SDNode * N,unsigned ResNo)37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39 dbgs() << "\n");
40 SDValue Res = SDValue();
41
42 // See if the target wants to custom expand this node.
43 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45 return;
46 }
47
48 switch (N->getOpcode()) {
49 default:
50 #ifndef NDEBUG
51 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52 N->dump(&DAG); dbgs() << "\n";
53 #endif
54 llvm_unreachable("Do not know how to promote this operator!");
55 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
63 case ISD::CTLZ_ZERO_UNDEF:
64 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
66 case ISD::CTTZ_ZERO_UNDEF:
67 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
68 case ISD::EXTRACT_VECTOR_ELT:
69 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72 break;
73 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74 break;
75 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78 case ISD::STRICT_FSETCC:
79 case ISD::STRICT_FSETCCS:
80 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
81 case ISD::SMIN:
82 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
83 case ISD::UMIN:
84 case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
85
86 case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
87 case ISD::SIGN_EXTEND_INREG:
88 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
89 case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
90 case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
91 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
92 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
93 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
94 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
95
96 case ISD::EXTRACT_SUBVECTOR:
97 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
98 case ISD::VECTOR_SHUFFLE:
99 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
100 case ISD::INSERT_VECTOR_ELT:
101 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
102 case ISD::BUILD_VECTOR:
103 Res = PromoteIntRes_BUILD_VECTOR(N); break;
104 case ISD::SCALAR_TO_VECTOR:
105 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
106 case ISD::SPLAT_VECTOR:
107 Res = PromoteIntRes_SPLAT_VECTOR(N); break;
108 case ISD::CONCAT_VECTORS:
109 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
110
111 case ISD::ANY_EXTEND_VECTOR_INREG:
112 case ISD::SIGN_EXTEND_VECTOR_INREG:
113 case ISD::ZERO_EXTEND_VECTOR_INREG:
114 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
115
116 case ISD::SIGN_EXTEND:
117 case ISD::ZERO_EXTEND:
118 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
119
120 case ISD::STRICT_FP_TO_SINT:
121 case ISD::STRICT_FP_TO_UINT:
122 case ISD::FP_TO_SINT:
123 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
124
125 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
126
127 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
128
129 case ISD::AND:
130 case ISD::OR:
131 case ISD::XOR:
132 case ISD::ADD:
133 case ISD::SUB:
134 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
135
136 case ISD::SDIV:
137 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
138
139 case ISD::UDIV:
140 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
141
142 case ISD::SADDO:
143 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
144 case ISD::UADDO:
145 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
146 case ISD::SMULO:
147 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
148
149 case ISD::ADDE:
150 case ISD::SUBE:
151 case ISD::ADDCARRY:
152 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
153
154 case ISD::SADDSAT:
155 case ISD::UADDSAT:
156 case ISD::SSUBSAT:
157 case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
158
159 case ISD::SMULFIX:
160 case ISD::SMULFIXSAT:
161 case ISD::UMULFIX:
162 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
163
164 case ISD::SDIVFIX:
165 case ISD::SDIVFIXSAT:
166 case ISD::UDIVFIX:
167 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
168
169 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
170
171 case ISD::PTRTOINT: Res = PromoteIntRes_PTRTOINT(N); break;
172
173 case ISD::ATOMIC_LOAD:
174 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
175
176 case ISD::ATOMIC_LOAD_ADD:
177 case ISD::ATOMIC_LOAD_SUB:
178 case ISD::ATOMIC_LOAD_AND:
179 case ISD::ATOMIC_LOAD_CLR:
180 case ISD::ATOMIC_LOAD_OR:
181 case ISD::ATOMIC_LOAD_XOR:
182 case ISD::ATOMIC_LOAD_NAND:
183 case ISD::ATOMIC_LOAD_MIN:
184 case ISD::ATOMIC_LOAD_MAX:
185 case ISD::ATOMIC_LOAD_UMIN:
186 case ISD::ATOMIC_LOAD_UMAX:
187 case ISD::ATOMIC_SWAP:
188 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
189
190 case ISD::ATOMIC_CMP_SWAP:
191 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
192 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
193 break;
194
195 case ISD::VECREDUCE_ADD:
196 case ISD::VECREDUCE_MUL:
197 case ISD::VECREDUCE_AND:
198 case ISD::VECREDUCE_OR:
199 case ISD::VECREDUCE_XOR:
200 case ISD::VECREDUCE_SMAX:
201 case ISD::VECREDUCE_SMIN:
202 case ISD::VECREDUCE_UMAX:
203 case ISD::VECREDUCE_UMIN:
204 Res = PromoteIntRes_VECREDUCE(N);
205 break;
206
207 case ISD::FREEZE:
208 Res = PromoteIntRes_FREEZE(N);
209 break;
210 }
211
212 // If the result is null then the sub-method took care of registering it.
213 if (Res.getNode())
214 SetPromotedInteger(SDValue(N, ResNo), Res);
215 }
216
PromoteIntRes_MERGE_VALUES(SDNode * N,unsigned ResNo)217 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
218 unsigned ResNo) {
219 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
220 return GetPromotedInteger(Op);
221 }
222
PromoteIntRes_AssertSext(SDNode * N)223 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
224 // Sign-extend the new bits, and continue the assertion.
225 SDValue Op = SExtPromotedInteger(N->getOperand(0));
226 return DAG.getNode(ISD::AssertSext, SDLoc(N),
227 Op.getValueType(), Op, N->getOperand(1));
228 }
229
PromoteIntRes_AssertZext(SDNode * N)230 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
231 // Zero the new bits, and continue the assertion.
232 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
233 return DAG.getNode(ISD::AssertZext, SDLoc(N),
234 Op.getValueType(), Op, N->getOperand(1));
235 }
236
PromoteIntRes_Atomic0(AtomicSDNode * N)237 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
238 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
239 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
240 N->getMemoryVT(), ResVT,
241 N->getChain(), N->getBasePtr(),
242 N->getMemOperand());
243 // Legalize the chain result - switch anything that used the old chain to
244 // use the new one.
245 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
246 return Res;
247 }
248
PromoteIntRes_Atomic1(AtomicSDNode * N)249 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
250 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
251 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
252 N->getMemoryVT(),
253 N->getChain(), N->getBasePtr(),
254 Op2, N->getMemOperand());
255 // Legalize the chain result - switch anything that used the old chain to
256 // use the new one.
257 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
258 return Res;
259 }
260
PromoteIntRes_AtomicCmpSwap(AtomicSDNode * N,unsigned ResNo)261 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
262 unsigned ResNo) {
263 if (ResNo == 1) {
264 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
265 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
266 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
267
268 // Only use the result of getSetCCResultType if it is legal,
269 // otherwise just use the promoted result type (NVT).
270 if (!TLI.isTypeLegal(SVT))
271 SVT = NVT;
272
273 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
274 SDValue Res = DAG.getAtomicCmpSwap(
275 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
276 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
277 N->getMemOperand());
278 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
279 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
280 return Res.getValue(1);
281 }
282
283 // Op2 is used for the comparison and thus must be extended according to the
284 // target's atomic operations. Op3 is merely stored and so can be left alone.
285 SDValue Op2 = N->getOperand(2);
286 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
287 switch (TLI.getExtendForAtomicCmpSwapArg()) {
288 case ISD::SIGN_EXTEND:
289 Op2 = SExtPromotedInteger(Op2);
290 break;
291 case ISD::ZERO_EXTEND:
292 Op2 = ZExtPromotedInteger(Op2);
293 break;
294 case ISD::ANY_EXTEND:
295 Op2 = GetPromotedInteger(Op2);
296 break;
297 default:
298 llvm_unreachable("Invalid atomic op extension");
299 }
300
301 SDVTList VTs =
302 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
303 SDValue Res = DAG.getAtomicCmpSwap(
304 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
305 N->getBasePtr(), Op2, Op3, N->getMemOperand());
306 // Update the use to N with the newly created Res.
307 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
308 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
309 return Res;
310 }
311
PromoteIntRes_PTRTOINT(SDNode * N)312 SDValue DAGTypeLegalizer::PromoteIntRes_PTRTOINT(SDNode *N) {
313 SDValue InOp = N->getOperand(0);
314 EVT OutVT = N->getValueType(0);
315 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
316 SDLoc dl(N);
317 return DAG.getNode(ISD::PTRTOINT, dl, NOutVT, InOp);
318 }
319
PromoteIntRes_BITCAST(SDNode * N)320 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
321 SDValue InOp = N->getOperand(0);
322 EVT InVT = InOp.getValueType();
323 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
324 EVT OutVT = N->getValueType(0);
325 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
326 SDLoc dl(N);
327
328 switch (getTypeAction(InVT)) {
329 case TargetLowering::TypeLegal:
330 break;
331 case TargetLowering::TypePromoteInteger:
332 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
333 // The input promotes to the same size. Convert the promoted value.
334 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
335 break;
336 case TargetLowering::TypeSoftenFloat:
337 // Promote the integer operand by hand.
338 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
339 case TargetLowering::TypeSoftPromoteHalf:
340 // Promote the integer operand by hand.
341 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
342 case TargetLowering::TypePromoteFloat: {
343 // Convert the promoted float by hand.
344 if (!NOutVT.isVector())
345 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
346 break;
347 }
348 case TargetLowering::TypeExpandInteger:
349 case TargetLowering::TypeExpandFloat:
350 break;
351 case TargetLowering::TypeScalarizeVector:
352 // Convert the element to an integer and promote it by hand.
353 if (!NOutVT.isVector())
354 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
355 BitConvertToInteger(GetScalarizedVector(InOp)));
356 break;
357 case TargetLowering::TypeScalarizeScalableVector:
358 report_fatal_error("Scalarization of scalable vectors is not supported.");
359 case TargetLowering::TypeSplitVector: {
360 if (!NOutVT.isVector()) {
361 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
362 // pieces of the input into integers and reassemble in the final type.
363 SDValue Lo, Hi;
364 GetSplitVector(N->getOperand(0), Lo, Hi);
365 Lo = BitConvertToInteger(Lo);
366 Hi = BitConvertToInteger(Hi);
367
368 if (DAG.getDataLayout().isBigEndian())
369 std::swap(Lo, Hi);
370
371 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
372 EVT::getIntegerVT(*DAG.getContext(),
373 NOutVT.getSizeInBits()),
374 JoinIntegers(Lo, Hi));
375 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
376 }
377 break;
378 }
379 case TargetLowering::TypeWidenVector:
380 // The input is widened to the same size. Convert to the widened value.
381 // Make sure that the outgoing value is not a vector, because this would
382 // make us bitcast between two vectors which are legalized in different ways.
383 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
384 SDValue Res =
385 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
386
387 // For big endian targets we need to shift the casted value or the
388 // interesting bits will end up at the wrong place.
389 if (DAG.getDataLayout().isBigEndian()) {
390 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
391 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout());
392 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
393 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
394 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
395 }
396 return Res;
397 }
398 // If the output type is also a vector and widening it to the same size
399 // as the widened input type would be a legal type, we can widen the bitcast
400 // and handle the promotion after.
401 if (NOutVT.isVector()) {
402 unsigned WidenInSize = NInVT.getSizeInBits();
403 unsigned OutSize = OutVT.getSizeInBits();
404 if (WidenInSize % OutSize == 0) {
405 unsigned Scale = WidenInSize / OutSize;
406 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
407 OutVT.getVectorElementType(),
408 OutVT.getVectorNumElements() * Scale);
409 if (isTypeLegal(WideOutVT)) {
410 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
411 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
412 DAG.getVectorIdxConstant(0, dl));
413 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
414 }
415 }
416 }
417 }
418
419 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
420 CreateStackStoreLoad(InOp, OutVT));
421 }
422
423 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount
424 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
getShiftAmountTyForConstant(EVT VT,const TargetLowering & TLI,SelectionDAG & DAG)425 static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI,
426 SelectionDAG &DAG) {
427 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
428 // If any possible shift value won't fit in the prefered type, just use
429 // something safe. It will be legalized when the shift is expanded.
430 if (!ShiftVT.isVector() &&
431 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits()))
432 ShiftVT = MVT::i32;
433 return ShiftVT;
434 }
435
PromoteIntRes_FREEZE(SDNode * N)436 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
437 SDValue V = GetPromotedInteger(N->getOperand(0));
438 return DAG.getNode(ISD::FREEZE, SDLoc(N),
439 V.getValueType(), V);
440 }
441
PromoteIntRes_BSWAP(SDNode * N)442 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
443 SDValue Op = GetPromotedInteger(N->getOperand(0));
444 EVT OVT = N->getValueType(0);
445 EVT NVT = Op.getValueType();
446 SDLoc dl(N);
447
448 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
449 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
450 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
451 DAG.getConstant(DiffBits, dl, ShiftVT));
452 }
453
PromoteIntRes_BITREVERSE(SDNode * N)454 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
455 SDValue Op = GetPromotedInteger(N->getOperand(0));
456 EVT OVT = N->getValueType(0);
457 EVT NVT = Op.getValueType();
458 SDLoc dl(N);
459
460 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
461 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
462 return DAG.getNode(ISD::SRL, dl, NVT,
463 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
464 DAG.getConstant(DiffBits, dl, ShiftVT));
465 }
466
PromoteIntRes_BUILD_PAIR(SDNode * N)467 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
468 // The pair element type may be legal, or may not promote to the same type as
469 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
470 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
471 TLI.getTypeToTransformTo(*DAG.getContext(),
472 N->getValueType(0)), JoinIntegers(N->getOperand(0),
473 N->getOperand(1)));
474 }
475
PromoteIntRes_Constant(SDNode * N)476 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
477 EVT VT = N->getValueType(0);
478 // FIXME there is no actual debug info here
479 SDLoc dl(N);
480 // Zero extend things like i1, sign extend everything else. It shouldn't
481 // matter in theory which one we pick, but this tends to give better code?
482 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
483 SDValue Result = DAG.getNode(Opc, dl,
484 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
485 SDValue(N, 0));
486 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
487 return Result;
488 }
489
PromoteIntRes_CTLZ(SDNode * N)490 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
491 // Zero extend to the promoted type and do the count there.
492 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
493 SDLoc dl(N);
494 EVT OVT = N->getValueType(0);
495 EVT NVT = Op.getValueType();
496 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
497 // Subtract off the extra leading bits in the bigger type.
498 return DAG.getNode(
499 ISD::SUB, dl, NVT, Op,
500 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
501 NVT));
502 }
503
PromoteIntRes_CTPOP(SDNode * N)504 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
505 // Zero extend to the promoted type and do the count there.
506 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
507 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
508 }
509
PromoteIntRes_CTTZ(SDNode * N)510 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
511 SDValue Op = GetPromotedInteger(N->getOperand(0));
512 EVT OVT = N->getValueType(0);
513 EVT NVT = Op.getValueType();
514 SDLoc dl(N);
515 if (N->getOpcode() == ISD::CTTZ) {
516 // The count is the same in the promoted type except if the original
517 // value was zero. This can be handled by setting the bit just off
518 // the top of the original type.
519 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
520 OVT.getScalarSizeInBits());
521 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
522 }
523 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
524 }
525
PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode * N)526 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
527 SDLoc dl(N);
528 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
529
530 SDValue Op0 = N->getOperand(0);
531 SDValue Op1 = N->getOperand(1);
532
533 // If the input also needs to be promoted, do that first so we can get a
534 // get a good idea for the output type.
535 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
536 == TargetLowering::TypePromoteInteger) {
537 SDValue In = GetPromotedInteger(Op0);
538
539 // If the new type is larger than NVT, use it. We probably won't need to
540 // promote it again.
541 EVT SVT = In.getValueType().getScalarType();
542 if (SVT.bitsGE(NVT)) {
543 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
544 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
545 }
546 }
547
548 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
549 }
550
PromoteIntRes_FP_TO_XINT(SDNode * N)551 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
552 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
553 unsigned NewOpc = N->getOpcode();
554 SDLoc dl(N);
555
556 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
557 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
558 // and SINT conversions are Custom, there is no way to tell which is
559 // preferable. We choose SINT because that's the right thing on PPC.)
560 if (N->getOpcode() == ISD::FP_TO_UINT &&
561 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
562 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
563 NewOpc = ISD::FP_TO_SINT;
564
565 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
566 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
567 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
568 NewOpc = ISD::STRICT_FP_TO_SINT;
569
570 SDValue Res;
571 if (N->isStrictFPOpcode()) {
572 Res = DAG.getNode(NewOpc, dl, { NVT, MVT::Other },
573 { N->getOperand(0), N->getOperand(1) });
574 // Legalize the chain result - switch anything that used the old chain to
575 // use the new one.
576 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
577 } else
578 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
579
580 // Assert that the converted value fits in the original type. If it doesn't
581 // (eg: because the value being converted is too big), then the result of the
582 // original operation was undefined anyway, so the assert is still correct.
583 //
584 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
585 // before legalization: fp-to-uint16, 65534. -> 0xfffe
586 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
587 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
588 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ?
589 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
590 DAG.getValueType(N->getValueType(0).getScalarType()));
591 }
592
PromoteIntRes_FP_TO_FP16(SDNode * N)593 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
594 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
595 SDLoc dl(N);
596
597 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
598 }
599
PromoteIntRes_FLT_ROUNDS(SDNode * N)600 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
601 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
602 SDLoc dl(N);
603
604 SDValue Res =
605 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
606
607 // Legalize the chain result - switch anything that used the old chain to
608 // use the new one.
609 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
610 return Res;
611 }
612
PromoteIntRes_INT_EXTEND(SDNode * N)613 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
614 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
615 SDLoc dl(N);
616
617 if (getTypeAction(N->getOperand(0).getValueType())
618 == TargetLowering::TypePromoteInteger) {
619 SDValue Res = GetPromotedInteger(N->getOperand(0));
620 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
621
622 // If the result and operand types are the same after promotion, simplify
623 // to an in-register extension.
624 if (NVT == Res.getValueType()) {
625 // The high bits are not guaranteed to be anything. Insert an extend.
626 if (N->getOpcode() == ISD::SIGN_EXTEND)
627 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
628 DAG.getValueType(N->getOperand(0).getValueType()));
629 if (N->getOpcode() == ISD::ZERO_EXTEND)
630 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
631 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
632 return Res;
633 }
634 }
635
636 // Otherwise, just extend the original operand all the way to the larger type.
637 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
638 }
639
PromoteIntRes_LOAD(LoadSDNode * N)640 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
641 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
642 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
643 ISD::LoadExtType ExtType =
644 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
645 SDLoc dl(N);
646 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
647 N->getMemoryVT(), N->getMemOperand());
648
649 // Legalize the chain result - switch anything that used the old chain to
650 // use the new one.
651 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
652 return Res;
653 }
654
PromoteIntRes_MLOAD(MaskedLoadSDNode * N)655 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
656 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
657 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
658
659 SDLoc dl(N);
660 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
661 N->getOffset(), N->getMask(), ExtPassThru,
662 N->getMemoryVT(), N->getMemOperand(),
663 N->getAddressingMode(), ISD::EXTLOAD);
664 // Legalize the chain result - switch anything that used the old chain to
665 // use the new one.
666 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
667 return Res;
668 }
669
PromoteIntRes_MGATHER(MaskedGatherSDNode * N)670 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
671 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
672 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
673 assert(NVT == ExtPassThru.getValueType() &&
674 "Gather result type and the passThru argument type should be the same");
675
676 SDLoc dl(N);
677 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
678 N->getIndex(), N->getScale() };
679 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
680 N->getMemoryVT(), dl, Ops,
681 N->getMemOperand(), N->getIndexType());
682 // Legalize the chain result - switch anything that used the old chain to
683 // use the new one.
684 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
685 return Res;
686 }
687
688 /// Promote the overflow flag of an overflowing arithmetic node.
PromoteIntRes_Overflow(SDNode * N)689 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
690 // Change the return type of the boolean result while obeying
691 // getSetCCResultType.
692 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
693 EVT VT = N->getValueType(0);
694 EVT SVT = getSetCCResultType(VT);
695 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
696 unsigned NumOps = N->getNumOperands();
697 assert(NumOps <= 3 && "Too many operands");
698 if (NumOps == 3)
699 Ops[2] = N->getOperand(2);
700
701 SDLoc dl(N);
702 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
703 makeArrayRef(Ops, NumOps));
704
705 // Modified the sum result - switch anything that used the old sum to use
706 // the new one.
707 ReplaceValueWith(SDValue(N, 0), Res);
708
709 // Convert to the expected type.
710 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
711 }
712
PromoteIntRes_ADDSUBSAT(SDNode * N)713 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
714 // If the promoted type is legal, we can convert this to:
715 // 1. ANY_EXTEND iN to iM
716 // 2. SHL by M-N
717 // 3. [US][ADD|SUB]SAT
718 // 4. L/ASHR by M-N
719 // Else it is more efficient to convert this to a min and a max
720 // operation in the higher precision arithmetic.
721 SDLoc dl(N);
722 SDValue Op1 = N->getOperand(0);
723 SDValue Op2 = N->getOperand(1);
724 unsigned OldBits = Op1.getScalarValueSizeInBits();
725
726 unsigned Opcode = N->getOpcode();
727
728 SDValue Op1Promoted, Op2Promoted;
729 if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
730 Op1Promoted = ZExtPromotedInteger(Op1);
731 Op2Promoted = ZExtPromotedInteger(Op2);
732 } else {
733 Op1Promoted = SExtPromotedInteger(Op1);
734 Op2Promoted = SExtPromotedInteger(Op2);
735 }
736 EVT PromotedType = Op1Promoted.getValueType();
737 unsigned NewBits = PromotedType.getScalarSizeInBits();
738
739 if (TLI.isOperationLegalOrCustom(Opcode, PromotedType)) {
740 unsigned ShiftOp;
741 switch (Opcode) {
742 case ISD::SADDSAT:
743 case ISD::SSUBSAT:
744 ShiftOp = ISD::SRA;
745 break;
746 case ISD::UADDSAT:
747 case ISD::USUBSAT:
748 ShiftOp = ISD::SRL;
749 break;
750 default:
751 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
752 "addition or subtraction");
753 }
754
755 unsigned SHLAmount = NewBits - OldBits;
756 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
757 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
758 Op1Promoted =
759 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
760 Op2Promoted =
761 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
762
763 SDValue Result =
764 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
765 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
766 } else {
767 if (Opcode == ISD::USUBSAT) {
768 SDValue Max =
769 DAG.getNode(ISD::UMAX, dl, PromotedType, Op1Promoted, Op2Promoted);
770 return DAG.getNode(ISD::SUB, dl, PromotedType, Max, Op2Promoted);
771 }
772
773 if (Opcode == ISD::UADDSAT) {
774 APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits);
775 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
776 SDValue Add =
777 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
778 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
779 }
780
781 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
782 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
783 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
784 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
785 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
786 SDValue Result =
787 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
788 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
789 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
790 return Result;
791 }
792 }
793
PromoteIntRes_MULFIX(SDNode * N)794 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
795 // Can just promote the operands then continue with operation.
796 SDLoc dl(N);
797 SDValue Op1Promoted, Op2Promoted;
798 bool Signed =
799 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
800 bool Saturating =
801 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
802 if (Signed) {
803 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
804 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
805 } else {
806 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
807 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
808 }
809 EVT OldType = N->getOperand(0).getValueType();
810 EVT PromotedType = Op1Promoted.getValueType();
811 unsigned DiffSize =
812 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
813
814 if (Saturating) {
815 // Promoting the operand and result values changes the saturation width,
816 // which is extends the values that we clamp to on saturation. This could be
817 // resolved by shifting one of the operands the same amount, which would
818 // also shift the result we compare against, then shifting back.
819 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
820 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
821 DAG.getConstant(DiffSize, dl, ShiftTy));
822 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
823 Op2Promoted, N->getOperand(2));
824 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
825 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
826 DAG.getConstant(DiffSize, dl, ShiftTy));
827 }
828 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
829 N->getOperand(2));
830 }
831
SaturateWidenedDIVFIX(SDValue V,SDLoc & dl,unsigned SatW,bool Signed,const TargetLowering & TLI,SelectionDAG & DAG)832 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
833 unsigned SatW, bool Signed,
834 const TargetLowering &TLI,
835 SelectionDAG &DAG) {
836 EVT VT = V.getValueType();
837 unsigned VTW = VT.getScalarSizeInBits();
838
839 if (!Signed) {
840 // Saturate to the unsigned maximum by getting the minimum of V and the
841 // maximum.
842 return DAG.getNode(ISD::UMIN, dl, VT, V,
843 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
844 dl, VT));
845 }
846
847 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
848 // signed minimum of it and V.
849 V = DAG.getNode(ISD::SMIN, dl, VT, V,
850 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
851 dl, VT));
852 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
853 // signed maximum of it and V.
854 V = DAG.getNode(ISD::SMAX, dl, VT, V,
855 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
856 dl, VT));
857 return V;
858 }
859
earlyExpandDIVFIX(SDNode * N,SDValue LHS,SDValue RHS,unsigned Scale,const TargetLowering & TLI,SelectionDAG & DAG,unsigned SatW=0)860 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
861 unsigned Scale, const TargetLowering &TLI,
862 SelectionDAG &DAG, unsigned SatW = 0) {
863 EVT VT = LHS.getValueType();
864 unsigned VTSize = VT.getScalarSizeInBits();
865 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
866 N->getOpcode() == ISD::SDIVFIXSAT;
867 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
868 N->getOpcode() == ISD::UDIVFIXSAT;
869
870 SDLoc dl(N);
871 // Widen the types by a factor of two. This is guaranteed to expand, since it
872 // will always have enough high bits in the LHS to shift into.
873 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
874 if (VT.isVector())
875 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
876 VT.getVectorElementCount());
877 if (Signed) {
878 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
879 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
880 } else {
881 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
882 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
883 }
884
885 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
886 DAG);
887 assert(Res && "Expanding DIVFIX with wide type failed?");
888 if (Saturating) {
889 // If the caller has told us to saturate at something less, use that width
890 // instead of the type before doubling. However, it cannot be more than
891 // what we just widened!
892 assert(SatW <= VTSize &&
893 "Tried to saturate to more than the original type?");
894 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
895 TLI, DAG);
896 }
897 return DAG.getZExtOrTrunc(Res, dl, VT);
898 }
899
PromoteIntRes_DIVFIX(SDNode * N)900 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
901 SDLoc dl(N);
902 SDValue Op1Promoted, Op2Promoted;
903 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
904 N->getOpcode() == ISD::SDIVFIXSAT;
905 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
906 N->getOpcode() == ISD::UDIVFIXSAT;
907 if (Signed) {
908 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
909 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
910 } else {
911 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
912 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
913 }
914 EVT PromotedType = Op1Promoted.getValueType();
915 unsigned Scale = N->getConstantOperandVal(2);
916
917 // If the type is already legal and the operation is legal in that type, we
918 // should not early expand.
919 if (TLI.isTypeLegal(PromotedType)) {
920 TargetLowering::LegalizeAction Action =
921 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
922 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
923 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
924 unsigned Diff = PromotedType.getScalarSizeInBits() -
925 N->getValueType(0).getScalarSizeInBits();
926 if (Saturating)
927 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
928 DAG.getConstant(Diff, dl, ShiftTy));
929 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
930 Op2Promoted, N->getOperand(2));
931 if (Saturating)
932 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
933 DAG.getConstant(Diff, dl, ShiftTy));
934 return Res;
935 }
936 }
937
938 // See if we can perform the division in this type without expanding.
939 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
940 Op2Promoted, Scale, DAG)) {
941 if (Saturating)
942 Res = SaturateWidenedDIVFIX(Res, dl,
943 N->getValueType(0).getScalarSizeInBits(),
944 Signed, TLI, DAG);
945 return Res;
946 }
947 // If we cannot, expand it to twice the type width. If we are saturating, give
948 // it the original width as a saturating width so we don't need to emit
949 // two saturations.
950 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
951 N->getValueType(0).getScalarSizeInBits());
952 }
953
PromoteIntRes_SADDSUBO(SDNode * N,unsigned ResNo)954 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
955 if (ResNo == 1)
956 return PromoteIntRes_Overflow(N);
957
958 // The operation overflowed iff the result in the larger type is not the
959 // sign extension of its truncation to the original type.
960 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
961 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
962 EVT OVT = N->getOperand(0).getValueType();
963 EVT NVT = LHS.getValueType();
964 SDLoc dl(N);
965
966 // Do the arithmetic in the larger type.
967 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
968 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
969
970 // Calculate the overflow flag: sign extend the arithmetic result from
971 // the original type.
972 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
973 DAG.getValueType(OVT));
974 // Overflowed if and only if this is not equal to Res.
975 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
976
977 // Use the calculated overflow everywhere.
978 ReplaceValueWith(SDValue(N, 1), Ofl);
979
980 return Res;
981 }
982
PromoteIntRes_SELECT(SDNode * N)983 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
984 SDValue LHS = GetPromotedInteger(N->getOperand(1));
985 SDValue RHS = GetPromotedInteger(N->getOperand(2));
986 return DAG.getSelect(SDLoc(N),
987 LHS.getValueType(), N->getOperand(0), LHS, RHS);
988 }
989
PromoteIntRes_VSELECT(SDNode * N)990 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
991 SDValue Mask = N->getOperand(0);
992
993 SDValue LHS = GetPromotedInteger(N->getOperand(1));
994 SDValue RHS = GetPromotedInteger(N->getOperand(2));
995 return DAG.getNode(ISD::VSELECT, SDLoc(N),
996 LHS.getValueType(), Mask, LHS, RHS);
997 }
998
PromoteIntRes_SELECT_CC(SDNode * N)999 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1000 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1001 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1002 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1003 LHS.getValueType(), N->getOperand(0),
1004 N->getOperand(1), LHS, RHS, N->getOperand(4));
1005 }
1006
PromoteIntRes_SETCC(SDNode * N)1007 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1008 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1009 EVT InVT = N->getOperand(OpNo).getValueType();
1010 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1011
1012 EVT SVT = getSetCCResultType(InVT);
1013
1014 // If we got back a type that needs to be promoted, this likely means the
1015 // the input type also needs to be promoted. So get the promoted type for
1016 // the input and try the query again.
1017 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1018 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1019 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1020 SVT = getSetCCResultType(InVT);
1021 } else {
1022 // Input type isn't promoted, just use the default promoted type.
1023 SVT = NVT;
1024 }
1025 }
1026
1027 SDLoc dl(N);
1028 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1029 "Vector compare must return a vector result!");
1030
1031 // Get the SETCC result using the canonical SETCC type.
1032 SDValue SetCC;
1033 if (N->isStrictFPOpcode()) {
1034 EVT VTs[] = {SVT, MVT::Other};
1035 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1036 N->getOperand(2), N->getOperand(3)};
1037 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
1038 // Legalize the chain result - switch anything that used the old chain to
1039 // use the new one.
1040 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1041 } else
1042 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1043 N->getOperand(1), N->getOperand(2));
1044
1045 // Convert to the expected type.
1046 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1047 }
1048
PromoteIntRes_SHL(SDNode * N)1049 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1050 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1051 SDValue RHS = N->getOperand(1);
1052 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1053 RHS = ZExtPromotedInteger(RHS);
1054 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
1055 }
1056
PromoteIntRes_SIGN_EXTEND_INREG(SDNode * N)1057 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1058 SDValue Op = GetPromotedInteger(N->getOperand(0));
1059 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1060 Op.getValueType(), Op, N->getOperand(1));
1061 }
1062
PromoteIntRes_SimpleIntBinOp(SDNode * N)1063 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1064 // The input may have strange things in the top bits of the registers, but
1065 // these operations don't care. They may have weird bits going out, but
1066 // that too is okay if they are integer operations.
1067 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1068 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1069 return DAG.getNode(N->getOpcode(), SDLoc(N),
1070 LHS.getValueType(), LHS, RHS);
1071 }
1072
PromoteIntRes_SExtIntBinOp(SDNode * N)1073 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1074 // Sign extend the input.
1075 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1076 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1077 return DAG.getNode(N->getOpcode(), SDLoc(N),
1078 LHS.getValueType(), LHS, RHS);
1079 }
1080
PromoteIntRes_ZExtIntBinOp(SDNode * N)1081 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1082 // Zero extend the input.
1083 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1084 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1085 return DAG.getNode(N->getOpcode(), SDLoc(N),
1086 LHS.getValueType(), LHS, RHS);
1087 }
1088
PromoteIntRes_SRA(SDNode * N)1089 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1090 // The input value must be properly sign extended.
1091 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1092 SDValue RHS = N->getOperand(1);
1093 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1094 RHS = ZExtPromotedInteger(RHS);
1095 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
1096 }
1097
PromoteIntRes_SRL(SDNode * N)1098 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1099 // The input value must be properly zero extended.
1100 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1101 SDValue RHS = N->getOperand(1);
1102 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1103 RHS = ZExtPromotedInteger(RHS);
1104 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
1105 }
1106
PromoteIntRes_TRUNCATE(SDNode * N)1107 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1108 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1109 SDValue Res;
1110 SDValue InOp = N->getOperand(0);
1111 SDLoc dl(N);
1112
1113 switch (getTypeAction(InOp.getValueType())) {
1114 default: llvm_unreachable("Unknown type action!");
1115 case TargetLowering::TypeLegal:
1116 case TargetLowering::TypeExpandInteger:
1117 Res = InOp;
1118 break;
1119 case TargetLowering::TypePromoteInteger:
1120 Res = GetPromotedInteger(InOp);
1121 break;
1122 case TargetLowering::TypeSplitVector: {
1123 EVT InVT = InOp.getValueType();
1124 assert(InVT.isVector() && "Cannot split scalar types");
1125 unsigned NumElts = InVT.getVectorNumElements();
1126 assert(NumElts == NVT.getVectorNumElements() &&
1127 "Dst and Src must have the same number of elements");
1128 assert(isPowerOf2_32(NumElts) &&
1129 "Promoted vector type must be a power of two");
1130
1131 SDValue EOp1, EOp2;
1132 GetSplitVector(InOp, EOp1, EOp2);
1133
1134 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1135 NumElts/2);
1136 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1137 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1138
1139 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1140 }
1141 case TargetLowering::TypeWidenVector: {
1142 SDValue WideInOp = GetWidenedVector(InOp);
1143
1144 // Truncate widened InOp.
1145 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1146 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1147 N->getValueType(0).getScalarType(), NumElem);
1148 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1149
1150 // Zero extend so that the elements are of same type as those of NVT
1151 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1152 NumElem);
1153 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1154
1155 // Extract the low NVT subvector.
1156 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1157 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1158 }
1159 }
1160
1161 // Truncate to NVT instead of VT
1162 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1163 }
1164
PromoteIntRes_UADDSUBO(SDNode * N,unsigned ResNo)1165 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1166 if (ResNo == 1)
1167 return PromoteIntRes_Overflow(N);
1168
1169 // The operation overflowed iff the result in the larger type is not the
1170 // zero extension of its truncation to the original type.
1171 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1172 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1173 EVT OVT = N->getOperand(0).getValueType();
1174 EVT NVT = LHS.getValueType();
1175 SDLoc dl(N);
1176
1177 // Do the arithmetic in the larger type.
1178 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1179 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1180
1181 // Calculate the overflow flag: zero extend the arithmetic result from
1182 // the original type.
1183 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1184 // Overflowed if and only if this is not equal to Res.
1185 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1186
1187 // Use the calculated overflow everywhere.
1188 ReplaceValueWith(SDValue(N, 1), Ofl);
1189
1190 return Res;
1191 }
1192
1193 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1194 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
1195 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
PromoteIntRes_ADDSUBCARRY(SDNode * N,unsigned ResNo)1196 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1197 if (ResNo == 1)
1198 return PromoteIntRes_Overflow(N);
1199
1200 // We need to sign-extend the operands so the carry value computed by the
1201 // wide operation will be equivalent to the carry value computed by the
1202 // narrow operation.
1203 // An ADDCARRY can generate carry only if any of the operands has its
1204 // most significant bit set. Sign extension propagates the most significant
1205 // bit into the higher bits which means the extra bit that the narrow
1206 // addition would need (i.e. the carry) will be propagated through the higher
1207 // bits of the wide addition.
1208 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1209 // preserved by sign extension.
1210 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1211 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1212
1213 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1214
1215 // Do the arithmetic in the wide type.
1216 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1217 LHS, RHS, N->getOperand(2));
1218
1219 // Update the users of the original carry/borrow value.
1220 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1221
1222 return SDValue(Res.getNode(), 0);
1223 }
1224
PromoteIntRes_ABS(SDNode * N)1225 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1226 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1227 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1228 }
1229
PromoteIntRes_XMULO(SDNode * N,unsigned ResNo)1230 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1231 // Promote the overflow bit trivially.
1232 if (ResNo == 1)
1233 return PromoteIntRes_Overflow(N);
1234
1235 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1236 SDLoc DL(N);
1237 EVT SmallVT = LHS.getValueType();
1238
1239 // To determine if the result overflowed in a larger type, we extend the
1240 // input to the larger type, do the multiply (checking if it overflows),
1241 // then also check the high bits of the result to see if overflow happened
1242 // there.
1243 if (N->getOpcode() == ISD::SMULO) {
1244 LHS = SExtPromotedInteger(LHS);
1245 RHS = SExtPromotedInteger(RHS);
1246 } else {
1247 LHS = ZExtPromotedInteger(LHS);
1248 RHS = ZExtPromotedInteger(RHS);
1249 }
1250 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1251 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1252
1253 // Overflow occurred if it occurred in the larger type, or if the high part
1254 // of the result does not zero/sign-extend the low part. Check this second
1255 // possibility first.
1256 SDValue Overflow;
1257 if (N->getOpcode() == ISD::UMULO) {
1258 // Unsigned overflow occurred if the high part is non-zero.
1259 unsigned Shift = SmallVT.getScalarSizeInBits();
1260 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG);
1261 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1262 DAG.getConstant(Shift, DL, ShiftTy));
1263 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1264 DAG.getConstant(0, DL, Hi.getValueType()),
1265 ISD::SETNE);
1266 } else {
1267 // Signed overflow occurred if the high part does not sign extend the low.
1268 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1269 Mul, DAG.getValueType(SmallVT));
1270 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1271 }
1272
1273 // The only other way for overflow to occur is if the multiplication in the
1274 // larger type itself overflowed.
1275 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1276 SDValue(Mul.getNode(), 1));
1277
1278 // Use the calculated overflow everywhere.
1279 ReplaceValueWith(SDValue(N, 1), Overflow);
1280 return Mul;
1281 }
1282
PromoteIntRes_UNDEF(SDNode * N)1283 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1284 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1285 N->getValueType(0)));
1286 }
1287
PromoteIntRes_VSCALE(SDNode * N)1288 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1289 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1290
1291 APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
1292 return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1293 }
1294
PromoteIntRes_VAARG(SDNode * N)1295 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1296 SDValue Chain = N->getOperand(0); // Get the chain.
1297 SDValue Ptr = N->getOperand(1); // Get the pointer.
1298 EVT VT = N->getValueType(0);
1299 SDLoc dl(N);
1300
1301 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1302 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1303 // The argument is passed as NumRegs registers of type RegVT.
1304
1305 SmallVector<SDValue, 8> Parts(NumRegs);
1306 for (unsigned i = 0; i < NumRegs; ++i) {
1307 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1308 N->getConstantOperandVal(3));
1309 Chain = Parts[i].getValue(1);
1310 }
1311
1312 // Handle endianness of the load.
1313 if (DAG.getDataLayout().isBigEndian())
1314 std::reverse(Parts.begin(), Parts.end());
1315
1316 // Assemble the parts in the promoted type.
1317 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1318 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1319 for (unsigned i = 1; i < NumRegs; ++i) {
1320 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1321 // Shift it to the right position and "or" it in.
1322 Part = DAG.getNode(
1323 ISD::SHL, dl, NVT, Part,
1324 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1325 TLI.getPointerRangeTy(DAG.getDataLayout())));
1326 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1327 }
1328
1329 // Modified the chain result - switch anything that used the old chain to
1330 // use the new one.
1331 ReplaceValueWith(SDValue(N, 1), Chain);
1332
1333 return Res;
1334 }
1335
1336 //===----------------------------------------------------------------------===//
1337 // Integer Operand Promotion
1338 //===----------------------------------------------------------------------===//
1339
1340 /// PromoteIntegerOperand - This method is called when the specified operand of
1341 /// the specified node is found to need promotion. At this point, all of the
1342 /// result types of the node are known to be legal, but other operands of the
1343 /// node may need promotion or expansion as well as the specified one.
PromoteIntegerOperand(SDNode * N,unsigned OpNo)1344 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1345 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1346 dbgs() << "\n");
1347 SDValue Res = SDValue();
1348 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1349 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1350 return false;
1351 }
1352
1353 switch (N->getOpcode()) {
1354 default:
1355 #ifndef NDEBUG
1356 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1357 N->dump(&DAG); dbgs() << "\n";
1358 #endif
1359 llvm_unreachable("Do not know how to promote this operator's operand!");
1360
1361 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1362 case ISD::ATOMIC_STORE:
1363 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1364 break;
1365 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1366 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1367 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1368 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1369 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1370 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1371 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1372 case ISD::INTTOPTR:
1373 Res = PromoteIntOp_INTTOPTR(N);break;
1374 case ISD::INSERT_VECTOR_ELT:
1375 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1376 case ISD::SCALAR_TO_VECTOR:
1377 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1378 case ISD::SPLAT_VECTOR:
1379 Res = PromoteIntOp_SPLAT_VECTOR(N); break;
1380 case ISD::VSELECT:
1381 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1382 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1383 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1384 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1385 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1386 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1387 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1388 OpNo); break;
1389 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1390 OpNo); break;
1391 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1392 OpNo); break;
1393 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1394 OpNo); break;
1395 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1396 OpNo); break;
1397 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1398 case ISD::FP16_TO_FP:
1399 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1400 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1401 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1402 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1403
1404 case ISD::SHL:
1405 case ISD::SRA:
1406 case ISD::SRL:
1407 case ISD::ROTL:
1408 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1409
1410 case ISD::ADDCARRY:
1411 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1412
1413 case ISD::FRAMEADDR:
1414 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1415
1416 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1417
1418 case ISD::SMULFIX:
1419 case ISD::SMULFIXSAT:
1420 case ISD::UMULFIX:
1421 case ISD::UMULFIXSAT:
1422 case ISD::SDIVFIX:
1423 case ISD::SDIVFIXSAT:
1424 case ISD::UDIVFIX:
1425 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1426
1427 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1428
1429 case ISD::VECREDUCE_ADD:
1430 case ISD::VECREDUCE_MUL:
1431 case ISD::VECREDUCE_AND:
1432 case ISD::VECREDUCE_OR:
1433 case ISD::VECREDUCE_XOR:
1434 case ISD::VECREDUCE_SMAX:
1435 case ISD::VECREDUCE_SMIN:
1436 case ISD::VECREDUCE_UMAX:
1437 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1438 }
1439
1440 // If the result is null, the sub-method took care of registering results etc.
1441 if (!Res.getNode()) return false;
1442
1443 // If the result is N, the sub-method updated N in place. Tell the legalizer
1444 // core about this.
1445 if (Res.getNode() == N)
1446 return true;
1447
1448 const bool IsStrictFp = N->isStrictFPOpcode();
1449 assert(Res.getValueType() == N->getValueType(0) &&
1450 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1451 "Invalid operand expansion");
1452 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1453 Res.dump());
1454
1455 ReplaceValueWith(SDValue(N, 0), Res);
1456 if (IsStrictFp)
1457 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1458
1459 return false;
1460 }
1461
1462 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1463 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
PromoteSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode CCCode)1464 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1465 ISD::CondCode CCCode) {
1466 // We have to insert explicit sign or zero extends. Note that we could
1467 // insert sign extends for ALL conditions. For those operations where either
1468 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1469 // which will choose the cheapest for the target.
1470 switch (CCCode) {
1471 default: llvm_unreachable("Unknown integer comparison!");
1472 case ISD::SETEQ:
1473 case ISD::SETNE: {
1474 SDValue OpL = GetPromotedInteger(NewLHS);
1475 SDValue OpR = GetPromotedInteger(NewRHS);
1476
1477 // We would prefer to promote the comparison operand with sign extension.
1478 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1479 // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1480 // instruction, which is redundant eventually.
1481 unsigned OpLEffectiveBits =
1482 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1483 unsigned OpREffectiveBits =
1484 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1485 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1486 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1487 NewLHS = OpL;
1488 NewRHS = OpR;
1489 } else {
1490 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1491 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1492 }
1493 break;
1494 }
1495 case ISD::SETUGE:
1496 case ISD::SETUGT:
1497 case ISD::SETULE:
1498 case ISD::SETULT:
1499 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1500 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1501 break;
1502 case ISD::SETGE:
1503 case ISD::SETGT:
1504 case ISD::SETLT:
1505 case ISD::SETLE:
1506 NewLHS = SExtPromotedInteger(NewLHS);
1507 NewRHS = SExtPromotedInteger(NewRHS);
1508 break;
1509 }
1510 }
1511
PromoteIntOp_ANY_EXTEND(SDNode * N)1512 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1513 SDValue Op = GetPromotedInteger(N->getOperand(0));
1514 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1515 }
1516
PromoteIntOp_ATOMIC_STORE(AtomicSDNode * N)1517 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1518 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1519 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1520 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1521 }
1522
PromoteIntOp_BITCAST(SDNode * N)1523 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1524 // This should only occur in unusual situations like bitcasting to an
1525 // x86_fp80, so just turn it into a store+load
1526 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1527 }
1528
PromoteIntOp_BR_CC(SDNode * N,unsigned OpNo)1529 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1530 assert(OpNo == 2 && "Don't know how to promote this operand!");
1531
1532 SDValue LHS = N->getOperand(2);
1533 SDValue RHS = N->getOperand(3);
1534 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1535
1536 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1537 // legal types.
1538 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1539 N->getOperand(1), LHS, RHS, N->getOperand(4)),
1540 0);
1541 }
1542
PromoteIntOp_BRCOND(SDNode * N,unsigned OpNo)1543 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1544 assert(OpNo == 1 && "only know how to promote condition");
1545
1546 // Promote all the way up to the canonical SetCC type.
1547 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1548
1549 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1550 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1551 N->getOperand(2)), 0);
1552 }
1553
PromoteIntOp_BUILD_PAIR(SDNode * N)1554 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1555 // Since the result type is legal, the operands must promote to it.
1556 EVT OVT = N->getOperand(0).getValueType();
1557 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1558 SDValue Hi = GetPromotedInteger(N->getOperand(1));
1559 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1560 SDLoc dl(N);
1561
1562 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1563 DAG.getConstant(OVT.getSizeInBits(), dl,
1564 TLI.getPointerRangeTy(DAG.getDataLayout())));
1565 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1566 }
1567
PromoteIntOp_BUILD_VECTOR(SDNode * N)1568 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1569 // The vector type is legal but the element type is not. This implies
1570 // that the vector is a power-of-two in length and that the element
1571 // type does not have a strange size (eg: it is not i1).
1572 EVT VecVT = N->getValueType(0);
1573 unsigned NumElts = VecVT.getVectorNumElements();
1574 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1575 "Legal vector of one illegal element?");
1576
1577 // Promote the inserted value. The type does not need to match the
1578 // vector element type. Check that any extra bits introduced will be
1579 // truncated away.
1580 assert(N->getOperand(0).getValueSizeInBits() >=
1581 N->getValueType(0).getScalarSizeInBits() &&
1582 "Type of inserted value narrower than vector element type!");
1583
1584 SmallVector<SDValue, 16> NewOps;
1585 for (unsigned i = 0; i < NumElts; ++i)
1586 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1587
1588 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1589 }
1590
PromoteIntOp_INTTOPTR(SDNode * N)1591 SDValue DAGTypeLegalizer::PromoteIntOp_INTTOPTR(SDNode *N) {
1592 SDValue InOp = GetPromotedInteger(N->getOperand(0));
1593 return DAG.getNode(ISD::INTTOPTR, SDLoc(N), N->getValueType(0), InOp);
1594 }
1595
PromoteIntOp_INSERT_VECTOR_ELT(SDNode * N,unsigned OpNo)1596 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1597 unsigned OpNo) {
1598 if (OpNo == 1) {
1599 // Promote the inserted value. This is valid because the type does not
1600 // have to match the vector element type.
1601
1602 // Check that any extra bits introduced will be truncated away.
1603 assert(N->getOperand(1).getValueSizeInBits() >=
1604 N->getValueType(0).getScalarSizeInBits() &&
1605 "Type of inserted value narrower than vector element type!");
1606 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1607 GetPromotedInteger(N->getOperand(1)),
1608 N->getOperand(2)),
1609 0);
1610 }
1611
1612 assert(OpNo == 2 && "Different operand and result vector types?");
1613
1614 // Promote the index.
1615 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1616 TLI.getVectorIdxTy(DAG.getDataLayout()));
1617 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1618 N->getOperand(1), Idx), 0);
1619 }
1620
PromoteIntOp_SCALAR_TO_VECTOR(SDNode * N)1621 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1622 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1623 // the operand in place.
1624 return SDValue(DAG.UpdateNodeOperands(N,
1625 GetPromotedInteger(N->getOperand(0))), 0);
1626 }
1627
PromoteIntOp_SPLAT_VECTOR(SDNode * N)1628 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1629 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
1630 // operand in place.
1631 return SDValue(
1632 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1633 }
1634
PromoteIntOp_SELECT(SDNode * N,unsigned OpNo)1635 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1636 assert(OpNo == 0 && "Only know how to promote the condition!");
1637 SDValue Cond = N->getOperand(0);
1638 EVT OpTy = N->getOperand(1).getValueType();
1639
1640 if (N->getOpcode() == ISD::VSELECT)
1641 if (SDValue Res = WidenVSELECTAndMask(N))
1642 return Res;
1643
1644 // Promote all the way up to the canonical SetCC type.
1645 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1646 Cond = PromoteTargetBoolean(Cond, OpVT);
1647
1648 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1649 N->getOperand(2)), 0);
1650 }
1651
PromoteIntOp_SELECT_CC(SDNode * N,unsigned OpNo)1652 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1653 assert(OpNo == 0 && "Don't know how to promote this operand!");
1654
1655 SDValue LHS = N->getOperand(0);
1656 SDValue RHS = N->getOperand(1);
1657 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1658
1659 // The CC (#4) and the possible return values (#2 and #3) have legal types.
1660 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1661 N->getOperand(3), N->getOperand(4)), 0);
1662 }
1663
PromoteIntOp_SETCC(SDNode * N,unsigned OpNo)1664 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1665 assert(OpNo == 0 && "Don't know how to promote this operand!");
1666
1667 SDValue LHS = N->getOperand(0);
1668 SDValue RHS = N->getOperand(1);
1669 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1670
1671 // The CC (#2) is always legal.
1672 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1673 }
1674
PromoteIntOp_Shift(SDNode * N)1675 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1676 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1677 ZExtPromotedInteger(N->getOperand(1))), 0);
1678 }
1679
PromoteIntOp_SIGN_EXTEND(SDNode * N)1680 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1681 SDValue Op = GetPromotedInteger(N->getOperand(0));
1682 SDLoc dl(N);
1683 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1684 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1685 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1686 }
1687
PromoteIntOp_SINT_TO_FP(SDNode * N)1688 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1689 return SDValue(DAG.UpdateNodeOperands(N,
1690 SExtPromotedInteger(N->getOperand(0))), 0);
1691 }
1692
PromoteIntOp_STRICT_SINT_TO_FP(SDNode * N)1693 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
1694 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1695 SExtPromotedInteger(N->getOperand(1))), 0);
1696 }
1697
PromoteIntOp_STORE(StoreSDNode * N,unsigned OpNo)1698 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1699 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1700 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1701 SDLoc dl(N);
1702
1703 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1704
1705 // Truncate the value and store the result.
1706 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1707 N->getMemoryVT(), N->getMemOperand());
1708 }
1709
PromoteIntOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)1710 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1711 unsigned OpNo) {
1712
1713 SDValue DataOp = N->getValue();
1714 EVT DataVT = DataOp.getValueType();
1715 SDValue Mask = N->getMask();
1716 SDLoc dl(N);
1717
1718 bool TruncateStore = false;
1719 if (OpNo == 4) {
1720 Mask = PromoteTargetBoolean(Mask, DataVT);
1721 // Update in place.
1722 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1723 NewOps[4] = Mask;
1724 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1725 } else { // Data operand
1726 assert(OpNo == 1 && "Unexpected operand for promotion");
1727 DataOp = GetPromotedInteger(DataOp);
1728 TruncateStore = true;
1729 }
1730
1731 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(),
1732 N->getOffset(), Mask, N->getMemoryVT(),
1733 N->getMemOperand(), N->getAddressingMode(),
1734 TruncateStore, N->isCompressingStore());
1735 }
1736
PromoteIntOp_MLOAD(MaskedLoadSDNode * N,unsigned OpNo)1737 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1738 unsigned OpNo) {
1739 assert(OpNo == 3 && "Only know how to promote the mask!");
1740 EVT DataVT = N->getValueType(0);
1741 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1742 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1743 NewOps[OpNo] = Mask;
1744 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1745 if (Res == N)
1746 return SDValue(Res, 0);
1747
1748 // Update triggered CSE, do our own replacement since caller can't.
1749 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1750 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1751 return SDValue();
1752 }
1753
PromoteIntOp_MGATHER(MaskedGatherSDNode * N,unsigned OpNo)1754 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1755 unsigned OpNo) {
1756
1757 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1758 if (OpNo == 2) {
1759 // The Mask
1760 EVT DataVT = N->getValueType(0);
1761 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1762 } else if (OpNo == 4) {
1763 // The Index
1764 if (N->isIndexSigned())
1765 // Need to sign extend the index since the bits will likely be used.
1766 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1767 else
1768 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1769 } else
1770 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1771
1772 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1773 if (Res == N)
1774 return SDValue(Res, 0);
1775
1776 // Update triggered CSE, do our own replacement since caller can't.
1777 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1778 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1779 return SDValue();
1780 }
1781
PromoteIntOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)1782 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1783 unsigned OpNo) {
1784 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1785 if (OpNo == 2) {
1786 // The Mask
1787 EVT DataVT = N->getValue().getValueType();
1788 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1789 } else if (OpNo == 4) {
1790 // The Index
1791 if (N->isIndexSigned())
1792 // Need to sign extend the index since the bits will likely be used.
1793 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1794 else
1795 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1796 } else
1797 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1798 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1799 }
1800
PromoteIntOp_TRUNCATE(SDNode * N)1801 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1802 SDValue Op = GetPromotedInteger(N->getOperand(0));
1803 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1804 }
1805
PromoteIntOp_UINT_TO_FP(SDNode * N)1806 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1807 return SDValue(DAG.UpdateNodeOperands(N,
1808 ZExtPromotedInteger(N->getOperand(0))), 0);
1809 }
1810
PromoteIntOp_STRICT_UINT_TO_FP(SDNode * N)1811 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
1812 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1813 ZExtPromotedInteger(N->getOperand(1))), 0);
1814 }
1815
PromoteIntOp_ZERO_EXTEND(SDNode * N)1816 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1817 SDLoc dl(N);
1818 SDValue Op = GetPromotedInteger(N->getOperand(0));
1819 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1820 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
1821 }
1822
PromoteIntOp_ADDSUBCARRY(SDNode * N,unsigned OpNo)1823 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1824 assert(OpNo == 2 && "Don't know how to promote this operand!");
1825
1826 SDValue LHS = N->getOperand(0);
1827 SDValue RHS = N->getOperand(1);
1828 SDValue Carry = N->getOperand(2);
1829 SDLoc DL(N);
1830
1831 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1832
1833 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1834 }
1835
PromoteIntOp_FIX(SDNode * N)1836 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
1837 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1838 return SDValue(
1839 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1840 }
1841
PromoteIntOp_FRAMERETURNADDR(SDNode * N)1842 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1843 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1844 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1845 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1846 }
1847
PromoteIntOp_PREFETCH(SDNode * N,unsigned OpNo)1848 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1849 assert(OpNo > 1 && "Don't know how to promote this operand!");
1850 // Promote the rw, locality, and cache type arguments to a supported integer
1851 // width.
1852 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1853 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1854 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1855 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1856 Op2, Op3, Op4),
1857 0);
1858 }
1859
PromoteIntOp_FPOWI(SDNode * N)1860 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1861 SDValue Op = SExtPromotedInteger(N->getOperand(1));
1862 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1863 }
1864
PromoteIntOp_VECREDUCE(SDNode * N)1865 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1866 SDLoc dl(N);
1867 SDValue Op;
1868 switch (N->getOpcode()) {
1869 default: llvm_unreachable("Expected integer vector reduction");
1870 case ISD::VECREDUCE_ADD:
1871 case ISD::VECREDUCE_MUL:
1872 case ISD::VECREDUCE_AND:
1873 case ISD::VECREDUCE_OR:
1874 case ISD::VECREDUCE_XOR:
1875 Op = GetPromotedInteger(N->getOperand(0));
1876 break;
1877 case ISD::VECREDUCE_SMAX:
1878 case ISD::VECREDUCE_SMIN:
1879 Op = SExtPromotedInteger(N->getOperand(0));
1880 break;
1881 case ISD::VECREDUCE_UMAX:
1882 case ISD::VECREDUCE_UMIN:
1883 Op = ZExtPromotedInteger(N->getOperand(0));
1884 break;
1885 }
1886
1887 EVT EltVT = Op.getValueType().getVectorElementType();
1888 EVT VT = N->getValueType(0);
1889 if (VT.bitsGE(EltVT))
1890 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1891
1892 // Result size must be >= element size. If this is not the case after
1893 // promotion, also promote the result type and then truncate.
1894 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1895 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1896 }
1897
1898 //===----------------------------------------------------------------------===//
1899 // Integer Result Expansion
1900 //===----------------------------------------------------------------------===//
1901
1902 /// ExpandIntegerResult - This method is called when the specified result of the
1903 /// specified node is found to need expansion. At this point, the node may also
1904 /// have invalid operands or may have other results that need promotion, we just
1905 /// know that (at least) one result needs expansion.
ExpandIntegerResult(SDNode * N,unsigned ResNo)1906 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1907 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1908 dbgs() << "\n");
1909 SDValue Lo, Hi;
1910 Lo = Hi = SDValue();
1911
1912 // See if the target wants to custom expand this node.
1913 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1914 return;
1915
1916 switch (N->getOpcode()) {
1917 default:
1918 #ifndef NDEBUG
1919 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1920 N->dump(&DAG); dbgs() << "\n";
1921 #endif
1922 report_fatal_error("Do not know how to expand the result of this "
1923 "operator!");
1924
1925 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1926 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1927 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1928 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1929 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
1930
1931 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1932 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1933 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1934 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1935 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1936
1937 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1938 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1939 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1940 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1941 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1942 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1943 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
1944 case ISD::CTLZ_ZERO_UNDEF:
1945 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1946 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1947 case ISD::CTTZ_ZERO_UNDEF:
1948 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1949 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1950 case ISD::STRICT_FP_TO_SINT:
1951 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1952 case ISD::STRICT_FP_TO_UINT:
1953 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1954 case ISD::STRICT_LLROUND:
1955 case ISD::STRICT_LLRINT:
1956 case ISD::LLROUND:
1957 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
1958 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1959 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1960 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1961 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1962 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1963 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1964 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1965 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1966 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1967 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1968 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1969 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1970
1971 case ISD::ATOMIC_LOAD_ADD:
1972 case ISD::ATOMIC_LOAD_SUB:
1973 case ISD::ATOMIC_LOAD_AND:
1974 case ISD::ATOMIC_LOAD_CLR:
1975 case ISD::ATOMIC_LOAD_OR:
1976 case ISD::ATOMIC_LOAD_XOR:
1977 case ISD::ATOMIC_LOAD_NAND:
1978 case ISD::ATOMIC_LOAD_MIN:
1979 case ISD::ATOMIC_LOAD_MAX:
1980 case ISD::ATOMIC_LOAD_UMIN:
1981 case ISD::ATOMIC_LOAD_UMAX:
1982 case ISD::ATOMIC_SWAP:
1983 case ISD::ATOMIC_CMP_SWAP: {
1984 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1985 SplitInteger(Tmp.first, Lo, Hi);
1986 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1987 break;
1988 }
1989 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
1990 AtomicSDNode *AN = cast<AtomicSDNode>(N);
1991 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1992 SDValue Tmp = DAG.getAtomicCmpSwap(
1993 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1994 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1995 AN->getMemOperand());
1996
1997 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1998 // success simply by comparing the loaded value against the ingoing
1999 // comparison.
2000 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2001 N->getOperand(2), ISD::SETEQ);
2002
2003 SplitInteger(Tmp, Lo, Hi);
2004 ReplaceValueWith(SDValue(N, 1), Success);
2005 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2006 break;
2007 }
2008
2009 case ISD::AND:
2010 case ISD::OR:
2011 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2012
2013 case ISD::UMAX:
2014 case ISD::SMAX:
2015 case ISD::UMIN:
2016 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2017
2018 case ISD::ADD:
2019 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2020
2021 case ISD::ADDC:
2022 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2023
2024 case ISD::ADDE:
2025 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2026
2027 case ISD::ADDCARRY:
2028 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
2029
2030 case ISD::SHL:
2031 case ISD::SRA:
2032 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2033
2034 case ISD::SADDO:
2035 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2036 case ISD::UADDO:
2037 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2038 case ISD::UMULO:
2039 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2040
2041 case ISD::SADDSAT:
2042 case ISD::UADDSAT:
2043 case ISD::SSUBSAT:
2044 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2045
2046 case ISD::SMULFIX:
2047 case ISD::SMULFIXSAT:
2048 case ISD::UMULFIX:
2049 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2050
2051 case ISD::SDIVFIX:
2052 case ISD::SDIVFIXSAT:
2053 case ISD::UDIVFIX:
2054 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2055
2056 case ISD::VECREDUCE_ADD:
2057 case ISD::VECREDUCE_MUL:
2058 case ISD::VECREDUCE_AND:
2059 case ISD::VECREDUCE_OR:
2060 case ISD::VECREDUCE_XOR:
2061 case ISD::VECREDUCE_SMAX:
2062 case ISD::VECREDUCE_SMIN:
2063 case ISD::VECREDUCE_UMAX:
2064 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2065 }
2066
2067 // If Lo/Hi is null, the sub-method took care of registering results etc.
2068 if (Lo.getNode())
2069 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2070 }
2071
2072 /// Lower an atomic node to the appropriate builtin call.
ExpandAtomic(SDNode * Node)2073 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2074 unsigned Opc = Node->getOpcode();
2075 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2076 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
2077 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
2078
2079 EVT RetVT = Node->getValueType(0);
2080 SmallVector<SDValue, 4> Ops(Node->op_begin() + 1, Node->op_end());
2081 TargetLowering::MakeLibCallOptions CallOptions;
2082 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2083 Node->getOperand(0));
2084 }
2085
2086 /// N is a shift by a value that needs to be expanded,
2087 /// and the shift amount is a constant 'Amt'. Expand the operation.
ExpandShiftByConstant(SDNode * N,const APInt & Amt,SDValue & Lo,SDValue & Hi)2088 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2089 SDValue &Lo, SDValue &Hi) {
2090 SDLoc DL(N);
2091 // Expand the incoming operand to be shifted, so that we have its parts
2092 SDValue InL, InH;
2093 GetExpandedInteger(N->getOperand(0), InL, InH);
2094
2095 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2096 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2097 if (!Amt) {
2098 Lo = InL;
2099 Hi = InH;
2100 return;
2101 }
2102
2103 EVT NVT = InL.getValueType();
2104 unsigned VTBits = N->getValueType(0).getSizeInBits();
2105 unsigned NVTBits = NVT.getSizeInBits();
2106 EVT ShTy = N->getOperand(1).getValueType();
2107
2108 if (N->getOpcode() == ISD::SHL) {
2109 if (Amt.ugt(VTBits)) {
2110 Lo = Hi = DAG.getConstant(0, DL, NVT);
2111 } else if (Amt.ugt(NVTBits)) {
2112 Lo = DAG.getConstant(0, DL, NVT);
2113 Hi = DAG.getNode(ISD::SHL, DL,
2114 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2115 } else if (Amt == NVTBits) {
2116 Lo = DAG.getConstant(0, DL, NVT);
2117 Hi = InL;
2118 } else {
2119 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
2120 Hi = DAG.getNode(ISD::OR, DL, NVT,
2121 DAG.getNode(ISD::SHL, DL, NVT, InH,
2122 DAG.getConstant(Amt, DL, ShTy)),
2123 DAG.getNode(ISD::SRL, DL, NVT, InL,
2124 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2125 }
2126 return;
2127 }
2128
2129 if (N->getOpcode() == ISD::SRL) {
2130 if (Amt.ugt(VTBits)) {
2131 Lo = Hi = DAG.getConstant(0, DL, NVT);
2132 } else if (Amt.ugt(NVTBits)) {
2133 Lo = DAG.getNode(ISD::SRL, DL,
2134 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2135 Hi = DAG.getConstant(0, DL, NVT);
2136 } else if (Amt == NVTBits) {
2137 Lo = InH;
2138 Hi = DAG.getConstant(0, DL, NVT);
2139 } else {
2140 Lo = DAG.getNode(ISD::OR, DL, NVT,
2141 DAG.getNode(ISD::SRL, DL, NVT, InL,
2142 DAG.getConstant(Amt, DL, ShTy)),
2143 DAG.getNode(ISD::SHL, DL, NVT, InH,
2144 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2145 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2146 }
2147 return;
2148 }
2149
2150 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2151 if (Amt.ugt(VTBits)) {
2152 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2153 DAG.getConstant(NVTBits - 1, DL, ShTy));
2154 } else if (Amt.ugt(NVTBits)) {
2155 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2156 DAG.getConstant(Amt - NVTBits, DL, ShTy));
2157 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2158 DAG.getConstant(NVTBits - 1, DL, ShTy));
2159 } else if (Amt == NVTBits) {
2160 Lo = InH;
2161 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2162 DAG.getConstant(NVTBits - 1, DL, ShTy));
2163 } else {
2164 Lo = DAG.getNode(ISD::OR, DL, NVT,
2165 DAG.getNode(ISD::SRL, DL, NVT, InL,
2166 DAG.getConstant(Amt, DL, ShTy)),
2167 DAG.getNode(ISD::SHL, DL, NVT, InH,
2168 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2169 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2170 }
2171 }
2172
2173 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2174 /// this shift based on knowledge of the high bit of the shift amount. If we
2175 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2176 /// shift amount.
2177 bool DAGTypeLegalizer::
ExpandShiftWithKnownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2178 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2179 SDValue Amt = N->getOperand(1);
2180 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2181 EVT ShTy = Amt.getValueType();
2182 unsigned ShBits = ShTy.getScalarSizeInBits();
2183 unsigned NVTBits = NVT.getScalarSizeInBits();
2184 assert(isPowerOf2_32(NVTBits) &&
2185 "Expanded integer type size not a power of two!");
2186 SDLoc dl(N);
2187
2188 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2189 KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2190
2191 // If we don't know anything about the high bits, exit.
2192 if (((Known.Zero|Known.One) & HighBitMask) == 0)
2193 return false;
2194
2195 // Get the incoming operand to be shifted.
2196 SDValue InL, InH;
2197 GetExpandedInteger(N->getOperand(0), InL, InH);
2198
2199 // If we know that any of the high bits of the shift amount are one, then we
2200 // can do this as a couple of simple shifts.
2201 if (Known.One.intersects(HighBitMask)) {
2202 // Mask out the high bit, which we know is set.
2203 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2204 DAG.getConstant(~HighBitMask, dl, ShTy));
2205
2206 switch (N->getOpcode()) {
2207 default: llvm_unreachable("Unknown shift");
2208 case ISD::SHL:
2209 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2210 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2211 return true;
2212 case ISD::SRL:
2213 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2214 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2215 return true;
2216 case ISD::SRA:
2217 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2218 DAG.getConstant(NVTBits - 1, dl, ShTy));
2219 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2220 return true;
2221 }
2222 }
2223
2224 // If we know that all of the high bits of the shift amount are zero, then we
2225 // can do this as a couple of simple shifts.
2226 if (HighBitMask.isSubsetOf(Known.Zero)) {
2227 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2228 // shift if x is zero. We can use XOR here because x is known to be smaller
2229 // than 32.
2230 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2231 DAG.getConstant(NVTBits - 1, dl, ShTy));
2232
2233 unsigned Op1, Op2;
2234 switch (N->getOpcode()) {
2235 default: llvm_unreachable("Unknown shift");
2236 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2237 case ISD::SRL:
2238 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2239 }
2240
2241 // When shifting right the arithmetic for Lo and Hi is swapped.
2242 if (N->getOpcode() != ISD::SHL)
2243 std::swap(InL, InH);
2244
2245 // Use a little trick to get the bits that move from Lo to Hi. First
2246 // shift by one bit.
2247 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2248 // Then compute the remaining shift with amount-1.
2249 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2250
2251 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2252 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2253
2254 if (N->getOpcode() != ISD::SHL)
2255 std::swap(Hi, Lo);
2256 return true;
2257 }
2258
2259 return false;
2260 }
2261
2262 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2263 /// of any size.
2264 bool DAGTypeLegalizer::
ExpandShiftWithUnknownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2265 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2266 SDValue Amt = N->getOperand(1);
2267 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2268 EVT ShTy = Amt.getValueType();
2269 unsigned NVTBits = NVT.getSizeInBits();
2270 assert(isPowerOf2_32(NVTBits) &&
2271 "Expanded integer type size not a power of two!");
2272 SDLoc dl(N);
2273
2274 // Get the incoming operand to be shifted.
2275 SDValue InL, InH;
2276 GetExpandedInteger(N->getOperand(0), InL, InH);
2277
2278 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2279 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2280 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2281 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2282 Amt, NVBitsNode, ISD::SETULT);
2283 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2284 Amt, DAG.getConstant(0, dl, ShTy),
2285 ISD::SETEQ);
2286
2287 SDValue LoS, HiS, LoL, HiL;
2288 switch (N->getOpcode()) {
2289 default: llvm_unreachable("Unknown shift");
2290 case ISD::SHL:
2291 // Short: ShAmt < NVTBits
2292 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2293 HiS = DAG.getNode(ISD::OR, dl, NVT,
2294 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2295 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2296
2297 // Long: ShAmt >= NVTBits
2298 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2299 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2300
2301 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2302 Hi = DAG.getSelect(dl, NVT, isZero, InH,
2303 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2304 return true;
2305 case ISD::SRL:
2306 // Short: ShAmt < NVTBits
2307 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2308 LoS = DAG.getNode(ISD::OR, dl, NVT,
2309 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2310 // FIXME: If Amt is zero, the following shift generates an undefined result
2311 // on some architectures.
2312 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2313
2314 // Long: ShAmt >= NVTBits
2315 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2316 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2317
2318 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2319 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2320 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2321 return true;
2322 case ISD::SRA:
2323 // Short: ShAmt < NVTBits
2324 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2325 LoS = DAG.getNode(ISD::OR, dl, NVT,
2326 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2327 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2328
2329 // Long: ShAmt >= NVTBits
2330 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2331 DAG.getConstant(NVTBits - 1, dl, ShTy));
2332 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2333
2334 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2335 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2336 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2337 return true;
2338 }
2339 }
2340
getExpandedMinMaxOps(int Op)2341 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2342
2343 switch (Op) {
2344 default: llvm_unreachable("invalid min/max opcode");
2345 case ISD::SMAX:
2346 return std::make_pair(ISD::SETGT, ISD::UMAX);
2347 case ISD::UMAX:
2348 return std::make_pair(ISD::SETUGT, ISD::UMAX);
2349 case ISD::SMIN:
2350 return std::make_pair(ISD::SETLT, ISD::UMIN);
2351 case ISD::UMIN:
2352 return std::make_pair(ISD::SETULT, ISD::UMIN);
2353 }
2354 }
2355
ExpandIntRes_MINMAX(SDNode * N,SDValue & Lo,SDValue & Hi)2356 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2357 SDValue &Lo, SDValue &Hi) {
2358 SDLoc DL(N);
2359 ISD::NodeType LoOpc;
2360 ISD::CondCode CondC;
2361 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2362
2363 // Expand the subcomponents.
2364 SDValue LHSL, LHSH, RHSL, RHSH;
2365 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2366 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2367
2368 // Value types
2369 EVT NVT = LHSL.getValueType();
2370 EVT CCT = getSetCCResultType(NVT);
2371
2372 // Hi part is always the same op
2373 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2374
2375 // We need to know whether to select Lo part that corresponds to 'winning'
2376 // Hi part or if Hi parts are equal.
2377 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2378 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2379
2380 // Lo part corresponding to the 'winning' Hi part
2381 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2382
2383 // Recursed Lo part if Hi parts are equal, this uses unsigned version
2384 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2385
2386 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2387 }
2388
ExpandIntRes_ADDSUB(SDNode * N,SDValue & Lo,SDValue & Hi)2389 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2390 SDValue &Lo, SDValue &Hi) {
2391 SDLoc dl(N);
2392 // Expand the subcomponents.
2393 SDValue LHSL, LHSH, RHSL, RHSH;
2394 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2395 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2396
2397 EVT NVT = LHSL.getValueType();
2398 SDValue LoOps[2] = { LHSL, RHSL };
2399 SDValue HiOps[3] = { LHSH, RHSH };
2400
2401 bool HasOpCarry = TLI.isOperationLegalOrCustom(
2402 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2403 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2404 if (HasOpCarry) {
2405 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2406 if (N->getOpcode() == ISD::ADD) {
2407 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2408 HiOps[2] = Lo.getValue(1);
2409 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2410 } else {
2411 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2412 HiOps[2] = Lo.getValue(1);
2413 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2414 }
2415 return;
2416 }
2417
2418 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2419 // them. TODO: Teach operation legalization how to expand unsupported
2420 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2421 // a carry of type MVT::Glue, but there doesn't seem to be any way to
2422 // generate a value of this type in the expanded code sequence.
2423 bool hasCarry =
2424 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2425 ISD::ADDC : ISD::SUBC,
2426 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2427
2428 if (hasCarry) {
2429 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2430 if (N->getOpcode() == ISD::ADD) {
2431 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2432 HiOps[2] = Lo.getValue(1);
2433 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2434 } else {
2435 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2436 HiOps[2] = Lo.getValue(1);
2437 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2438 }
2439 return;
2440 }
2441
2442 bool hasOVF =
2443 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2444 ISD::UADDO : ISD::USUBO,
2445 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2446 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
2447
2448 if (hasOVF) {
2449 EVT OvfVT = getSetCCResultType(NVT);
2450 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2451 int RevOpc;
2452 if (N->getOpcode() == ISD::ADD) {
2453 RevOpc = ISD::SUB;
2454 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2455 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2456 } else {
2457 RevOpc = ISD::ADD;
2458 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2459 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2460 }
2461 SDValue OVF = Lo.getValue(1);
2462
2463 switch (BoolType) {
2464 case TargetLoweringBase::UndefinedBooleanContent:
2465 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2466 LLVM_FALLTHROUGH;
2467 case TargetLoweringBase::ZeroOrOneBooleanContent:
2468 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2469 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2470 break;
2471 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2472 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2473 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2474 }
2475 return;
2476 }
2477
2478 if (N->getOpcode() == ISD::ADD) {
2479 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2480 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2481 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2482 ISD::SETULT);
2483
2484 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
2485 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2486 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2487 return;
2488 }
2489
2490 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2491 DAG.getConstant(1, dl, NVT),
2492 DAG.getConstant(0, dl, NVT));
2493 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2494 ISD::SETULT);
2495 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2496 DAG.getConstant(1, dl, NVT), Carry1);
2497 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2498 } else {
2499 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2500 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2501 SDValue Cmp =
2502 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2503 LoOps[0], LoOps[1], ISD::SETULT);
2504
2505 SDValue Borrow;
2506 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
2507 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2508 else
2509 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2510 DAG.getConstant(0, dl, NVT));
2511
2512 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2513 }
2514 }
2515
ExpandIntRes_ADDSUBC(SDNode * N,SDValue & Lo,SDValue & Hi)2516 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2517 SDValue &Lo, SDValue &Hi) {
2518 // Expand the subcomponents.
2519 SDValue LHSL, LHSH, RHSL, RHSH;
2520 SDLoc dl(N);
2521 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2522 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2523 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2524 SDValue LoOps[2] = { LHSL, RHSL };
2525 SDValue HiOps[3] = { LHSH, RHSH };
2526
2527 if (N->getOpcode() == ISD::ADDC) {
2528 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2529 HiOps[2] = Lo.getValue(1);
2530 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2531 } else {
2532 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2533 HiOps[2] = Lo.getValue(1);
2534 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2535 }
2536
2537 // Legalized the flag result - switch anything that used the old flag to
2538 // use the new one.
2539 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2540 }
2541
ExpandIntRes_ADDSUBE(SDNode * N,SDValue & Lo,SDValue & Hi)2542 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2543 SDValue &Lo, SDValue &Hi) {
2544 // Expand the subcomponents.
2545 SDValue LHSL, LHSH, RHSL, RHSH;
2546 SDLoc dl(N);
2547 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2548 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2549 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2550 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2551 SDValue HiOps[3] = { LHSH, RHSH };
2552
2553 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2554 HiOps[2] = Lo.getValue(1);
2555 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2556
2557 // Legalized the flag result - switch anything that used the old flag to
2558 // use the new one.
2559 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2560 }
2561
ExpandIntRes_UADDSUBO(SDNode * N,SDValue & Lo,SDValue & Hi)2562 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2563 SDValue &Lo, SDValue &Hi) {
2564 SDValue LHS = N->getOperand(0);
2565 SDValue RHS = N->getOperand(1);
2566 SDLoc dl(N);
2567
2568 SDValue Ovf;
2569
2570 unsigned CarryOp, NoCarryOp;
2571 ISD::CondCode Cond;
2572 switch(N->getOpcode()) {
2573 case ISD::UADDO:
2574 CarryOp = ISD::ADDCARRY;
2575 NoCarryOp = ISD::ADD;
2576 Cond = ISD::SETULT;
2577 break;
2578 case ISD::USUBO:
2579 CarryOp = ISD::SUBCARRY;
2580 NoCarryOp = ISD::SUB;
2581 Cond = ISD::SETUGT;
2582 break;
2583 default:
2584 llvm_unreachable("Node has unexpected Opcode");
2585 }
2586
2587 bool HasCarryOp = TLI.isOperationLegalOrCustom(
2588 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2589
2590 if (HasCarryOp) {
2591 // Expand the subcomponents.
2592 SDValue LHSL, LHSH, RHSL, RHSH;
2593 GetExpandedInteger(LHS, LHSL, LHSH);
2594 GetExpandedInteger(RHS, RHSL, RHSH);
2595 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2596 SDValue LoOps[2] = { LHSL, RHSL };
2597 SDValue HiOps[3] = { LHSH, RHSH };
2598
2599 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2600 HiOps[2] = Lo.getValue(1);
2601 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
2602
2603 Ovf = Hi.getValue(1);
2604 } else {
2605 // Expand the result by simply replacing it with the equivalent
2606 // non-overflow-checking operation.
2607 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2608 SplitInteger(Sum, Lo, Hi);
2609
2610 // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2611 // overflows iff a - b > a.
2612 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2613 }
2614
2615 // Legalized the flag result - switch anything that used the old flag to
2616 // use the new one.
2617 ReplaceValueWith(SDValue(N, 1), Ovf);
2618 }
2619
ExpandIntRes_ADDSUBCARRY(SDNode * N,SDValue & Lo,SDValue & Hi)2620 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2621 SDValue &Lo, SDValue &Hi) {
2622 // Expand the subcomponents.
2623 SDValue LHSL, LHSH, RHSL, RHSH;
2624 SDLoc dl(N);
2625 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2626 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2627 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2628 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2629 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2630
2631 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2632 HiOps[2] = Lo.getValue(1);
2633 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2634
2635 // Legalized the flag result - switch anything that used the old flag to
2636 // use the new one.
2637 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2638 }
2639
ExpandIntRes_ANY_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)2640 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2641 SDValue &Lo, SDValue &Hi) {
2642 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2643 SDLoc dl(N);
2644 SDValue Op = N->getOperand(0);
2645 if (Op.getValueType().bitsLE(NVT)) {
2646 // The low part is any extension of the input (which degenerates to a copy).
2647 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2648 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2649 } else {
2650 // For example, extension of an i48 to an i64. The operand type necessarily
2651 // promotes to the result type, so will end up being expanded too.
2652 assert(getTypeAction(Op.getValueType()) ==
2653 TargetLowering::TypePromoteInteger &&
2654 "Only know how to promote this result!");
2655 SDValue Res = GetPromotedInteger(Op);
2656 assert(Res.getValueType() == N->getValueType(0) &&
2657 "Operand over promoted?");
2658 // Split the promoted operand. This will simplify when it is expanded.
2659 SplitInteger(Res, Lo, Hi);
2660 }
2661 }
2662
ExpandIntRes_AssertSext(SDNode * N,SDValue & Lo,SDValue & Hi)2663 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2664 SDValue &Lo, SDValue &Hi) {
2665 SDLoc dl(N);
2666 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2667 EVT NVT = Lo.getValueType();
2668 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2669 unsigned NVTBits = NVT.getSizeInBits();
2670 unsigned EVTBits = EVT.getSizeInBits();
2671
2672 if (NVTBits < EVTBits) {
2673 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2674 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2675 EVTBits - NVTBits)));
2676 } else {
2677 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2678 // The high part replicates the sign bit of Lo, make it explicit.
2679 Hi = DAG.getNode(
2680 ISD::SRA, dl, NVT, Lo,
2681 DAG.getConstant(NVTBits - 1, dl,
2682 TLI.getPointerRangeTy(DAG.getDataLayout())));
2683 }
2684 }
2685
ExpandIntRes_AssertZext(SDNode * N,SDValue & Lo,SDValue & Hi)2686 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2687 SDValue &Lo, SDValue &Hi) {
2688 SDLoc dl(N);
2689 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2690 EVT NVT = Lo.getValueType();
2691 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2692 unsigned NVTBits = NVT.getSizeInBits();
2693 unsigned EVTBits = EVT.getSizeInBits();
2694
2695 if (NVTBits < EVTBits) {
2696 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2697 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2698 EVTBits - NVTBits)));
2699 } else {
2700 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2701 // The high part must be zero, make it explicit.
2702 Hi = DAG.getConstant(0, dl, NVT);
2703 }
2704 }
2705
ExpandIntRes_BITREVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)2706 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2707 SDValue &Lo, SDValue &Hi) {
2708 SDLoc dl(N);
2709 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2710 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2711 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2712 }
2713
ExpandIntRes_BSWAP(SDNode * N,SDValue & Lo,SDValue & Hi)2714 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2715 SDValue &Lo, SDValue &Hi) {
2716 SDLoc dl(N);
2717 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2718 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2719 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2720 }
2721
ExpandIntRes_Constant(SDNode * N,SDValue & Lo,SDValue & Hi)2722 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2723 SDValue &Lo, SDValue &Hi) {
2724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2725 unsigned NBitWidth = NVT.getSizeInBits();
2726 auto Constant = cast<ConstantSDNode>(N);
2727 const APInt &Cst = Constant->getAPIntValue();
2728 bool IsTarget = Constant->isTargetOpcode();
2729 bool IsOpaque = Constant->isOpaque();
2730 SDLoc dl(N);
2731 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2732 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2733 IsOpaque);
2734 }
2735
ExpandIntRes_ABS(SDNode * N,SDValue & Lo,SDValue & Hi)2736 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2737 SDLoc dl(N);
2738
2739 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2740 EVT VT = N->getValueType(0);
2741 SDValue N0 = N->getOperand(0);
2742 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2743 DAG.getConstant(0, dl, VT), N0);
2744 SDValue NegLo, NegHi;
2745 SplitInteger(Neg, NegLo, NegHi);
2746
2747 GetExpandedInteger(N0, Lo, Hi);
2748 EVT NVT = Lo.getValueType();
2749 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2750 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2751 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2752 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2753 }
2754
ExpandIntRes_CTLZ(SDNode * N,SDValue & Lo,SDValue & Hi)2755 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2756 SDValue &Lo, SDValue &Hi) {
2757 SDLoc dl(N);
2758 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2759 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2760 EVT NVT = Lo.getValueType();
2761
2762 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2763 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2764
2765 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2766 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2767
2768 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2769 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2770 DAG.getConstant(NVT.getSizeInBits(), dl,
2771 NVT)));
2772 Hi = DAG.getConstant(0, dl, NVT);
2773 }
2774
ExpandIntRes_CTPOP(SDNode * N,SDValue & Lo,SDValue & Hi)2775 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2776 SDValue &Lo, SDValue &Hi) {
2777 SDLoc dl(N);
2778 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2779 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2780 EVT NVT = Lo.getValueType();
2781 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2782 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2783 Hi = DAG.getConstant(0, dl, NVT);
2784 }
2785
ExpandIntRes_CTTZ(SDNode * N,SDValue & Lo,SDValue & Hi)2786 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2787 SDValue &Lo, SDValue &Hi) {
2788 SDLoc dl(N);
2789 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2790 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2791 EVT NVT = Lo.getValueType();
2792
2793 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2794 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2795
2796 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2797 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2798
2799 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2800 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2801 DAG.getConstant(NVT.getSizeInBits(), dl,
2802 NVT)));
2803 Hi = DAG.getConstant(0, dl, NVT);
2804 }
2805
ExpandIntRes_FLT_ROUNDS(SDNode * N,SDValue & Lo,SDValue & Hi)2806 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2807 SDValue &Hi) {
2808 SDLoc dl(N);
2809 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2810 unsigned NBitWidth = NVT.getSizeInBits();
2811
2812 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2813 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0));
2814 SDValue Chain = Lo.getValue(1);
2815 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2816 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2817 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2818
2819 // Legalize the chain result - switch anything that used the old chain to
2820 // use the new one.
2821 ReplaceValueWith(SDValue(N, 1), Chain);
2822 }
2823
ExpandIntRes_FP_TO_SINT(SDNode * N,SDValue & Lo,SDValue & Hi)2824 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2825 SDValue &Hi) {
2826 SDLoc dl(N);
2827 EVT VT = N->getValueType(0);
2828
2829 bool IsStrict = N->isStrictFPOpcode();
2830 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2831 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2832 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2833 Op = GetPromotedFloat(Op);
2834
2835 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
2836 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2837 Op = GetSoftPromotedHalf(Op);
2838 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
2839 }
2840
2841 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
2842 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2843 TargetLowering::MakeLibCallOptions CallOptions;
2844 CallOptions.setSExt(true);
2845 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
2846 CallOptions, dl, Chain);
2847 SplitInteger(Tmp.first, Lo, Hi);
2848
2849 if (IsStrict)
2850 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2851 }
2852
ExpandIntRes_FP_TO_UINT(SDNode * N,SDValue & Lo,SDValue & Hi)2853 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2854 SDValue &Hi) {
2855 SDLoc dl(N);
2856 EVT VT = N->getValueType(0);
2857
2858 bool IsStrict = N->isStrictFPOpcode();
2859 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2860 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2861 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2862 Op = GetPromotedFloat(Op);
2863
2864 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
2865 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2866 Op = GetSoftPromotedHalf(Op);
2867 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
2868 }
2869
2870 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
2871 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2872 TargetLowering::MakeLibCallOptions CallOptions;
2873 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
2874 CallOptions, dl, Chain);
2875 SplitInteger(Tmp.first, Lo, Hi);
2876
2877 if (IsStrict)
2878 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2879 }
2880
ExpandIntRes_LLROUND_LLRINT(SDNode * N,SDValue & Lo,SDValue & Hi)2881 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
2882 SDValue &Hi) {
2883 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
2884
2885 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
2886 "Input type needs to be promoted!");
2887
2888 EVT VT = Op.getValueType();
2889
2890 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2891 if (N->getOpcode() == ISD::LLROUND ||
2892 N->getOpcode() == ISD::STRICT_LLROUND) {
2893 if (VT == MVT::f32)
2894 LC = RTLIB::LLROUND_F32;
2895 else if (VT == MVT::f64)
2896 LC = RTLIB::LLROUND_F64;
2897 else if (VT == MVT::f80)
2898 LC = RTLIB::LLROUND_F80;
2899 else if (VT == MVT::f128)
2900 LC = RTLIB::LLROUND_F128;
2901 else if (VT == MVT::ppcf128)
2902 LC = RTLIB::LLROUND_PPCF128;
2903 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
2904 } else if (N->getOpcode() == ISD::LLRINT ||
2905 N->getOpcode() == ISD::STRICT_LLRINT) {
2906 if (VT == MVT::f32)
2907 LC = RTLIB::LLRINT_F32;
2908 else if (VT == MVT::f64)
2909 LC = RTLIB::LLRINT_F64;
2910 else if (VT == MVT::f80)
2911 LC = RTLIB::LLRINT_F80;
2912 else if (VT == MVT::f128)
2913 LC = RTLIB::LLRINT_F128;
2914 else if (VT == MVT::ppcf128)
2915 LC = RTLIB::LLRINT_PPCF128;
2916 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
2917 } else
2918 llvm_unreachable("Unexpected opcode!");
2919
2920 SDLoc dl(N);
2921 EVT RetVT = N->getValueType(0);
2922 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();
2923
2924 TargetLowering::MakeLibCallOptions CallOptions;
2925 CallOptions.setSExt(true);
2926 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2927 Op, CallOptions, dl,
2928 Chain);
2929 SplitInteger(Tmp.first, Lo, Hi);
2930
2931 if (N->isStrictFPOpcode())
2932 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2933 }
2934
ExpandIntRes_LOAD(LoadSDNode * N,SDValue & Lo,SDValue & Hi)2935 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2936 SDValue &Lo, SDValue &Hi) {
2937 if (N->isAtomic()) {
2938 // It's typical to have larger CAS than atomic load instructions.
2939 SDLoc dl(N);
2940 EVT VT = N->getMemoryVT();
2941 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2942 SDValue Zero = DAG.getConstant(0, dl, VT);
2943 SDValue Swap = DAG.getAtomicCmpSwap(
2944 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
2945 VT, VTs, N->getOperand(0),
2946 N->getOperand(1), Zero, Zero, N->getMemOperand());
2947 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2948 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2949 return;
2950 }
2951
2952 if (ISD::isNormalLoad(N)) {
2953 ExpandRes_NormalLoad(N, Lo, Hi);
2954 return;
2955 }
2956
2957 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2958
2959 EVT VT = N->getValueType(0);
2960 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2961 SDValue Ch = N->getChain();
2962 SDValue Ptr = N->getBasePtr();
2963 ISD::LoadExtType ExtType = N->getExtensionType();
2964 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2965 AAMDNodes AAInfo = N->getAAInfo();
2966 SDLoc dl(N);
2967
2968 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2969
2970 if (N->getMemoryVT().bitsLE(NVT)) {
2971 EVT MemVT = N->getMemoryVT();
2972
2973 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2974 N->getOriginalAlign(), MMOFlags, AAInfo);
2975
2976 // Remember the chain.
2977 Ch = Lo.getValue(1);
2978
2979 if (ExtType == ISD::SEXTLOAD) {
2980 // The high part is obtained by SRA'ing all but one of the bits of the
2981 // lo part.
2982 unsigned LoSize = Lo.getValueSizeInBits();
2983 Hi = DAG.getNode(
2984 ISD::SRA, dl, NVT, Lo,
2985 DAG.getConstant(LoSize - 1, dl,
2986 TLI.getPointerRangeTy(DAG.getDataLayout())));
2987 } else if (ExtType == ISD::ZEXTLOAD) {
2988 // The high part is just a zero.
2989 Hi = DAG.getConstant(0, dl, NVT);
2990 } else {
2991 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2992 // The high part is undefined.
2993 Hi = DAG.getUNDEF(NVT);
2994 }
2995 } else if (DAG.getDataLayout().isLittleEndian()) {
2996 // Little-endian - low bits are at low addresses.
2997 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
2998 N->getOriginalAlign(), MMOFlags, AAInfo);
2999
3000 unsigned ExcessBits =
3001 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3002 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3003
3004 // Increment the pointer to the other half.
3005 unsigned IncrementSize = NVT.getSizeInBits()/8;
3006 Ptr = DAG.getMemBasePlusOffset(Ptr, IncrementSize, dl);
3007 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3008 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3009 N->getOriginalAlign(), MMOFlags, AAInfo);
3010
3011 // Build a factor node to remember that this load is independent of the
3012 // other one.
3013 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3014 Hi.getValue(1));
3015 } else {
3016 // Big-endian - high bits are at low addresses. Favor aligned loads at
3017 // the cost of some bit-fiddling.
3018 EVT MemVT = N->getMemoryVT();
3019 unsigned EBytes = MemVT.getStoreSize();
3020 unsigned IncrementSize = NVT.getSizeInBits()/8;
3021 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3022
3023 // Load both the high bits and maybe some of the low bits.
3024 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3025 EVT::getIntegerVT(*DAG.getContext(),
3026 MemVT.getSizeInBits() - ExcessBits),
3027 N->getOriginalAlign(), MMOFlags, AAInfo);
3028
3029 // Increment the pointer to the other half.
3030 Ptr = DAG.getMemBasePlusOffset(Ptr, IncrementSize, dl);
3031 // Load the rest of the low bits.
3032 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3033 N->getPointerInfo().getWithOffset(IncrementSize),
3034 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3035 N->getOriginalAlign(), MMOFlags, AAInfo);
3036
3037 // Build a factor node to remember that this load is independent of the
3038 // other one.
3039 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3040 Hi.getValue(1));
3041
3042 if (ExcessBits < NVT.getSizeInBits()) {
3043 // Transfer low bits from the bottom of Hi to the top of Lo.
3044 Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
3045 DAG.getNode(ISD::SHL, dl, NVT, Hi,
3046 DAG.getConstant(ExcessBits, dl,
3047 TLI.getPointerRangeTy(
3048 DAG.getDataLayout()))));
3049 // Move high bits to the right position in Hi.
3050 Hi = DAG.getNode(
3051 ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, Hi,
3052 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3053 TLI.getPointerRangeTy(DAG.getDataLayout())));
3054 }
3055 }
3056
3057 // Legalize the chain result - switch anything that used the old chain to
3058 // use the new one.
3059 ReplaceValueWith(SDValue(N, 1), Ch);
3060 }
3061
ExpandIntRes_Logical(SDNode * N,SDValue & Lo,SDValue & Hi)3062 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
3063 SDValue &Lo, SDValue &Hi) {
3064 SDLoc dl(N);
3065 SDValue LL, LH, RL, RH;
3066 GetExpandedInteger(N->getOperand(0), LL, LH);
3067 GetExpandedInteger(N->getOperand(1), RL, RH);
3068 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
3069 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3070 }
3071
ExpandIntRes_MUL(SDNode * N,SDValue & Lo,SDValue & Hi)3072 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
3073 SDValue &Lo, SDValue &Hi) {
3074 EVT VT = N->getValueType(0);
3075 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3076 SDLoc dl(N);
3077
3078 SDValue LL, LH, RL, RH;
3079 GetExpandedInteger(N->getOperand(0), LL, LH);
3080 GetExpandedInteger(N->getOperand(1), RL, RH);
3081
3082 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
3083 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3084 LL, LH, RL, RH))
3085 return;
3086
3087 // If nothing else, we can make a libcall.
3088 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3089 if (VT == MVT::i16)
3090 LC = RTLIB::MUL_I16;
3091 else if (VT == MVT::i32)
3092 LC = RTLIB::MUL_I32;
3093 else if (VT == MVT::i64)
3094 LC = RTLIB::MUL_I64;
3095 else if (VT == MVT::i128)
3096 LC = RTLIB::MUL_I128;
3097
3098 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
3099 // We'll expand the multiplication by brute force because we have no other
3100 // options. This is a trivially-generalized version of the code from
3101 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
3102 // 4.3.1).
3103 unsigned Bits = NVT.getSizeInBits();
3104 unsigned HalfBits = Bits >> 1;
3105 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
3106 NVT);
3107 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
3108 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
3109
3110 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
3111 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
3112
3113 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3114 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
3115 // The type from TLI is too small to fit the shift amount we want.
3116 // Override it with i32. The shift will have to be legalized.
3117 ShiftAmtTy = MVT::i32;
3118 }
3119 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
3120 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
3121 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
3122 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
3123
3124 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
3125 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
3126 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
3127 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
3128
3129 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
3130 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
3131 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
3132
3133 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
3134 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
3135 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
3136 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
3137 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
3138
3139 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
3140 DAG.getNode(ISD::ADD, dl, NVT,
3141 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
3142 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
3143 return;
3144 }
3145
3146 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3147 TargetLowering::MakeLibCallOptions CallOptions;
3148 CallOptions.setSExt(true);
3149 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3150 Lo, Hi);
3151 }
3152
ExpandIntRes_READCYCLECOUNTER(SDNode * N,SDValue & Lo,SDValue & Hi)3153 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
3154 SDValue &Hi) {
3155 SDLoc DL(N);
3156 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3157 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
3158 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
3159 Lo = R.getValue(0);
3160 Hi = R.getValue(1);
3161 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3162 }
3163
ExpandIntRes_ADDSUBSAT(SDNode * N,SDValue & Lo,SDValue & Hi)3164 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
3165 SDValue &Hi) {
3166 SDValue Result = TLI.expandAddSubSat(N, DAG);
3167 SplitInteger(Result, Lo, Hi);
3168 }
3169
3170 /// This performs an expansion of the integer result for a fixed point
3171 /// multiplication. The default expansion performs rounding down towards
3172 /// negative infinity, though targets that do care about rounding should specify
3173 /// a target hook for rounding and provide their own expansion or lowering of
3174 /// fixed point multiplication to be consistent with rounding.
ExpandIntRes_MULFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3175 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
3176 SDValue &Hi) {
3177 SDLoc dl(N);
3178 EVT VT = N->getValueType(0);
3179 unsigned VTSize = VT.getScalarSizeInBits();
3180 SDValue LHS = N->getOperand(0);
3181 SDValue RHS = N->getOperand(1);
3182 uint64_t Scale = N->getConstantOperandVal(2);
3183 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
3184 N->getOpcode() == ISD::UMULFIXSAT);
3185 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
3186 N->getOpcode() == ISD::SMULFIXSAT);
3187
3188 // Handle special case when scale is equal to zero.
3189 if (!Scale) {
3190 SDValue Result;
3191 if (!Saturating) {
3192 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3193 } else {
3194 EVT BoolVT = getSetCCResultType(VT);
3195 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
3196 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
3197 SDValue Product = Result.getValue(0);
3198 SDValue Overflow = Result.getValue(1);
3199 if (Signed) {
3200 APInt MinVal = APInt::getSignedMinValue(VTSize);
3201 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
3202 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
3203 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3204 SDValue Zero = DAG.getConstant(0, dl, VT);
3205 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
3206 Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
3207 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
3208 } else {
3209 // For unsigned multiplication, we only need to check the max since we
3210 // can't really overflow towards zero.
3211 APInt MaxVal = APInt::getMaxValue(VTSize);
3212 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3213 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
3214 }
3215 }
3216 SplitInteger(Result, Lo, Hi);
3217 return;
3218 }
3219
3220 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
3221 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
3222 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
3223
3224 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3225 SDValue LL, LH, RL, RH;
3226 GetExpandedInteger(LHS, LL, LH);
3227 GetExpandedInteger(RHS, RL, RH);
3228 SmallVector<SDValue, 4> Result;
3229
3230 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
3231 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
3232 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3233 LL, LH, RL, RH)) {
3234 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
3235 return;
3236 }
3237
3238 unsigned NVTSize = NVT.getScalarSizeInBits();
3239 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
3240 "the size of the current value type");
3241 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3242
3243 // After getting the multiplication result in 4 parts, we need to perform a
3244 // shift right by the amount of the scale to get the result in that scale.
3245 //
3246 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
3247 // 128 bits that are cut into 4 32-bit parts:
3248 //
3249 // HH HL LH LL
3250 // |---32---|---32---|---32---|---32---|
3251 // 128 96 64 32 0
3252 //
3253 // |------VTSize-----|
3254 //
3255 // |NVTSize-|
3256 //
3257 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
3258 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
3259 // amount and get Lo and Hi using two funnel shifts. Or for the special case
3260 // when Scale is a multiple of NVTSize we can just pick the result without
3261 // shifting.
3262 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
3263 if (Scale % NVTSize) {
3264 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
3265 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
3266 ShiftAmount);
3267 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
3268 ShiftAmount);
3269 } else {
3270 Lo = Result[Part0];
3271 Hi = Result[Part0 + 1];
3272 }
3273
3274 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
3275 if (!Saturating)
3276 return;
3277
3278 // Can not overflow when there is no integer part.
3279 if (Scale == VTSize)
3280 return;
3281
3282 // To handle saturation we must check for overflow in the multiplication.
3283 //
3284 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
3285 // aren't all zeroes.
3286 //
3287 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
3288 // aren't all ones or all zeroes.
3289 //
3290 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
3291 // highest bit of HH determines saturation direction in the event of signed
3292 // saturation.
3293
3294 SDValue ResultHL = Result[2];
3295 SDValue ResultHH = Result[3];
3296
3297 SDValue SatMax, SatMin;
3298 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
3299 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3300 EVT BoolNVT = getSetCCResultType(NVT);
3301
3302 if (!Signed) {
3303 if (Scale < NVTSize) {
3304 // Overflow happened if ((HH | (HL >> Scale)) != 0).
3305 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3306 DAG.getConstant(Scale, dl, ShiftTy));
3307 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3308 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3309 } else if (Scale == NVTSize) {
3310 // Overflow happened if (HH != 0).
3311 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3312 } else if (Scale < VTSize) {
3313 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
3314 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3315 DAG.getConstant(Scale - NVTSize, dl,
3316 ShiftTy));
3317 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3318 } else
3319 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
3320 "(and saturation can't happen with Scale==VTSize).");
3321
3322 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3323 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3324 return;
3325 }
3326
3327 if (Scale < NVTSize) {
3328 // The number of overflow bits we can check are VTSize - Scale + 1 (we
3329 // include the sign bit). If these top bits are > 0, then we overflowed past
3330 // the max value. If these top bits are < -1, then we overflowed past the
3331 // min value. Otherwise, we did not overflow.
3332 unsigned OverflowBits = VTSize - Scale + 1;
3333 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3334 "Extent of overflow bits must start within HL");
3335 SDValue HLHiMask = DAG.getConstant(
3336 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3337 SDValue HLLoMask = DAG.getConstant(
3338 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3339 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3340 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3341 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3342 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3343 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3344 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3345 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3346 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3347 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3348 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3349 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3350 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3351 } else if (Scale == NVTSize) {
3352 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3353 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3354 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3355 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3356 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3357 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3358 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3359 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3360 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3361 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3362 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3363 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3364 } else if (Scale < VTSize) {
3365 // This is similar to the case when we saturate if Scale < NVTSize, but we
3366 // only need to check HH.
3367 unsigned OverflowBits = VTSize - Scale + 1;
3368 SDValue HHHiMask = DAG.getConstant(
3369 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3370 SDValue HHLoMask = DAG.getConstant(
3371 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3372 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3373 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3374 } else
3375 llvm_unreachable("Illegal scale for signed fixed point mul.");
3376
3377 // Saturate to signed maximum.
3378 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3379 APInt MaxLo = APInt::getAllOnesValue(NVTSize);
3380 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3381 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3382 // Saturate to signed minimum.
3383 APInt MinHi = APInt::getSignedMinValue(NVTSize);
3384 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3385 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3386 }
3387
ExpandIntRes_DIVFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3388 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
3389 SDValue &Hi) {
3390 SDLoc dl(N);
3391 // Try expanding in the existing type first.
3392 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
3393 N->getOperand(1),
3394 N->getConstantOperandVal(2), DAG);
3395
3396 if (!Res)
3397 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
3398 N->getConstantOperandVal(2), TLI, DAG);
3399 SplitInteger(Res, Lo, Hi);
3400 }
3401
ExpandIntRes_SADDSUBO(SDNode * Node,SDValue & Lo,SDValue & Hi)3402 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3403 SDValue &Lo, SDValue &Hi) {
3404 SDValue LHS = Node->getOperand(0);
3405 SDValue RHS = Node->getOperand(1);
3406 SDLoc dl(Node);
3407
3408 // Expand the result by simply replacing it with the equivalent
3409 // non-overflow-checking operation.
3410 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3411 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3412 LHS, RHS);
3413 SplitInteger(Sum, Lo, Hi);
3414
3415 // Compute the overflow.
3416 //
3417 // LHSSign -> LHS >= 0
3418 // RHSSign -> RHS >= 0
3419 // SumSign -> Sum >= 0
3420 //
3421 // Add:
3422 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3423 // Sub:
3424 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3425 //
3426 EVT OType = Node->getValueType(1);
3427 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3428
3429 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3430 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3431 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3432 Node->getOpcode() == ISD::SADDO ?
3433 ISD::SETEQ : ISD::SETNE);
3434
3435 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3436 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3437
3438 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3439
3440 // Use the calculated overflow everywhere.
3441 ReplaceValueWith(SDValue(Node, 1), Cmp);
3442 }
3443
ExpandIntRes_SDIV(SDNode * N,SDValue & Lo,SDValue & Hi)3444 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3445 SDValue &Lo, SDValue &Hi) {
3446 EVT VT = N->getValueType(0);
3447 SDLoc dl(N);
3448 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3449
3450 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3451 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3452 SplitInteger(Res.getValue(0), Lo, Hi);
3453 return;
3454 }
3455
3456 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3457 if (VT == MVT::i16)
3458 LC = RTLIB::SDIV_I16;
3459 else if (VT == MVT::i32)
3460 LC = RTLIB::SDIV_I32;
3461 else if (VT == MVT::i64)
3462 LC = RTLIB::SDIV_I64;
3463 else if (VT == MVT::i128)
3464 LC = RTLIB::SDIV_I128;
3465 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3466
3467 TargetLowering::MakeLibCallOptions CallOptions;
3468 CallOptions.setSExt(true);
3469 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3470 }
3471
ExpandIntRes_Shift(SDNode * N,SDValue & Lo,SDValue & Hi)3472 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3473 SDValue &Lo, SDValue &Hi) {
3474 EVT VT = N->getValueType(0);
3475 SDLoc dl(N);
3476
3477 // If we can emit an efficient shift operation, do so now. Check to see if
3478 // the RHS is a constant.
3479 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3480 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3481
3482 // If we can determine that the high bit of the shift is zero or one, even if
3483 // the low bits are variable, emit this shift in an optimized form.
3484 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3485 return;
3486
3487 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3488 unsigned PartsOpc;
3489 if (N->getOpcode() == ISD::SHL) {
3490 PartsOpc = ISD::SHL_PARTS;
3491 } else if (N->getOpcode() == ISD::SRL) {
3492 PartsOpc = ISD::SRL_PARTS;
3493 } else {
3494 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3495 PartsOpc = ISD::SRA_PARTS;
3496 }
3497
3498 // Next check to see if the target supports this SHL_PARTS operation or if it
3499 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3500 // size, but create a libcall instead.
3501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3502 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3503 const bool LegalOrCustom =
3504 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3505 Action == TargetLowering::Custom;
3506
3507 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3508 // Expand the subcomponents.
3509 SDValue LHSL, LHSH;
3510 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3511 EVT VT = LHSL.getValueType();
3512
3513 // If the shift amount operand is coming from a vector legalization it may
3514 // have an illegal type. Fix that first by casting the operand, otherwise
3515 // the new SHL_PARTS operation would need further legalization.
3516 SDValue ShiftOp = N->getOperand(1);
3517 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3518 assert(ShiftTy.getScalarSizeInBits() >=
3519 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3520 "ShiftAmountTy is too small to cover the range of this type!");
3521 if (ShiftOp.getValueType() != ShiftTy)
3522 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3523
3524 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3525 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3526 Hi = Lo.getValue(1);
3527 return;
3528 }
3529
3530 // Otherwise, emit a libcall.
3531 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3532 bool isSigned;
3533 if (N->getOpcode() == ISD::SHL) {
3534 isSigned = false; /*sign irrelevant*/
3535 if (VT == MVT::i16)
3536 LC = RTLIB::SHL_I16;
3537 else if (VT == MVT::i32)
3538 LC = RTLIB::SHL_I32;
3539 else if (VT == MVT::i64)
3540 LC = RTLIB::SHL_I64;
3541 else if (VT == MVT::i128)
3542 LC = RTLIB::SHL_I128;
3543 } else if (N->getOpcode() == ISD::SRL) {
3544 isSigned = false;
3545 if (VT == MVT::i16)
3546 LC = RTLIB::SRL_I16;
3547 else if (VT == MVT::i32)
3548 LC = RTLIB::SRL_I32;
3549 else if (VT == MVT::i64)
3550 LC = RTLIB::SRL_I64;
3551 else if (VT == MVT::i128)
3552 LC = RTLIB::SRL_I128;
3553 } else {
3554 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3555 isSigned = true;
3556 if (VT == MVT::i16)
3557 LC = RTLIB::SRA_I16;
3558 else if (VT == MVT::i32)
3559 LC = RTLIB::SRA_I32;
3560 else if (VT == MVT::i64)
3561 LC = RTLIB::SRA_I64;
3562 else if (VT == MVT::i128)
3563 LC = RTLIB::SRA_I128;
3564 }
3565
3566 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3567 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3568 TargetLowering::MakeLibCallOptions CallOptions;
3569 CallOptions.setSExt(isSigned);
3570 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3571 return;
3572 }
3573
3574 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3575 llvm_unreachable("Unsupported shift!");
3576 }
3577
ExpandIntRes_SIGN_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)3578 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3579 SDValue &Lo, SDValue &Hi) {
3580 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3581 SDLoc dl(N);
3582 SDValue Op = N->getOperand(0);
3583 if (Op.getValueType().bitsLE(NVT)) {
3584 // The low part is sign extension of the input (degenerates to a copy).
3585 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3586 // The high part is obtained by SRA'ing all but one of the bits of low part.
3587 unsigned LoSize = NVT.getSizeInBits();
3588 Hi = DAG.getNode(
3589 ISD::SRA, dl, NVT, Lo,
3590 DAG.getConstant(LoSize - 1, dl,
3591 TLI.getPointerRangeTy(DAG.getDataLayout())));
3592 } else {
3593 // For example, extension of an i48 to an i64. The operand type necessarily
3594 // promotes to the result type, so will end up being expanded too.
3595 assert(getTypeAction(Op.getValueType()) ==
3596 TargetLowering::TypePromoteInteger &&
3597 "Only know how to promote this result!");
3598 SDValue Res = GetPromotedInteger(Op);
3599 assert(Res.getValueType() == N->getValueType(0) &&
3600 "Operand over promoted?");
3601 // Split the promoted operand. This will simplify when it is expanded.
3602 SplitInteger(Res, Lo, Hi);
3603 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3604 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3605 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3606 ExcessBits)));
3607 }
3608 }
3609
3610 void DAGTypeLegalizer::
ExpandIntRes_SIGN_EXTEND_INREG(SDNode * N,SDValue & Lo,SDValue & Hi)3611 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3612 SDLoc dl(N);
3613 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3614 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3615
3616 if (EVT.bitsLE(Lo.getValueType())) {
3617 // sext_inreg the low part if needed.
3618 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3619 N->getOperand(1));
3620
3621 // The high part gets the sign extension from the lo-part. This handles
3622 // things like sextinreg V:i64 from i8.
3623 Hi = DAG.getNode(
3624 ISD::SRA, dl, Hi.getValueType(), Lo,
3625 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3626 TLI.getPointerRangeTy(DAG.getDataLayout())));
3627 } else {
3628 // For example, extension of an i48 to an i64. Leave the low part alone,
3629 // sext_inreg the high part.
3630 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3631 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3632 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3633 ExcessBits)));
3634 }
3635 }
3636
ExpandIntRes_SREM(SDNode * N,SDValue & Lo,SDValue & Hi)3637 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3638 SDValue &Lo, SDValue &Hi) {
3639 EVT VT = N->getValueType(0);
3640 SDLoc dl(N);
3641 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3642
3643 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3644 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3645 SplitInteger(Res.getValue(1), Lo, Hi);
3646 return;
3647 }
3648
3649 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3650 if (VT == MVT::i16)
3651 LC = RTLIB::SREM_I16;
3652 else if (VT == MVT::i32)
3653 LC = RTLIB::SREM_I32;
3654 else if (VT == MVT::i64)
3655 LC = RTLIB::SREM_I64;
3656 else if (VT == MVT::i128)
3657 LC = RTLIB::SREM_I128;
3658 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3659
3660 TargetLowering::MakeLibCallOptions CallOptions;
3661 CallOptions.setSExt(true);
3662 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3663 }
3664
ExpandIntRes_TRUNCATE(SDNode * N,SDValue & Lo,SDValue & Hi)3665 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3666 SDValue &Lo, SDValue &Hi) {
3667 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3668 SDLoc dl(N);
3669 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3670 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3671 N->getOperand(0),
3672 DAG.getConstant(NVT.getSizeInBits(), dl,
3673 TLI.getPointerRangeTy(DAG.getDataLayout())));
3674 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3675 }
3676
ExpandIntRes_XMULO(SDNode * N,SDValue & Lo,SDValue & Hi)3677 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3678 SDValue &Lo, SDValue &Hi) {
3679 EVT VT = N->getValueType(0);
3680 SDLoc dl(N);
3681
3682 if (N->getOpcode() == ISD::UMULO) {
3683 // This section expands the operation into the following sequence of
3684 // instructions. `iNh` here refers to a type which has half the bit width of
3685 // the type the original operation operated on.
3686 //
3687 // %0 = %LHS.HI != 0 && %RHS.HI != 0
3688 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3689 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3690 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3691 // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3692 // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3693 //
3694 // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3695 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3696 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3697 SplitInteger(LHS, LHSLow, LHSHigh);
3698 SplitInteger(RHS, RHSLow, RHSHigh);
3699 EVT HalfVT = LHSLow.getValueType()
3700 , BitVT = N->getValueType(1);
3701 SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3702 SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3703
3704 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3705 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3706 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3707 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3708
3709 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3710 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3711 SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3712 One.getValue(0));
3713
3714 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3715 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3716 SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3717 Two.getValue(0));
3718
3719 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3720 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3721 // operation recursively legalized?).
3722 //
3723 // Many backends understand this pattern and will convert into LOHI
3724 // themselves, if applicable.
3725 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3726 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3727 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3728 SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3729 SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3730 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3731 SplitInteger(Five, Lo, Hi);
3732 ReplaceValueWith(SDValue(N, 1), Overflow);
3733 return;
3734 }
3735
3736 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3737 EVT PtrRangeVT = TLI.getPointerRangeTy(DAG.getDataLayout());
3738 Type *PtrTy = PtrRangeVT.getTypeForEVT(*DAG.getContext());
3739
3740 // Replace this with a libcall that will check overflow.
3741 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3742 if (VT == MVT::i32)
3743 LC = RTLIB::MULO_I32;
3744 else if (VT == MVT::i64)
3745 LC = RTLIB::MULO_I64;
3746 else if (VT == MVT::i128)
3747 LC = RTLIB::MULO_I128;
3748 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3749
3750 SDValue Temp = DAG.CreateStackTemporary(PtrRangeVT);
3751 // Temporary for the overflow value, default it to zero.
3752 SDValue Chain =
3753 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrRangeVT),
3754 Temp, MachinePointerInfo());
3755
3756 TargetLowering::ArgListTy Args;
3757 TargetLowering::ArgListEntry Entry;
3758 for (const SDValue &Op : N->op_values()) {
3759 EVT ArgVT = Op.getValueType();
3760 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3761 Entry.Node = Op;
3762 Entry.Ty = ArgTy;
3763 Entry.IsSExt = true;
3764 Entry.IsZExt = false;
3765 Args.push_back(Entry);
3766 }
3767
3768 // Also pass the address of the overflow check.
3769 Entry.Node = Temp;
3770 Entry.Ty = PtrTy->getPointerTo();
3771 Entry.IsSExt = true;
3772 Entry.IsZExt = false;
3773 Args.push_back(Entry);
3774
3775 SDValue Func = DAG.getExternalFunctionSymbol(TLI.getLibcallName(LC));
3776
3777 TargetLowering::CallLoweringInfo CLI(DAG);
3778 CLI.setDebugLoc(dl)
3779 .setChain(Chain)
3780 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3781 .setSExtResult();
3782
3783 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3784
3785 SplitInteger(CallInfo.first, Lo, Hi);
3786 SDValue Temp2 =
3787 DAG.getLoad(PtrRangeVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3788 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3789 DAG.getConstant(0, dl, PtrRangeVT), ISD::SETNE);
3790 // Use the overflow from the libcall everywhere.
3791 ReplaceValueWith(SDValue(N, 1), Ofl);
3792 }
3793
ExpandIntRes_UDIV(SDNode * N,SDValue & Lo,SDValue & Hi)3794 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3795 SDValue &Lo, SDValue &Hi) {
3796 EVT VT = N->getValueType(0);
3797 SDLoc dl(N);
3798 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3799
3800 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
3801 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3802 SplitInteger(Res.getValue(0), Lo, Hi);
3803 return;
3804 }
3805
3806 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3807 if (VT == MVT::i16)
3808 LC = RTLIB::UDIV_I16;
3809 else if (VT == MVT::i32)
3810 LC = RTLIB::UDIV_I32;
3811 else if (VT == MVT::i64)
3812 LC = RTLIB::UDIV_I64;
3813 else if (VT == MVT::i128)
3814 LC = RTLIB::UDIV_I128;
3815 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3816
3817 TargetLowering::MakeLibCallOptions CallOptions;
3818 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3819 }
3820
ExpandIntRes_UREM(SDNode * N,SDValue & Lo,SDValue & Hi)3821 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3822 SDValue &Lo, SDValue &Hi) {
3823 EVT VT = N->getValueType(0);
3824 SDLoc dl(N);
3825 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3826
3827 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
3828 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3829 SplitInteger(Res.getValue(1), Lo, Hi);
3830 return;
3831 }
3832
3833 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3834 if (VT == MVT::i16)
3835 LC = RTLIB::UREM_I16;
3836 else if (VT == MVT::i32)
3837 LC = RTLIB::UREM_I32;
3838 else if (VT == MVT::i64)
3839 LC = RTLIB::UREM_I64;
3840 else if (VT == MVT::i128)
3841 LC = RTLIB::UREM_I128;
3842 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3843
3844 TargetLowering::MakeLibCallOptions CallOptions;
3845 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3846 }
3847
ExpandIntRes_ZERO_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)3848 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3849 SDValue &Lo, SDValue &Hi) {
3850 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3851 SDLoc dl(N);
3852 SDValue Op = N->getOperand(0);
3853 if (Op.getValueType().bitsLE(NVT)) {
3854 // The low part is zero extension of the input (degenerates to a copy).
3855 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3856 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3857 } else {
3858 // For example, extension of an i48 to an i64. The operand type necessarily
3859 // promotes to the result type, so will end up being expanded too.
3860 assert(getTypeAction(Op.getValueType()) ==
3861 TargetLowering::TypePromoteInteger &&
3862 "Only know how to promote this result!");
3863 SDValue Res = GetPromotedInteger(Op);
3864 assert(Res.getValueType() == N->getValueType(0) &&
3865 "Operand over promoted?");
3866 // Split the promoted operand. This will simplify when it is expanded.
3867 SplitInteger(Res, Lo, Hi);
3868 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3869 Hi = DAG.getZeroExtendInReg(Hi, dl,
3870 EVT::getIntegerVT(*DAG.getContext(),
3871 ExcessBits));
3872 }
3873 }
3874
ExpandIntRes_ATOMIC_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)3875 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3876 SDValue &Lo, SDValue &Hi) {
3877 SDLoc dl(N);
3878 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3879 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3880 SDValue Zero = DAG.getConstant(0, dl, VT);
3881 SDValue Swap = DAG.getAtomicCmpSwap(
3882 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
3883 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3884 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3885
3886 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3887 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3888 }
3889
ExpandIntRes_VECREDUCE(SDNode * N,SDValue & Lo,SDValue & Hi)3890 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
3891 SDValue &Lo, SDValue &Hi) {
3892 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
3893 // both halves independently.
3894 SDValue Res = TLI.expandVecReduce(N, DAG);
3895 SplitInteger(Res, Lo, Hi);
3896 }
3897
3898 //===----------------------------------------------------------------------===//
3899 // Integer Operand Expansion
3900 //===----------------------------------------------------------------------===//
3901
3902 /// ExpandIntegerOperand - This method is called when the specified operand of
3903 /// the specified node is found to need expansion. At this point, all of the
3904 /// result types of the node are known to be legal, but other operands of the
3905 /// node may need promotion or expansion as well as the specified one.
ExpandIntegerOperand(SDNode * N,unsigned OpNo)3906 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3907 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3908 dbgs() << "\n");
3909 SDValue Res = SDValue();
3910
3911 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3912 return false;
3913
3914 switch (N->getOpcode()) {
3915 default:
3916 #ifndef NDEBUG
3917 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3918 N->dump(&DAG); dbgs() << "\n";
3919 #endif
3920 report_fatal_error("Do not know how to expand this operator's operand!");
3921
3922 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3923 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3924 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3925 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3926 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3927 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3928 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3929 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3930 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3931 case ISD::STRICT_SINT_TO_FP:
3932 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3933 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3934 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3935 case ISD::STRICT_UINT_TO_FP:
3936 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3937
3938 case ISD::SHL:
3939 case ISD::SRA:
3940 case ISD::SRL:
3941 case ISD::ROTL:
3942 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3943 case ISD::RETURNADDR:
3944 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3945
3946 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3947 }
3948
3949 // If the result is null, the sub-method took care of registering results etc.
3950 if (!Res.getNode()) return false;
3951
3952 // If the result is N, the sub-method updated N in place. Tell the legalizer
3953 // core about this.
3954 if (Res.getNode() == N)
3955 return true;
3956
3957 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3958 "Invalid operand expansion");
3959
3960 ReplaceValueWith(SDValue(N, 0), Res);
3961 return false;
3962 }
3963
3964 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3965 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
IntegerExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl)3966 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3967 SDValue &NewRHS,
3968 ISD::CondCode &CCCode,
3969 const SDLoc &dl) {
3970 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3971 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3972 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3973
3974 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3975 if (RHSLo == RHSHi) {
3976 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3977 if (RHSCST->isAllOnesValue()) {
3978 // Equality comparison to -1.
3979 NewLHS = DAG.getNode(ISD::AND, dl,
3980 LHSLo.getValueType(), LHSLo, LHSHi);
3981 NewRHS = RHSLo;
3982 return;
3983 }
3984 }
3985 }
3986
3987 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3988 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3989 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3990 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3991 return;
3992 }
3993
3994 // If this is a comparison of the sign bit, just look at the top part.
3995 // X > -1, x < 0
3996 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3997 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3998 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3999 NewLHS = LHSHi;
4000 NewRHS = RHSHi;
4001 return;
4002 }
4003
4004 // FIXME: This generated code sucks.
4005 ISD::CondCode LowCC;
4006 switch (CCCode) {
4007 default: llvm_unreachable("Unknown integer setcc!");
4008 case ISD::SETLT:
4009 case ISD::SETULT: LowCC = ISD::SETULT; break;
4010 case ISD::SETGT:
4011 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
4012 case ISD::SETLE:
4013 case ISD::SETULE: LowCC = ISD::SETULE; break;
4014 case ISD::SETGE:
4015 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
4016 }
4017
4018 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
4019 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
4020 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
4021
4022 // NOTE: on targets without efficient SELECT of bools, we can always use
4023 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
4024 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
4025 nullptr);
4026 SDValue LoCmp, HiCmp;
4027 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
4028 TLI.isTypeLegal(RHSLo.getValueType()))
4029 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
4030 RHSLo, LowCC, false, DagCombineInfo, dl);
4031 if (!LoCmp.getNode())
4032 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
4033 RHSLo, LowCC);
4034 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
4035 TLI.isTypeLegal(RHSHi.getValueType()))
4036 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
4037 RHSHi, CCCode, false, DagCombineInfo, dl);
4038 if (!HiCmp.getNode())
4039 HiCmp =
4040 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
4041 LHSHi, RHSHi, DAG.getCondCode(CCCode));
4042
4043 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
4044 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
4045
4046 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
4047 CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
4048
4049 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
4050 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
4051 (LoCmpC && LoCmpC->isNullValue())))) {
4052 // For LE / GE, if high part is known false, ignore the low part.
4053 // For LT / GT: if low part is known false, return the high part.
4054 // if high part is known true, ignore the low part.
4055 NewLHS = HiCmp;
4056 NewRHS = SDValue();
4057 return;
4058 }
4059
4060 if (LHSHi == RHSHi) {
4061 // Comparing the low bits is enough.
4062 NewLHS = LoCmp;
4063 NewRHS = SDValue();
4064 return;
4065 }
4066
4067 // Lower with SETCCCARRY if the target supports it.
4068 EVT HiVT = LHSHi.getValueType();
4069 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
4070 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
4071
4072 // FIXME: Make all targets support this, then remove the other lowering.
4073 if (HasSETCCCARRY) {
4074 // SETCCCARRY can detect < and >= directly. For > and <=, flip
4075 // operands and condition code.
4076 bool FlipOperands = false;
4077 switch (CCCode) {
4078 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
4079 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
4080 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
4081 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
4082 default: break;
4083 }
4084 if (FlipOperands) {
4085 std::swap(LHSLo, RHSLo);
4086 std::swap(LHSHi, RHSHi);
4087 }
4088 // Perform a wide subtraction, feeding the carry from the low part into
4089 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
4090 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
4091 // zero or positive iff LHS >= RHS.
4092 EVT LoVT = LHSLo.getValueType();
4093 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
4094 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
4095 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
4096 LHSHi, RHSHi, LowCmp.getValue(1),
4097 DAG.getCondCode(CCCode));
4098 NewLHS = Res;
4099 NewRHS = SDValue();
4100 return;
4101 }
4102
4103 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
4104 false, DagCombineInfo, dl);
4105 if (!NewLHS.getNode())
4106 NewLHS =
4107 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
4108 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
4109 NewRHS = SDValue();
4110 }
4111
ExpandIntOp_BR_CC(SDNode * N)4112 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
4113 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
4114 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
4115 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4116
4117 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4118 // against zero to select between true and false values.
4119 if (!NewRHS.getNode()) {
4120 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4121 CCCode = ISD::SETNE;
4122 }
4123
4124 // Update N to have the operands specified.
4125 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
4126 DAG.getCondCode(CCCode), NewLHS, NewRHS,
4127 N->getOperand(4)), 0);
4128 }
4129
ExpandIntOp_SELECT_CC(SDNode * N)4130 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
4131 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4132 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
4133 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4134
4135 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4136 // against zero to select between true and false values.
4137 if (!NewRHS.getNode()) {
4138 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4139 CCCode = ISD::SETNE;
4140 }
4141
4142 // Update N to have the operands specified.
4143 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
4144 N->getOperand(2), N->getOperand(3),
4145 DAG.getCondCode(CCCode)), 0);
4146 }
4147
ExpandIntOp_SETCC(SDNode * N)4148 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
4149 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4150 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
4151 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4152
4153 // If ExpandSetCCOperands returned a scalar, use it.
4154 if (!NewRHS.getNode()) {
4155 assert(NewLHS.getValueType() == N->getValueType(0) &&
4156 "Unexpected setcc expansion!");
4157 return NewLHS;
4158 }
4159
4160 // Otherwise, update N to have the operands specified.
4161 return SDValue(
4162 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
4163 }
4164
ExpandIntOp_SETCCCARRY(SDNode * N)4165 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
4166 SDValue LHS = N->getOperand(0);
4167 SDValue RHS = N->getOperand(1);
4168 SDValue Carry = N->getOperand(2);
4169 SDValue Cond = N->getOperand(3);
4170 SDLoc dl = SDLoc(N);
4171
4172 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4173 GetExpandedInteger(LHS, LHSLo, LHSHi);
4174 GetExpandedInteger(RHS, RHSLo, RHSHi);
4175
4176 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
4177 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
4178 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
4179 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
4180 LowCmp.getValue(1), Cond);
4181 }
4182
ExpandIntOp_Shift(SDNode * N)4183 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
4184 // The value being shifted is legal, but the shift amount is too big.
4185 // It follows that either the result of the shift is undefined, or the
4186 // upper half of the shift amount is zero. Just use the lower half.
4187 SDValue Lo, Hi;
4188 GetExpandedInteger(N->getOperand(1), Lo, Hi);
4189 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
4190 }
4191
ExpandIntOp_RETURNADDR(SDNode * N)4192 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
4193 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
4194 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
4195 // constant to valid type.
4196 SDValue Lo, Hi;
4197 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4198 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
4199 }
4200
ExpandIntOp_SINT_TO_FP(SDNode * N)4201 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
4202 bool IsStrict = N->isStrictFPOpcode();
4203 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4204 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4205 EVT DstVT = N->getValueType(0);
4206 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
4207 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4208 "Don't know how to expand this SINT_TO_FP!");
4209 TargetLowering::MakeLibCallOptions CallOptions;
4210 CallOptions.setSExt(true);
4211 std::pair<SDValue, SDValue> Tmp =
4212 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4213
4214 if (!IsStrict)
4215 return Tmp.first;
4216
4217 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4218 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4219 return SDValue();
4220 }
4221
ExpandIntOp_STORE(StoreSDNode * N,unsigned OpNo)4222 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
4223 if (N->isAtomic()) {
4224 // It's typical to have larger CAS than atomic store instructions.
4225 SDLoc dl(N);
4226 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4227 N->getMemoryVT(),
4228 N->getOperand(0), N->getOperand(2),
4229 N->getOperand(1),
4230 N->getMemOperand());
4231 return Swap.getValue(1);
4232 }
4233 if (ISD::isNormalStore(N))
4234 return ExpandOp_NormalStore(N, OpNo);
4235
4236 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
4237 assert(OpNo == 1 && "Can only expand the stored value so far");
4238
4239 EVT VT = N->getOperand(1).getValueType();
4240 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4241 SDValue Ch = N->getChain();
4242 SDValue Ptr = N->getBasePtr();
4243 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4244 AAMDNodes AAInfo = N->getAAInfo();
4245 SDLoc dl(N);
4246 SDValue Lo, Hi;
4247
4248 assert(NVT.isByteSized() && "Expanded type not byte sized!");
4249
4250 if (N->getMemoryVT().bitsLE(NVT)) {
4251 GetExpandedInteger(N->getValue(), Lo, Hi);
4252 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4253 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags,
4254 AAInfo);
4255 }
4256
4257 if (DAG.getDataLayout().isLittleEndian()) {
4258 // Little-endian - low bits are at low addresses.
4259 GetExpandedInteger(N->getValue(), Lo, Hi);
4260
4261 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4262 N->getOriginalAlign(), MMOFlags, AAInfo);
4263
4264 unsigned ExcessBits =
4265 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4266 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
4267
4268 // Increment the pointer to the other half.
4269 unsigned IncrementSize = NVT.getSizeInBits()/8;
4270 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4271 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
4272 N->getPointerInfo().getWithOffset(IncrementSize),
4273 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo);
4274 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4275 }
4276
4277 // Big-endian - high bits are at low addresses. Favor aligned stores at
4278 // the cost of some bit-fiddling.
4279 GetExpandedInteger(N->getValue(), Lo, Hi);
4280
4281 EVT ExtVT = N->getMemoryVT();
4282 unsigned EBytes = ExtVT.getStoreSize();
4283 unsigned IncrementSize = NVT.getSizeInBits()/8;
4284 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4285 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
4286 ExtVT.getSizeInBits() - ExcessBits);
4287
4288 if (ExcessBits < NVT.getSizeInBits()) {
4289 // Transfer high bits from the top of Lo to the bottom of Hi.
4290 Hi = DAG.getNode(
4291 ISD::SHL, dl, NVT, Hi,
4292 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
4293 TLI.getPointerRangeTy(DAG.getDataLayout())));
4294 Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
4295 DAG.getNode(ISD::SRL, dl, NVT, Lo,
4296 DAG.getConstant(ExcessBits, dl,
4297 TLI.getPointerRangeTy(
4298 DAG.getDataLayout()))));
4299 }
4300
4301 // Store both the high bits and maybe some of the low bits.
4302 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
4303 N->getOriginalAlign(), MMOFlags, AAInfo);
4304
4305 // Increment the pointer to the other half.
4306 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4307 // Store the lowest ExcessBits bits in the second half.
4308 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
4309 N->getPointerInfo().getWithOffset(IncrementSize),
4310 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
4311 N->getOriginalAlign(), MMOFlags, AAInfo);
4312 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4313 }
4314
ExpandIntOp_TRUNCATE(SDNode * N)4315 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
4316 SDValue InL, InH;
4317 GetExpandedInteger(N->getOperand(0), InL, InH);
4318 // Just truncate the low part of the source.
4319 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
4320 }
4321
ExpandIntOp_UINT_TO_FP(SDNode * N)4322 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
4323 bool IsStrict = N->isStrictFPOpcode();
4324 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4325 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4326 EVT DstVT = N->getValueType(0);
4327 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
4328 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4329 "Don't know how to expand this UINT_TO_FP!");
4330 TargetLowering::MakeLibCallOptions CallOptions;
4331 CallOptions.setSExt(true);
4332 std::pair<SDValue, SDValue> Tmp =
4333 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4334
4335 if (!IsStrict)
4336 return Tmp.first;
4337
4338 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4339 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4340 return SDValue();
4341 }
4342
ExpandIntOp_ATOMIC_STORE(SDNode * N)4343 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
4344 SDLoc dl(N);
4345 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4346 cast<AtomicSDNode>(N)->getMemoryVT(),
4347 N->getOperand(0),
4348 N->getOperand(1), N->getOperand(2),
4349 cast<AtomicSDNode>(N)->getMemOperand());
4350 return Swap.getValue(1);
4351 }
4352
4353
PromoteIntRes_EXTRACT_SUBVECTOR(SDNode * N)4354 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4355
4356 EVT OutVT = N->getValueType(0);
4357 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4358 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4359 EVT NOutVTElem = NOutVT.getVectorElementType();
4360
4361 SDLoc dl(N);
4362 SDValue BaseIdx = N->getOperand(1);
4363
4364 // TODO: We may be able to use this for types other than scalable
4365 // vectors and fix those tests that expect BUILD_VECTOR to be used
4366 if (OutVT.isScalableVector()) {
4367 SDValue InOp0 = N->getOperand(0);
4368 EVT InVT = InOp0.getValueType();
4369
4370 // Promote operands and see if this is handled by target lowering,
4371 // Otherwise, use the BUILD_VECTOR approach below
4372 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
4373 // Collect the (promoted) operands
4374 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
4375
4376 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
4377 assert(PromEltVT.bitsLE(NOutVTElem) &&
4378 "Promoted operand has an element type greater than result");
4379
4380 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
4381 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
4382 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
4383 }
4384 }
4385
4386 if (OutVT.isScalableVector())
4387 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
4388
4389 SDValue InOp0 = N->getOperand(0);
4390 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
4391 InOp0 = GetPromotedInteger(N->getOperand(0));
4392
4393 EVT InVT = InOp0.getValueType();
4394
4395 unsigned OutNumElems = OutVT.getVectorNumElements();
4396 SmallVector<SDValue, 8> Ops;
4397 Ops.reserve(OutNumElems);
4398 for (unsigned i = 0; i != OutNumElems; ++i) {
4399
4400 // Extract the element from the original vector.
4401 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4402 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4403 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4404 InVT.getVectorElementType(), N->getOperand(0), Index);
4405
4406 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4407 // Insert the converted element to the new vector.
4408 Ops.push_back(Op);
4409 }
4410
4411 return DAG.getBuildVector(NOutVT, dl, Ops);
4412 }
4413
4414
PromoteIntRes_VECTOR_SHUFFLE(SDNode * N)4415 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4416 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4417 EVT VT = N->getValueType(0);
4418 SDLoc dl(N);
4419
4420 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4421
4422 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4423 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4424 EVT OutVT = V0.getValueType();
4425
4426 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4427 }
4428
4429
PromoteIntRes_BUILD_VECTOR(SDNode * N)4430 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4431 EVT OutVT = N->getValueType(0);
4432 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4433 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4434 unsigned NumElems = N->getNumOperands();
4435 EVT NOutVTElem = NOutVT.getVectorElementType();
4436
4437 SDLoc dl(N);
4438
4439 SmallVector<SDValue, 8> Ops;
4440 Ops.reserve(NumElems);
4441 for (unsigned i = 0; i != NumElems; ++i) {
4442 SDValue Op;
4443 // BUILD_VECTOR integer operand types are allowed to be larger than the
4444 // result's element type. This may still be true after the promotion. For
4445 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4446 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4447 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4448 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4449 else
4450 Op = N->getOperand(i);
4451 Ops.push_back(Op);
4452 }
4453
4454 return DAG.getBuildVector(NOutVT, dl, Ops);
4455 }
4456
PromoteIntRes_SCALAR_TO_VECTOR(SDNode * N)4457 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4458
4459 SDLoc dl(N);
4460
4461 assert(!N->getOperand(0).getValueType().isVector() &&
4462 "Input must be a scalar");
4463
4464 EVT OutVT = N->getValueType(0);
4465 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4466 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4467 EVT NOutVTElem = NOutVT.getVectorElementType();
4468
4469 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4470
4471 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4472 }
4473
PromoteIntRes_SPLAT_VECTOR(SDNode * N)4474 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
4475 SDLoc dl(N);
4476
4477 SDValue SplatVal = N->getOperand(0);
4478
4479 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar");
4480
4481 EVT OutVT = N->getValueType(0);
4482 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4483 assert(NOutVT.isVector() && "Type must be promoted to a vector type");
4484 EVT NOutElemVT = NOutVT.getVectorElementType();
4485
4486 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
4487
4488 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
4489 }
4490
PromoteIntRes_CONCAT_VECTORS(SDNode * N)4491 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4492 SDLoc dl(N);
4493
4494 EVT OutVT = N->getValueType(0);
4495 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4496 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4497
4498 EVT OutElemTy = NOutVT.getVectorElementType();
4499
4500 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4501 unsigned NumOutElem = NOutVT.getVectorNumElements();
4502 unsigned NumOperands = N->getNumOperands();
4503 assert(NumElem * NumOperands == NumOutElem &&
4504 "Unexpected number of elements");
4505
4506 // Take the elements from the first vector.
4507 SmallVector<SDValue, 8> Ops(NumOutElem);
4508 for (unsigned i = 0; i < NumOperands; ++i) {
4509 SDValue Op = N->getOperand(i);
4510 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4511 Op = GetPromotedInteger(Op);
4512 EVT SclrTy = Op.getValueType().getVectorElementType();
4513 assert(NumElem == Op.getValueType().getVectorNumElements() &&
4514 "Unexpected number of elements");
4515
4516 for (unsigned j = 0; j < NumElem; ++j) {
4517 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4518 DAG.getVectorIdxConstant(j, dl));
4519 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4520 }
4521 }
4522
4523 return DAG.getBuildVector(NOutVT, dl, Ops);
4524 }
4525
PromoteIntRes_EXTEND_VECTOR_INREG(SDNode * N)4526 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4527 EVT VT = N->getValueType(0);
4528 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4529 assert(NVT.isVector() && "This type must be promoted to a vector type");
4530
4531 SDLoc dl(N);
4532
4533 // For operands whose TypeAction is to promote, extend the promoted node
4534 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4535 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4536 // type..
4537 if (getTypeAction(N->getOperand(0).getValueType())
4538 == TargetLowering::TypePromoteInteger) {
4539 SDValue Promoted;
4540
4541 switch(N->getOpcode()) {
4542 case ISD::SIGN_EXTEND_VECTOR_INREG:
4543 Promoted = SExtPromotedInteger(N->getOperand(0));
4544 break;
4545 case ISD::ZERO_EXTEND_VECTOR_INREG:
4546 Promoted = ZExtPromotedInteger(N->getOperand(0));
4547 break;
4548 case ISD::ANY_EXTEND_VECTOR_INREG:
4549 Promoted = GetPromotedInteger(N->getOperand(0));
4550 break;
4551 default:
4552 llvm_unreachable("Node has unexpected Opcode");
4553 }
4554 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4555 }
4556
4557 // Directly extend to the appropriate transform-to type.
4558 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4559 }
4560
PromoteIntRes_INSERT_VECTOR_ELT(SDNode * N)4561 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4562 EVT OutVT = N->getValueType(0);
4563 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4564 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4565
4566 EVT NOutVTElem = NOutVT.getVectorElementType();
4567
4568 SDLoc dl(N);
4569 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4570
4571 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4572 NOutVTElem, N->getOperand(1));
4573 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4574 V0, ConvElem, N->getOperand(2));
4575 }
4576
PromoteIntRes_VECREDUCE(SDNode * N)4577 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4578 // The VECREDUCE result size may be larger than the element size, so
4579 // we can simply change the result type.
4580 SDLoc dl(N);
4581 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4582 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4583 }
4584
PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode * N)4585 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4586 SDLoc dl(N);
4587 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4588 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4589 TLI.getVectorIdxTy(DAG.getDataLayout()));
4590 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4591 V0->getValueType(0).getScalarType(), V0, V1);
4592
4593 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4594 // element types. If this is the case then we need to expand the outgoing
4595 // value and not truncate it.
4596 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4597 }
4598
PromoteIntOp_EXTRACT_SUBVECTOR(SDNode * N)4599 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4600 SDLoc dl(N);
4601 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4602 MVT InVT = V0.getValueType().getSimpleVT();
4603 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4604 N->getValueType(0).getVectorNumElements());
4605 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4606 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4607 }
4608
PromoteIntOp_CONCAT_VECTORS(SDNode * N)4609 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4610 SDLoc dl(N);
4611 unsigned NumElems = N->getNumOperands();
4612
4613 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4614
4615 SmallVector<SDValue, 8> NewOps;
4616 NewOps.reserve(NumElems);
4617
4618 // For each incoming vector
4619 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
4620 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4621 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4622 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4623
4624 for (unsigned i=0; i<NumElem; ++i) {
4625 // Extract element from incoming vector
4626 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4627 DAG.getVectorIdxConstant(i, dl));
4628 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4629 NewOps.push_back(Tr);
4630 }
4631 }
4632
4633 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4634 }
4635