1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19
20 #include "LegalizeTypes.h"
21 #include "llvm/Analysis/TargetLibraryInfo.h"
22 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/KnownBits.h"
25 #include "llvm/Support/raw_ostream.h"
26 #include <algorithm>
27 using namespace llvm;
28
29 #define DEBUG_TYPE "legalize-types"
30
31 //===----------------------------------------------------------------------===//
32 // Integer Result Promotion
33 //===----------------------------------------------------------------------===//
34
35 /// PromoteIntegerResult - This method is called when a result of a node is
36 /// found to be in need of promotion to a larger type. At this point, the node
37 /// may also have invalid operands or may have other results that need
38 /// expansion, we just know that (at least) one result needs promotion.
PromoteIntegerResult(SDNode * N,unsigned ResNo)39 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
40 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
41 dbgs() << "\n");
42 SDValue Res = SDValue();
43
44 // See if the target wants to custom expand this node.
45 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
46 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
47 return;
48 }
49
50 switch (N->getOpcode()) {
51 default:
52 #ifndef NDEBUG
53 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
54 N->dump(&DAG); dbgs() << "\n";
55 #endif
56 llvm_unreachable("Do not know how to promote this operator!");
57 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
58 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
59 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
60 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
61 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
62 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
63 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
64 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
65 case ISD::CTLZ_ZERO_UNDEF:
66 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
67 case ISD::PARITY:
68 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
69 case ISD::CTTZ_ZERO_UNDEF:
70 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
71 case ISD::EXTRACT_VECTOR_ELT:
72 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
73 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
74 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
75 break;
76 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
77 break;
78 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
79 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
80 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
81 case ISD::STRICT_FSETCC:
82 case ISD::STRICT_FSETCCS:
83 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
84 case ISD::SMIN:
85 case ISD::SMAX:
86 Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ false);
87 break;
88 case ISD::UMIN:
89 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
90
91 case ISD::SHL:
92 Res = PromoteIntRes_SHL(N, /*IsVP*/ false);
93 break;
94 case ISD::SIGN_EXTEND_INREG:
95 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
96 case ISD::SRA:
97 Res = PromoteIntRes_SRA(N, /*IsVP*/ false);
98 break;
99 case ISD::SRL:
100 Res = PromoteIntRes_SRL(N, /*IsVP*/ false);
101 break;
102 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
103 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
104 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
105 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
106
107 case ISD::EXTRACT_SUBVECTOR:
108 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
109 case ISD::INSERT_SUBVECTOR:
110 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
111 case ISD::VECTOR_REVERSE:
112 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
113 case ISD::VECTOR_SHUFFLE:
114 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
115 case ISD::VECTOR_SPLICE:
116 Res = PromoteIntRes_VECTOR_SPLICE(N); break;
117 case ISD::INSERT_VECTOR_ELT:
118 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
119 case ISD::BUILD_VECTOR:
120 Res = PromoteIntRes_BUILD_VECTOR(N); break;
121 case ISD::SCALAR_TO_VECTOR:
122 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
123 case ISD::SPLAT_VECTOR:
124 Res = PromoteIntRes_SPLAT_VECTOR(N); break;
125 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
126 case ISD::CONCAT_VECTORS:
127 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
128
129 case ISD::ANY_EXTEND_VECTOR_INREG:
130 case ISD::SIGN_EXTEND_VECTOR_INREG:
131 case ISD::ZERO_EXTEND_VECTOR_INREG:
132 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
133
134 case ISD::SIGN_EXTEND:
135 case ISD::ZERO_EXTEND:
136 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
137
138 case ISD::STRICT_FP_TO_SINT:
139 case ISD::STRICT_FP_TO_UINT:
140 case ISD::FP_TO_SINT:
141 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
142
143 case ISD::FP_TO_SINT_SAT:
144 case ISD::FP_TO_UINT_SAT:
145 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
146
147 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
148
149 case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
150
151 case ISD::AND:
152 case ISD::OR:
153 case ISD::XOR:
154 case ISD::ADD:
155 case ISD::SUB:
156 case ISD::MUL:
157 Res = PromoteIntRes_SimpleIntBinOp(N, /*IsVP*/ false);
158 break;
159
160 case ISD::SDIV:
161 case ISD::SREM:
162 Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ false);
163 break;
164
165 case ISD::UDIV:
166 case ISD::UREM:
167 Res = PromoteIntRes_ZExtIntBinOp(N, /*IsVP*/ false);
168 break;
169
170 case ISD::SADDO:
171 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
172 case ISD::UADDO:
173 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
174 case ISD::SMULO:
175 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
176
177 case ISD::ADDE:
178 case ISD::SUBE:
179 case ISD::ADDCARRY:
180 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
181
182 case ISD::SADDO_CARRY:
183 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
184
185 case ISD::SADDSAT:
186 case ISD::UADDSAT:
187 case ISD::SSUBSAT:
188 case ISD::USUBSAT:
189 case ISD::SSHLSAT:
190 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break;
191
192 case ISD::SMULFIX:
193 case ISD::SMULFIXSAT:
194 case ISD::UMULFIX:
195 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
196
197 case ISD::SDIVFIX:
198 case ISD::SDIVFIXSAT:
199 case ISD::UDIVFIX:
200 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
201
202 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
203
204 case ISD::ATOMIC_LOAD:
205 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
206
207 case ISD::ATOMIC_LOAD_ADD:
208 case ISD::ATOMIC_LOAD_SUB:
209 case ISD::ATOMIC_LOAD_AND:
210 case ISD::ATOMIC_LOAD_CLR:
211 case ISD::ATOMIC_LOAD_OR:
212 case ISD::ATOMIC_LOAD_XOR:
213 case ISD::ATOMIC_LOAD_NAND:
214 case ISD::ATOMIC_LOAD_MIN:
215 case ISD::ATOMIC_LOAD_MAX:
216 case ISD::ATOMIC_LOAD_UMIN:
217 case ISD::ATOMIC_LOAD_UMAX:
218 case ISD::ATOMIC_SWAP:
219 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
220
221 case ISD::ATOMIC_CMP_SWAP:
222 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
223 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
224 break;
225
226 case ISD::VECREDUCE_ADD:
227 case ISD::VECREDUCE_MUL:
228 case ISD::VECREDUCE_AND:
229 case ISD::VECREDUCE_OR:
230 case ISD::VECREDUCE_XOR:
231 case ISD::VECREDUCE_SMAX:
232 case ISD::VECREDUCE_SMIN:
233 case ISD::VECREDUCE_UMAX:
234 case ISD::VECREDUCE_UMIN:
235 Res = PromoteIntRes_VECREDUCE(N);
236 break;
237
238 case ISD::VP_REDUCE_ADD:
239 case ISD::VP_REDUCE_MUL:
240 case ISD::VP_REDUCE_AND:
241 case ISD::VP_REDUCE_OR:
242 case ISD::VP_REDUCE_XOR:
243 case ISD::VP_REDUCE_SMAX:
244 case ISD::VP_REDUCE_SMIN:
245 case ISD::VP_REDUCE_UMAX:
246 case ISD::VP_REDUCE_UMIN:
247 Res = PromoteIntRes_VP_REDUCE(N);
248 break;
249
250 case ISD::FREEZE:
251 Res = PromoteIntRes_FREEZE(N);
252 break;
253
254 case ISD::ROTL:
255 case ISD::ROTR:
256 Res = PromoteIntRes_Rotate(N);
257 break;
258
259 case ISD::FSHL:
260 case ISD::FSHR:
261 Res = PromoteIntRes_FunnelShift(N);
262 break;
263
264 case ISD::VP_AND:
265 case ISD::VP_OR:
266 case ISD::VP_XOR:
267 case ISD::VP_ADD:
268 case ISD::VP_SUB:
269 case ISD::VP_MUL:
270 Res = PromoteIntRes_SimpleIntBinOp(N, /*IsVP*/ true);
271 break;
272 case ISD::VP_SDIV:
273 case ISD::VP_SREM:
274 Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ true);
275 break;
276 case ISD::VP_UDIV:
277 case ISD::VP_UREM:
278 Res = PromoteIntRes_ZExtIntBinOp(N, /*IsVP*/ true);
279 break;
280 case ISD::VP_SHL:
281 Res = PromoteIntRes_SHL(N, /*IsVP*/ true);
282 break;
283 case ISD::VP_ASHR:
284 Res = PromoteIntRes_SRA(N, /*IsVP*/ true);
285 break;
286 case ISD::VP_LSHR:
287 Res = PromoteIntRes_SRL(N, /*IsVP*/ true);
288 break;
289 }
290
291 // If the result is null then the sub-method took care of registering it.
292 if (Res.getNode())
293 SetPromotedInteger(SDValue(N, ResNo), Res);
294 }
295
PromoteIntRes_MERGE_VALUES(SDNode * N,unsigned ResNo)296 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
297 unsigned ResNo) {
298 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
299 return GetPromotedInteger(Op);
300 }
301
PromoteIntRes_AssertSext(SDNode * N)302 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
303 // Sign-extend the new bits, and continue the assertion.
304 SDValue Op = SExtPromotedInteger(N->getOperand(0));
305 return DAG.getNode(ISD::AssertSext, SDLoc(N),
306 Op.getValueType(), Op, N->getOperand(1));
307 }
308
PromoteIntRes_AssertZext(SDNode * N)309 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
310 // Zero the new bits, and continue the assertion.
311 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
312 return DAG.getNode(ISD::AssertZext, SDLoc(N),
313 Op.getValueType(), Op, N->getOperand(1));
314 }
315
PromoteIntRes_Atomic0(AtomicSDNode * N)316 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
317 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
318 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
319 N->getMemoryVT(), ResVT,
320 N->getChain(), N->getBasePtr(),
321 N->getMemOperand());
322 // Legalize the chain result - switch anything that used the old chain to
323 // use the new one.
324 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
325 return Res;
326 }
327
PromoteIntRes_Atomic1(AtomicSDNode * N)328 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
329 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
330 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
331 N->getMemoryVT(),
332 N->getChain(), N->getBasePtr(),
333 Op2, N->getMemOperand());
334 // Legalize the chain result - switch anything that used the old chain to
335 // use the new one.
336 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
337 return Res;
338 }
339
PromoteIntRes_AtomicCmpSwap(AtomicSDNode * N,unsigned ResNo)340 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
341 unsigned ResNo) {
342 if (ResNo == 1) {
343 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
344 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
345 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
346
347 // Only use the result of getSetCCResultType if it is legal,
348 // otherwise just use the promoted result type (NVT).
349 if (!TLI.isTypeLegal(SVT))
350 SVT = NVT;
351
352 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
353 SDValue Res = DAG.getAtomicCmpSwap(
354 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
355 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
356 N->getMemOperand());
357 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
358 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
359 return Res.getValue(1);
360 }
361
362 // Op2 is used for the comparison and thus must be extended according to the
363 // target's atomic operations. Op3 is merely stored and so can be left alone.
364 SDValue Op2 = N->getOperand(2);
365 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
366 switch (TLI.getExtendForAtomicCmpSwapArg()) {
367 case ISD::SIGN_EXTEND:
368 Op2 = SExtPromotedInteger(Op2);
369 break;
370 case ISD::ZERO_EXTEND:
371 Op2 = ZExtPromotedInteger(Op2);
372 break;
373 case ISD::ANY_EXTEND:
374 Op2 = GetPromotedInteger(Op2);
375 break;
376 default:
377 llvm_unreachable("Invalid atomic op extension");
378 }
379
380 SDVTList VTs =
381 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
382 SDValue Res = DAG.getAtomicCmpSwap(
383 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
384 N->getBasePtr(), Op2, Op3, N->getMemOperand());
385 // Update the use to N with the newly created Res.
386 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
387 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
388 return Res;
389 }
390
PromoteIntRes_BITCAST(SDNode * N)391 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
392 SDValue InOp = N->getOperand(0);
393 EVT InVT = InOp.getValueType();
394 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
395 EVT OutVT = N->getValueType(0);
396 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
397 SDLoc dl(N);
398
399 switch (getTypeAction(InVT)) {
400 case TargetLowering::TypeLegal:
401 break;
402 case TargetLowering::TypePromoteInteger:
403 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
404 // The input promotes to the same size. Convert the promoted value.
405 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
406 break;
407 case TargetLowering::TypeSoftenFloat:
408 // Promote the integer operand by hand.
409 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
410 case TargetLowering::TypeSoftPromoteHalf:
411 // Promote the integer operand by hand.
412 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
413 case TargetLowering::TypePromoteFloat: {
414 // Convert the promoted float by hand.
415 if (!NOutVT.isVector())
416 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
417 break;
418 }
419 case TargetLowering::TypeExpandInteger:
420 case TargetLowering::TypeExpandFloat:
421 break;
422 case TargetLowering::TypeScalarizeVector:
423 // Convert the element to an integer and promote it by hand.
424 if (!NOutVT.isVector())
425 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
426 BitConvertToInteger(GetScalarizedVector(InOp)));
427 break;
428 case TargetLowering::TypeScalarizeScalableVector:
429 report_fatal_error("Scalarization of scalable vectors is not supported.");
430 case TargetLowering::TypeSplitVector: {
431 if (!NOutVT.isVector()) {
432 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
433 // pieces of the input into integers and reassemble in the final type.
434 SDValue Lo, Hi;
435 GetSplitVector(N->getOperand(0), Lo, Hi);
436 Lo = BitConvertToInteger(Lo);
437 Hi = BitConvertToInteger(Hi);
438
439 if (DAG.getDataLayout().isBigEndian())
440 std::swap(Lo, Hi);
441
442 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
443 EVT::getIntegerVT(*DAG.getContext(),
444 NOutVT.getSizeInBits()),
445 JoinIntegers(Lo, Hi));
446 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
447 }
448 break;
449 }
450 case TargetLowering::TypeWidenVector:
451 // The input is widened to the same size. Convert to the widened value.
452 // Make sure that the outgoing value is not a vector, because this would
453 // make us bitcast between two vectors which are legalized in different ways.
454 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
455 SDValue Res =
456 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
457
458 // For big endian targets we need to shift the casted value or the
459 // interesting bits will end up at the wrong place.
460 if (DAG.getDataLayout().isBigEndian()) {
461 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
462 EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout());
463 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
464 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
465 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
466 }
467 return Res;
468 }
469 // If the output type is also a vector and widening it to the same size
470 // as the widened input type would be a legal type, we can widen the bitcast
471 // and handle the promotion after.
472 if (NOutVT.isVector()) {
473 unsigned WidenInSize = NInVT.getSizeInBits();
474 unsigned OutSize = OutVT.getSizeInBits();
475 if (WidenInSize % OutSize == 0) {
476 unsigned Scale = WidenInSize / OutSize;
477 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
478 OutVT.getVectorElementType(),
479 OutVT.getVectorNumElements() * Scale);
480 if (isTypeLegal(WideOutVT)) {
481 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
482 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
483 DAG.getVectorIdxConstant(0, dl));
484 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
485 }
486 }
487 }
488 }
489
490 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
491 CreateStackStoreLoad(InOp, OutVT));
492 }
493
494 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount
495 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
getShiftAmountTyForConstant(EVT VT,const TargetLowering & TLI,SelectionDAG & DAG)496 static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI,
497 SelectionDAG &DAG) {
498 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
499 // If any possible shift value won't fit in the prefered type, just use
500 // something safe. It will be legalized when the shift is expanded.
501 if (!ShiftVT.isVector() &&
502 ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits()))
503 ShiftVT = MVT::i32;
504 return ShiftVT;
505 }
506
PromoteIntRes_FREEZE(SDNode * N)507 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
508 SDValue V = GetPromotedInteger(N->getOperand(0));
509 return DAG.getNode(ISD::FREEZE, SDLoc(N),
510 V.getValueType(), V);
511 }
512
PromoteIntRes_BSWAP(SDNode * N)513 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
514 SDValue Op = GetPromotedInteger(N->getOperand(0));
515 EVT OVT = N->getValueType(0);
516 EVT NVT = Op.getValueType();
517 SDLoc dl(N);
518
519 // If the larger BSWAP isn't supported by the target, try to expand now.
520 // If we expand later we'll end up with more operations since we lost the
521 // original type. We only do this for scalars since we have a shuffle
522 // based lowering for vectors in LegalizeVectorOps.
523 if (!OVT.isVector() &&
524 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) {
525 if (SDValue Res = TLI.expandBSWAP(N, DAG))
526 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
527 }
528
529 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
530 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
531 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
532 DAG.getConstant(DiffBits, dl, ShiftVT));
533 }
534
PromoteIntRes_BITREVERSE(SDNode * N)535 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
536 SDValue Op = GetPromotedInteger(N->getOperand(0));
537 EVT OVT = N->getValueType(0);
538 EVT NVT = Op.getValueType();
539 SDLoc dl(N);
540
541 // If the larger BITREVERSE isn't supported by the target, try to expand now.
542 // If we expand later we'll end up with more operations since we lost the
543 // original type. We only do this for scalars since we have a shuffle
544 // based lowering for vectors in LegalizeVectorOps.
545 if (!OVT.isVector() && OVT.isSimple() &&
546 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) {
547 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
548 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
549 }
550
551 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
552 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
553 return DAG.getNode(ISD::SRL, dl, NVT,
554 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
555 DAG.getConstant(DiffBits, dl, ShiftVT));
556 }
557
PromoteIntRes_BUILD_PAIR(SDNode * N)558 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
559 // The pair element type may be legal, or may not promote to the same type as
560 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
561 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
562 TLI.getTypeToTransformTo(*DAG.getContext(),
563 N->getValueType(0)), JoinIntegers(N->getOperand(0),
564 N->getOperand(1)));
565 }
566
PromoteIntRes_Constant(SDNode * N)567 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
568 EVT VT = N->getValueType(0);
569 // FIXME there is no actual debug info here
570 SDLoc dl(N);
571 // Zero extend things like i1, sign extend everything else. It shouldn't
572 // matter in theory which one we pick, but this tends to give better code?
573 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
574 SDValue Result = DAG.getNode(Opc, dl,
575 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
576 SDValue(N, 0));
577 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
578 return Result;
579 }
580
PromoteIntRes_CTLZ(SDNode * N)581 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
582 // Zero extend to the promoted type and do the count there.
583 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
584 SDLoc dl(N);
585 EVT OVT = N->getValueType(0);
586 EVT NVT = Op.getValueType();
587 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
588 // Subtract off the extra leading bits in the bigger type.
589 return DAG.getNode(
590 ISD::SUB, dl, NVT, Op,
591 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
592 NVT));
593 }
594
PromoteIntRes_CTPOP_PARITY(SDNode * N)595 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
596 // Zero extend to the promoted type and do the count or parity there.
597 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
598 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
599 }
600
PromoteIntRes_CTTZ(SDNode * N)601 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
602 SDValue Op = GetPromotedInteger(N->getOperand(0));
603 EVT OVT = N->getValueType(0);
604 EVT NVT = Op.getValueType();
605 SDLoc dl(N);
606 if (N->getOpcode() == ISD::CTTZ) {
607 // The count is the same in the promoted type except if the original
608 // value was zero. This can be handled by setting the bit just off
609 // the top of the original type.
610 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
611 OVT.getScalarSizeInBits());
612 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
613 }
614 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
615 }
616
PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode * N)617 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
618 SDLoc dl(N);
619 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
620
621 SDValue Op0 = N->getOperand(0);
622 SDValue Op1 = N->getOperand(1);
623
624 // If the input also needs to be promoted, do that first so we can get a
625 // get a good idea for the output type.
626 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
627 == TargetLowering::TypePromoteInteger) {
628 SDValue In = GetPromotedInteger(Op0);
629
630 // If the new type is larger than NVT, use it. We probably won't need to
631 // promote it again.
632 EVT SVT = In.getValueType().getScalarType();
633 if (SVT.bitsGE(NVT)) {
634 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
635 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
636 }
637 }
638
639 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
640 }
641
PromoteIntRes_FP_TO_XINT(SDNode * N)642 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
643 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
644 unsigned NewOpc = N->getOpcode();
645 SDLoc dl(N);
646
647 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
648 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
649 // and SINT conversions are Custom, there is no way to tell which is
650 // preferable. We choose SINT because that's the right thing on PPC.)
651 if (N->getOpcode() == ISD::FP_TO_UINT &&
652 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
653 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
654 NewOpc = ISD::FP_TO_SINT;
655
656 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
657 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
658 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
659 NewOpc = ISD::STRICT_FP_TO_SINT;
660
661 SDValue Res;
662 if (N->isStrictFPOpcode()) {
663 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
664 {N->getOperand(0), N->getOperand(1)});
665 // Legalize the chain result - switch anything that used the old chain to
666 // use the new one.
667 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
668 } else
669 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
670
671 // Assert that the converted value fits in the original type. If it doesn't
672 // (eg: because the value being converted is too big), then the result of the
673 // original operation was undefined anyway, so the assert is still correct.
674 //
675 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
676 // before legalization: fp-to-uint16, 65534. -> 0xfffe
677 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
678 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
679 N->getOpcode() == ISD::STRICT_FP_TO_UINT) ?
680 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
681 DAG.getValueType(N->getValueType(0).getScalarType()));
682 }
683
PromoteIntRes_FP_TO_XINT_SAT(SDNode * N)684 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
685 // Promote the result type, while keeping the original width in Op1.
686 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
687 SDLoc dl(N);
688 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
689 N->getOperand(1));
690 }
691
PromoteIntRes_FP_TO_FP16(SDNode * N)692 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
693 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
694 SDLoc dl(N);
695
696 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
697 }
698
PromoteIntRes_FLT_ROUNDS(SDNode * N)699 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
700 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
701 SDLoc dl(N);
702
703 SDValue Res =
704 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
705
706 // Legalize the chain result - switch anything that used the old chain to
707 // use the new one.
708 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
709 return Res;
710 }
711
PromoteIntRes_INT_EXTEND(SDNode * N)712 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
713 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
714 SDLoc dl(N);
715
716 if (getTypeAction(N->getOperand(0).getValueType())
717 == TargetLowering::TypePromoteInteger) {
718 SDValue Res = GetPromotedInteger(N->getOperand(0));
719 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
720
721 // If the result and operand types are the same after promotion, simplify
722 // to an in-register extension.
723 if (NVT == Res.getValueType()) {
724 // The high bits are not guaranteed to be anything. Insert an extend.
725 if (N->getOpcode() == ISD::SIGN_EXTEND)
726 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
727 DAG.getValueType(N->getOperand(0).getValueType()));
728 if (N->getOpcode() == ISD::ZERO_EXTEND)
729 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
730 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
731 return Res;
732 }
733 }
734
735 // Otherwise, just extend the original operand all the way to the larger type.
736 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
737 }
738
PromoteIntRes_LOAD(LoadSDNode * N)739 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
740 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
741 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
742 ISD::LoadExtType ExtType =
743 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
744 SDLoc dl(N);
745 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
746 N->getMemoryVT(), N->getMemOperand());
747
748 // Legalize the chain result - switch anything that used the old chain to
749 // use the new one.
750 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
751 return Res;
752 }
753
PromoteIntRes_MLOAD(MaskedLoadSDNode * N)754 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
755 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
756 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
757
758 SDLoc dl(N);
759 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
760 N->getOffset(), N->getMask(), ExtPassThru,
761 N->getMemoryVT(), N->getMemOperand(),
762 N->getAddressingMode(), ISD::EXTLOAD);
763 // Legalize the chain result - switch anything that used the old chain to
764 // use the new one.
765 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
766 return Res;
767 }
768
PromoteIntRes_MGATHER(MaskedGatherSDNode * N)769 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
770 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
771 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
772 assert(NVT == ExtPassThru.getValueType() &&
773 "Gather result type and the passThru argument type should be the same");
774
775 ISD::LoadExtType ExtType = N->getExtensionType();
776 if (ExtType == ISD::NON_EXTLOAD)
777 ExtType = ISD::EXTLOAD;
778
779 SDLoc dl(N);
780 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
781 N->getIndex(), N->getScale() };
782 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
783 N->getMemoryVT(), dl, Ops,
784 N->getMemOperand(), N->getIndexType(),
785 ExtType);
786 // Legalize the chain result - switch anything that used the old chain to
787 // use the new one.
788 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
789 return Res;
790 }
791
792 /// Promote the overflow flag of an overflowing arithmetic node.
PromoteIntRes_Overflow(SDNode * N)793 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
794 // Change the return type of the boolean result while obeying
795 // getSetCCResultType.
796 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
797 EVT VT = N->getValueType(0);
798 EVT SVT = getSetCCResultType(VT);
799 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
800 unsigned NumOps = N->getNumOperands();
801 assert(NumOps <= 3 && "Too many operands");
802 if (NumOps == 3)
803 Ops[2] = N->getOperand(2);
804
805 SDLoc dl(N);
806 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
807 makeArrayRef(Ops, NumOps));
808
809 // Modified the sum result - switch anything that used the old sum to use
810 // the new one.
811 ReplaceValueWith(SDValue(N, 0), Res);
812
813 // Convert to the expected type.
814 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
815 }
816
PromoteIntRes_ADDSUBSHLSAT(SDNode * N)817 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
818 // If the promoted type is legal, we can convert this to:
819 // 1. ANY_EXTEND iN to iM
820 // 2. SHL by M-N
821 // 3. [US][ADD|SUB|SHL]SAT
822 // 4. L/ASHR by M-N
823 // Else it is more efficient to convert this to a min and a max
824 // operation in the higher precision arithmetic.
825 SDLoc dl(N);
826 SDValue Op1 = N->getOperand(0);
827 SDValue Op2 = N->getOperand(1);
828 unsigned OldBits = Op1.getScalarValueSizeInBits();
829
830 unsigned Opcode = N->getOpcode();
831 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
832
833 SDValue Op1Promoted, Op2Promoted;
834 if (IsShift) {
835 Op1Promoted = GetPromotedInteger(Op1);
836 Op2Promoted = ZExtPromotedInteger(Op2);
837 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
838 Op1Promoted = ZExtPromotedInteger(Op1);
839 Op2Promoted = ZExtPromotedInteger(Op2);
840 } else {
841 Op1Promoted = SExtPromotedInteger(Op1);
842 Op2Promoted = SExtPromotedInteger(Op2);
843 }
844 EVT PromotedType = Op1Promoted.getValueType();
845 unsigned NewBits = PromotedType.getScalarSizeInBits();
846
847 if (Opcode == ISD::UADDSAT) {
848 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
849 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
850 SDValue Add =
851 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
852 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
853 }
854
855 // USUBSAT can always be promoted as long as we have zero-extended the args.
856 if (Opcode == ISD::USUBSAT)
857 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
858 Op2Promoted);
859
860 // Shift cannot use a min/max expansion, we can't detect overflow if all of
861 // the bits have been shifted out.
862 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) {
863 unsigned ShiftOp;
864 switch (Opcode) {
865 case ISD::SADDSAT:
866 case ISD::SSUBSAT:
867 case ISD::SSHLSAT:
868 ShiftOp = ISD::SRA;
869 break;
870 case ISD::USHLSAT:
871 ShiftOp = ISD::SRL;
872 break;
873 default:
874 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
875 "addition, subtraction or left shift");
876 }
877
878 unsigned SHLAmount = NewBits - OldBits;
879 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
880 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
881 Op1Promoted =
882 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
883 if (!IsShift)
884 Op2Promoted =
885 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
886
887 SDValue Result =
888 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
889 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
890 }
891
892 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
893 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
894 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
895 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
896 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
897 SDValue Result =
898 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
899 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
900 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
901 return Result;
902 }
903
PromoteIntRes_MULFIX(SDNode * N)904 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
905 // Can just promote the operands then continue with operation.
906 SDLoc dl(N);
907 SDValue Op1Promoted, Op2Promoted;
908 bool Signed =
909 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
910 bool Saturating =
911 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
912 if (Signed) {
913 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
914 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
915 } else {
916 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
917 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
918 }
919 EVT OldType = N->getOperand(0).getValueType();
920 EVT PromotedType = Op1Promoted.getValueType();
921 unsigned DiffSize =
922 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
923
924 if (Saturating) {
925 // Promoting the operand and result values changes the saturation width,
926 // which is extends the values that we clamp to on saturation. This could be
927 // resolved by shifting one of the operands the same amount, which would
928 // also shift the result we compare against, then shifting back.
929 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
930 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
931 DAG.getConstant(DiffSize, dl, ShiftTy));
932 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
933 Op2Promoted, N->getOperand(2));
934 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
935 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
936 DAG.getConstant(DiffSize, dl, ShiftTy));
937 }
938 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
939 N->getOperand(2));
940 }
941
SaturateWidenedDIVFIX(SDValue V,SDLoc & dl,unsigned SatW,bool Signed,const TargetLowering & TLI,SelectionDAG & DAG)942 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
943 unsigned SatW, bool Signed,
944 const TargetLowering &TLI,
945 SelectionDAG &DAG) {
946 EVT VT = V.getValueType();
947 unsigned VTW = VT.getScalarSizeInBits();
948
949 if (!Signed) {
950 // Saturate to the unsigned maximum by getting the minimum of V and the
951 // maximum.
952 return DAG.getNode(ISD::UMIN, dl, VT, V,
953 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
954 dl, VT));
955 }
956
957 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
958 // signed minimum of it and V.
959 V = DAG.getNode(ISD::SMIN, dl, VT, V,
960 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
961 dl, VT));
962 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
963 // signed maximum of it and V.
964 V = DAG.getNode(ISD::SMAX, dl, VT, V,
965 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
966 dl, VT));
967 return V;
968 }
969
earlyExpandDIVFIX(SDNode * N,SDValue LHS,SDValue RHS,unsigned Scale,const TargetLowering & TLI,SelectionDAG & DAG,unsigned SatW=0)970 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
971 unsigned Scale, const TargetLowering &TLI,
972 SelectionDAG &DAG, unsigned SatW = 0) {
973 EVT VT = LHS.getValueType();
974 unsigned VTSize = VT.getScalarSizeInBits();
975 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
976 N->getOpcode() == ISD::SDIVFIXSAT;
977 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
978 N->getOpcode() == ISD::UDIVFIXSAT;
979
980 SDLoc dl(N);
981 // Widen the types by a factor of two. This is guaranteed to expand, since it
982 // will always have enough high bits in the LHS to shift into.
983 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
984 if (VT.isVector())
985 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
986 VT.getVectorElementCount());
987 if (Signed) {
988 LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
989 RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
990 } else {
991 LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
992 RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
993 }
994
995 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
996 DAG);
997 assert(Res && "Expanding DIVFIX with wide type failed?");
998 if (Saturating) {
999 // If the caller has told us to saturate at something less, use that width
1000 // instead of the type before doubling. However, it cannot be more than
1001 // what we just widened!
1002 assert(SatW <= VTSize &&
1003 "Tried to saturate to more than the original type?");
1004 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1005 TLI, DAG);
1006 }
1007 return DAG.getZExtOrTrunc(Res, dl, VT);
1008 }
1009
PromoteIntRes_DIVFIX(SDNode * N)1010 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1011 SDLoc dl(N);
1012 SDValue Op1Promoted, Op2Promoted;
1013 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1014 N->getOpcode() == ISD::SDIVFIXSAT;
1015 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1016 N->getOpcode() == ISD::UDIVFIXSAT;
1017 if (Signed) {
1018 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1019 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1020 } else {
1021 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1022 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1023 }
1024 EVT PromotedType = Op1Promoted.getValueType();
1025 unsigned Scale = N->getConstantOperandVal(2);
1026
1027 // If the type is already legal and the operation is legal in that type, we
1028 // should not early expand.
1029 if (TLI.isTypeLegal(PromotedType)) {
1030 TargetLowering::LegalizeAction Action =
1031 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1032 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1033 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
1034 unsigned Diff = PromotedType.getScalarSizeInBits() -
1035 N->getValueType(0).getScalarSizeInBits();
1036 if (Saturating)
1037 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1038 DAG.getConstant(Diff, dl, ShiftTy));
1039 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1040 Op2Promoted, N->getOperand(2));
1041 if (Saturating)
1042 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1043 DAG.getConstant(Diff, dl, ShiftTy));
1044 return Res;
1045 }
1046 }
1047
1048 // See if we can perform the division in this type without expanding.
1049 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1050 Op2Promoted, Scale, DAG)) {
1051 if (Saturating)
1052 Res = SaturateWidenedDIVFIX(Res, dl,
1053 N->getValueType(0).getScalarSizeInBits(),
1054 Signed, TLI, DAG);
1055 return Res;
1056 }
1057 // If we cannot, expand it to twice the type width. If we are saturating, give
1058 // it the original width as a saturating width so we don't need to emit
1059 // two saturations.
1060 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1061 N->getValueType(0).getScalarSizeInBits());
1062 }
1063
PromoteIntRes_SADDSUBO(SDNode * N,unsigned ResNo)1064 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1065 if (ResNo == 1)
1066 return PromoteIntRes_Overflow(N);
1067
1068 // The operation overflowed iff the result in the larger type is not the
1069 // sign extension of its truncation to the original type.
1070 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1071 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1072 EVT OVT = N->getOperand(0).getValueType();
1073 EVT NVT = LHS.getValueType();
1074 SDLoc dl(N);
1075
1076 // Do the arithmetic in the larger type.
1077 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1078 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1079
1080 // Calculate the overflow flag: sign extend the arithmetic result from
1081 // the original type.
1082 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1083 DAG.getValueType(OVT));
1084 // Overflowed if and only if this is not equal to Res.
1085 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1086
1087 // Use the calculated overflow everywhere.
1088 ReplaceValueWith(SDValue(N, 1), Ofl);
1089
1090 return Res;
1091 }
1092
PromoteIntRes_SELECT(SDNode * N)1093 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
1094 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1095 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1096 return DAG.getSelect(SDLoc(N),
1097 LHS.getValueType(), N->getOperand(0), LHS, RHS);
1098 }
1099
PromoteIntRes_VSELECT(SDNode * N)1100 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
1101 SDValue Mask = N->getOperand(0);
1102
1103 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1104 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1105 return DAG.getNode(ISD::VSELECT, SDLoc(N),
1106 LHS.getValueType(), Mask, LHS, RHS);
1107 }
1108
PromoteIntRes_SELECT_CC(SDNode * N)1109 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1110 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1111 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1112 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1113 LHS.getValueType(), N->getOperand(0),
1114 N->getOperand(1), LHS, RHS, N->getOperand(4));
1115 }
1116
PromoteIntRes_SETCC(SDNode * N)1117 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1118 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1119 EVT InVT = N->getOperand(OpNo).getValueType();
1120 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1121
1122 EVT SVT = getSetCCResultType(InVT);
1123
1124 // If we got back a type that needs to be promoted, this likely means the
1125 // the input type also needs to be promoted. So get the promoted type for
1126 // the input and try the query again.
1127 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1128 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1129 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1130 SVT = getSetCCResultType(InVT);
1131 } else {
1132 // Input type isn't promoted, just use the default promoted type.
1133 SVT = NVT;
1134 }
1135 }
1136
1137 SDLoc dl(N);
1138 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1139 "Vector compare must return a vector result!");
1140
1141 // Get the SETCC result using the canonical SETCC type.
1142 SDValue SetCC;
1143 if (N->isStrictFPOpcode()) {
1144 EVT VTs[] = {SVT, MVT::Other};
1145 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1146 N->getOperand(2), N->getOperand(3)};
1147 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
1148 // Legalize the chain result - switch anything that used the old chain to
1149 // use the new one.
1150 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1151 } else
1152 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1153 N->getOperand(1), N->getOperand(2));
1154
1155 // Convert to the expected type.
1156 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1157 }
1158
PromoteIntRes_SHL(SDNode * N,bool IsVP)1159 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N, bool IsVP) {
1160 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1161 SDValue RHS = N->getOperand(1);
1162 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1163 RHS = ZExtPromotedInteger(RHS);
1164 if (!IsVP)
1165 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1166 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1167 N->getOperand(2), N->getOperand(3));
1168 }
1169
PromoteIntRes_SIGN_EXTEND_INREG(SDNode * N)1170 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1171 SDValue Op = GetPromotedInteger(N->getOperand(0));
1172 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1173 Op.getValueType(), Op, N->getOperand(1));
1174 }
1175
PromoteIntRes_SimpleIntBinOp(SDNode * N,bool IsVP)1176 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N, bool IsVP) {
1177 // The input may have strange things in the top bits of the registers, but
1178 // these operations don't care. They may have weird bits going out, but
1179 // that too is okay if they are integer operations.
1180 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1181 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1182 if (!IsVP)
1183 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1184 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1185 N->getOperand(2), N->getOperand(3));
1186 }
1187
PromoteIntRes_SExtIntBinOp(SDNode * N,bool IsVP)1188 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N, bool IsVP) {
1189 // Sign extend the input.
1190 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1191 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1192 if (!IsVP)
1193 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1194 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1195 N->getOperand(2), N->getOperand(3));
1196 }
1197
PromoteIntRes_ZExtIntBinOp(SDNode * N,bool IsVP)1198 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N, bool IsVP) {
1199 // Zero extend the input.
1200 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1201 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1202 if (!IsVP)
1203 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1204 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1205 N->getOperand(2), N->getOperand(3));
1206 }
1207
PromoteIntRes_UMINUMAX(SDNode * N)1208 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1209 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1210 // whatever is best for the target.
1211 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0));
1212 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1));
1213 return DAG.getNode(N->getOpcode(), SDLoc(N),
1214 LHS.getValueType(), LHS, RHS);
1215 }
1216
PromoteIntRes_SRA(SDNode * N,bool IsVP)1217 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N, bool IsVP) {
1218 // The input value must be properly sign extended.
1219 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1220 SDValue RHS = N->getOperand(1);
1221 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1222 RHS = ZExtPromotedInteger(RHS);
1223 if (!IsVP)
1224 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1225 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1226 N->getOperand(2), N->getOperand(3));
1227 }
1228
PromoteIntRes_SRL(SDNode * N,bool IsVP)1229 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N, bool IsVP) {
1230 // The input value must be properly zero extended.
1231 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1232 SDValue RHS = N->getOperand(1);
1233 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1234 RHS = ZExtPromotedInteger(RHS);
1235 if (!IsVP)
1236 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1237 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1238 N->getOperand(2), N->getOperand(3));
1239 }
1240
PromoteIntRes_Rotate(SDNode * N)1241 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1242 // Lower the rotate to shifts and ORs which can be promoted.
1243 SDValue Res;
1244 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
1245 ReplaceValueWith(SDValue(N, 0), Res);
1246 return SDValue();
1247 }
1248
PromoteIntRes_FunnelShift(SDNode * N)1249 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1250 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1251 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1252 SDValue Amount = GetPromotedInteger(N->getOperand(2));
1253
1254 SDLoc DL(N);
1255 EVT OldVT = N->getOperand(0).getValueType();
1256 EVT VT = Lo.getValueType();
1257 unsigned Opcode = N->getOpcode();
1258 bool IsFSHR = Opcode == ISD::FSHR;
1259 unsigned OldBits = OldVT.getScalarSizeInBits();
1260 unsigned NewBits = VT.getScalarSizeInBits();
1261
1262 // Amount has to be interpreted modulo the old bit width.
1263 Amount =
1264 DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT));
1265
1266 // If the promoted type is twice the size (or more), then we use the
1267 // traditional funnel 'double' shift codegen. This isn't necessary if the
1268 // shift amount is constant.
1269 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1270 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1271 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) &&
1272 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1273 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1274 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1275 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1276 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1277 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount);
1278 if (!IsFSHR)
1279 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1280 return Res;
1281 }
1282
1283 // Shift Lo up to occupy the upper bits of the promoted type.
1284 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT);
1285 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1286
1287 // Increase Amount to shift the result into the lower bits of the promoted
1288 // type.
1289 if (IsFSHR)
1290 Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset);
1291
1292 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount);
1293 }
1294
PromoteIntRes_TRUNCATE(SDNode * N)1295 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1296 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1297 SDValue Res;
1298 SDValue InOp = N->getOperand(0);
1299 SDLoc dl(N);
1300
1301 switch (getTypeAction(InOp.getValueType())) {
1302 default: llvm_unreachable("Unknown type action!");
1303 case TargetLowering::TypeLegal:
1304 case TargetLowering::TypeExpandInteger:
1305 Res = InOp;
1306 break;
1307 case TargetLowering::TypePromoteInteger:
1308 Res = GetPromotedInteger(InOp);
1309 break;
1310 case TargetLowering::TypeSplitVector: {
1311 EVT InVT = InOp.getValueType();
1312 assert(InVT.isVector() && "Cannot split scalar types");
1313 ElementCount NumElts = InVT.getVectorElementCount();
1314 assert(NumElts == NVT.getVectorElementCount() &&
1315 "Dst and Src must have the same number of elements");
1316 assert(isPowerOf2_32(NumElts.getKnownMinValue()) &&
1317 "Promoted vector type must be a power of two");
1318
1319 SDValue EOp1, EOp2;
1320 GetSplitVector(InOp, EOp1, EOp2);
1321
1322 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1323 NumElts.divideCoefficientBy(2));
1324 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1325 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1326
1327 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1328 }
1329 case TargetLowering::TypeWidenVector: {
1330 SDValue WideInOp = GetWidenedVector(InOp);
1331
1332 // Truncate widened InOp.
1333 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1334 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1335 N->getValueType(0).getScalarType(), NumElem);
1336 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1337
1338 // Zero extend so that the elements are of same type as those of NVT
1339 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1340 NumElem);
1341 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1342
1343 // Extract the low NVT subvector.
1344 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1345 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1346 }
1347 }
1348
1349 // Truncate to NVT instead of VT
1350 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1351 }
1352
PromoteIntRes_UADDSUBO(SDNode * N,unsigned ResNo)1353 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1354 if (ResNo == 1)
1355 return PromoteIntRes_Overflow(N);
1356
1357 // The operation overflowed iff the result in the larger type is not the
1358 // zero extension of its truncation to the original type.
1359 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1360 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1361 EVT OVT = N->getOperand(0).getValueType();
1362 EVT NVT = LHS.getValueType();
1363 SDLoc dl(N);
1364
1365 // Do the arithmetic in the larger type.
1366 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1367 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1368
1369 // Calculate the overflow flag: zero extend the arithmetic result from
1370 // the original type.
1371 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1372 // Overflowed if and only if this is not equal to Res.
1373 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1374
1375 // Use the calculated overflow everywhere.
1376 ReplaceValueWith(SDValue(N, 1), Ofl);
1377
1378 return Res;
1379 }
1380
1381 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1382 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
1383 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
PromoteIntRes_ADDSUBCARRY(SDNode * N,unsigned ResNo)1384 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1385 if (ResNo == 1)
1386 return PromoteIntRes_Overflow(N);
1387
1388 // We need to sign-extend the operands so the carry value computed by the
1389 // wide operation will be equivalent to the carry value computed by the
1390 // narrow operation.
1391 // An ADDCARRY can generate carry only if any of the operands has its
1392 // most significant bit set. Sign extension propagates the most significant
1393 // bit into the higher bits which means the extra bit that the narrow
1394 // addition would need (i.e. the carry) will be propagated through the higher
1395 // bits of the wide addition.
1396 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1397 // preserved by sign extension.
1398 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1399 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1400
1401 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1402
1403 // Do the arithmetic in the wide type.
1404 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1405 LHS, RHS, N->getOperand(2));
1406
1407 // Update the users of the original carry/borrow value.
1408 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1409
1410 return SDValue(Res.getNode(), 0);
1411 }
1412
PromoteIntRes_SADDSUBO_CARRY(SDNode * N,unsigned ResNo)1413 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1414 unsigned ResNo) {
1415 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1416 return PromoteIntRes_Overflow(N);
1417 }
1418
PromoteIntRes_ABS(SDNode * N)1419 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1420 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1421 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1422 }
1423
PromoteIntRes_XMULO(SDNode * N,unsigned ResNo)1424 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1425 // Promote the overflow bit trivially.
1426 if (ResNo == 1)
1427 return PromoteIntRes_Overflow(N);
1428
1429 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1430 SDLoc DL(N);
1431 EVT SmallVT = LHS.getValueType();
1432
1433 // To determine if the result overflowed in a larger type, we extend the
1434 // input to the larger type, do the multiply (checking if it overflows),
1435 // then also check the high bits of the result to see if overflow happened
1436 // there.
1437 if (N->getOpcode() == ISD::SMULO) {
1438 LHS = SExtPromotedInteger(LHS);
1439 RHS = SExtPromotedInteger(RHS);
1440 } else {
1441 LHS = ZExtPromotedInteger(LHS);
1442 RHS = ZExtPromotedInteger(RHS);
1443 }
1444 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1445 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1446
1447 // Overflow occurred if it occurred in the larger type, or if the high part
1448 // of the result does not zero/sign-extend the low part. Check this second
1449 // possibility first.
1450 SDValue Overflow;
1451 if (N->getOpcode() == ISD::UMULO) {
1452 // Unsigned overflow occurred if the high part is non-zero.
1453 unsigned Shift = SmallVT.getScalarSizeInBits();
1454 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG);
1455 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1456 DAG.getConstant(Shift, DL, ShiftTy));
1457 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1458 DAG.getConstant(0, DL, Hi.getValueType()),
1459 ISD::SETNE);
1460 } else {
1461 // Signed overflow occurred if the high part does not sign extend the low.
1462 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1463 Mul, DAG.getValueType(SmallVT));
1464 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1465 }
1466
1467 // The only other way for overflow to occur is if the multiplication in the
1468 // larger type itself overflowed.
1469 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1470 SDValue(Mul.getNode(), 1));
1471
1472 // Use the calculated overflow everywhere.
1473 ReplaceValueWith(SDValue(N, 1), Overflow);
1474 return Mul;
1475 }
1476
PromoteIntRes_UNDEF(SDNode * N)1477 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1478 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1479 N->getValueType(0)));
1480 }
1481
PromoteIntRes_VSCALE(SDNode * N)1482 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1483 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1484
1485 APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
1486 return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1487 }
1488
PromoteIntRes_VAARG(SDNode * N)1489 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1490 SDValue Chain = N->getOperand(0); // Get the chain.
1491 SDValue Ptr = N->getOperand(1); // Get the pointer.
1492 EVT VT = N->getValueType(0);
1493 SDLoc dl(N);
1494
1495 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1496 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1497 // The argument is passed as NumRegs registers of type RegVT.
1498
1499 SmallVector<SDValue, 8> Parts(NumRegs);
1500 for (unsigned i = 0; i < NumRegs; ++i) {
1501 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1502 N->getConstantOperandVal(3));
1503 Chain = Parts[i].getValue(1);
1504 }
1505
1506 // Handle endianness of the load.
1507 if (DAG.getDataLayout().isBigEndian())
1508 std::reverse(Parts.begin(), Parts.end());
1509
1510 // Assemble the parts in the promoted type.
1511 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1512 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1513 for (unsigned i = 1; i < NumRegs; ++i) {
1514 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1515 // Shift it to the right position and "or" it in.
1516 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1517 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1518 TLI.getPointerTy(DAG.getDataLayout())));
1519 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1520 }
1521
1522 // Modified the chain result - switch anything that used the old chain to
1523 // use the new one.
1524 ReplaceValueWith(SDValue(N, 1), Chain);
1525
1526 return Res;
1527 }
1528
1529 //===----------------------------------------------------------------------===//
1530 // Integer Operand Promotion
1531 //===----------------------------------------------------------------------===//
1532
1533 /// PromoteIntegerOperand - This method is called when the specified operand of
1534 /// the specified node is found to need promotion. At this point, all of the
1535 /// result types of the node are known to be legal, but other operands of the
1536 /// node may need promotion or expansion as well as the specified one.
PromoteIntegerOperand(SDNode * N,unsigned OpNo)1537 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1538 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1539 dbgs() << "\n");
1540 SDValue Res = SDValue();
1541 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1542 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1543 return false;
1544 }
1545
1546 switch (N->getOpcode()) {
1547 default:
1548 #ifndef NDEBUG
1549 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1550 N->dump(&DAG); dbgs() << "\n";
1551 #endif
1552 llvm_unreachable("Do not know how to promote this operator's operand!");
1553
1554 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1555 case ISD::ATOMIC_STORE:
1556 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1557 break;
1558 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1559 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1560 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1561 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1562 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1563 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1564 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1565 case ISD::INSERT_VECTOR_ELT:
1566 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1567 case ISD::SCALAR_TO_VECTOR:
1568 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1569 case ISD::SPLAT_VECTOR:
1570 Res = PromoteIntOp_SPLAT_VECTOR(N); break;
1571 case ISD::VSELECT:
1572 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1573 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1574 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1575 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1576 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1577 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1578 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1579 OpNo); break;
1580 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1581 OpNo); break;
1582 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1583 OpNo); break;
1584 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1585 OpNo); break;
1586 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1587 OpNo); break;
1588 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1589 case ISD::FP16_TO_FP:
1590 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1591 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1592 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1593 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1594 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
1595
1596 case ISD::SHL:
1597 case ISD::SRA:
1598 case ISD::SRL:
1599 case ISD::ROTL:
1600 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1601
1602 case ISD::SADDO_CARRY:
1603 case ISD::SSUBO_CARRY:
1604 case ISD::ADDCARRY:
1605 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1606
1607 case ISD::FRAMEADDR:
1608 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1609
1610 case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1611
1612 case ISD::SMULFIX:
1613 case ISD::SMULFIXSAT:
1614 case ISD::UMULFIX:
1615 case ISD::UMULFIXSAT:
1616 case ISD::SDIVFIX:
1617 case ISD::SDIVFIXSAT:
1618 case ISD::UDIVFIX:
1619 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1620
1621 case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1622
1623 case ISD::VECREDUCE_ADD:
1624 case ISD::VECREDUCE_MUL:
1625 case ISD::VECREDUCE_AND:
1626 case ISD::VECREDUCE_OR:
1627 case ISD::VECREDUCE_XOR:
1628 case ISD::VECREDUCE_SMAX:
1629 case ISD::VECREDUCE_SMIN:
1630 case ISD::VECREDUCE_UMAX:
1631 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1632 case ISD::VP_REDUCE_ADD:
1633 case ISD::VP_REDUCE_MUL:
1634 case ISD::VP_REDUCE_AND:
1635 case ISD::VP_REDUCE_OR:
1636 case ISD::VP_REDUCE_XOR:
1637 case ISD::VP_REDUCE_SMAX:
1638 case ISD::VP_REDUCE_SMIN:
1639 case ISD::VP_REDUCE_UMAX:
1640 case ISD::VP_REDUCE_UMIN:
1641 Res = PromoteIntOp_VP_REDUCE(N, OpNo);
1642 break;
1643
1644 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
1645 }
1646
1647 // If the result is null, the sub-method took care of registering results etc.
1648 if (!Res.getNode()) return false;
1649
1650 // If the result is N, the sub-method updated N in place. Tell the legalizer
1651 // core about this.
1652 if (Res.getNode() == N)
1653 return true;
1654
1655 const bool IsStrictFp = N->isStrictFPOpcode();
1656 assert(Res.getValueType() == N->getValueType(0) &&
1657 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1658 "Invalid operand expansion");
1659 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1660 Res.dump());
1661
1662 ReplaceValueWith(SDValue(N, 0), Res);
1663 if (IsStrictFp)
1664 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1665
1666 return false;
1667 }
1668
1669 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1670 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
PromoteSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode CCCode)1671 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1672 ISD::CondCode CCCode) {
1673 // We have to insert explicit sign or zero extends. Note that we could
1674 // insert sign extends for ALL conditions. For those operations where either
1675 // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1676 // which will choose the cheapest for the target.
1677 switch (CCCode) {
1678 default: llvm_unreachable("Unknown integer comparison!");
1679 case ISD::SETEQ:
1680 case ISD::SETNE: {
1681 SDValue OpL = GetPromotedInteger(NewLHS);
1682 SDValue OpR = GetPromotedInteger(NewRHS);
1683
1684 // We would prefer to promote the comparison operand with sign extension.
1685 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1686 // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1687 // instruction, which is redundant eventually.
1688 unsigned OpLEffectiveBits =
1689 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1690 unsigned OpREffectiveBits =
1691 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1692 if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1693 OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1694 NewLHS = OpL;
1695 NewRHS = OpR;
1696 } else {
1697 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1698 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1699 }
1700 break;
1701 }
1702 case ISD::SETUGE:
1703 case ISD::SETUGT:
1704 case ISD::SETULE:
1705 case ISD::SETULT:
1706 NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1707 NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1708 break;
1709 case ISD::SETGE:
1710 case ISD::SETGT:
1711 case ISD::SETLT:
1712 case ISD::SETLE:
1713 NewLHS = SExtPromotedInteger(NewLHS);
1714 NewRHS = SExtPromotedInteger(NewRHS);
1715 break;
1716 }
1717 }
1718
PromoteIntOp_ANY_EXTEND(SDNode * N)1719 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1720 SDValue Op = GetPromotedInteger(N->getOperand(0));
1721 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1722 }
1723
PromoteIntOp_ATOMIC_STORE(AtomicSDNode * N)1724 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1725 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1726 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1727 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1728 }
1729
PromoteIntOp_BITCAST(SDNode * N)1730 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1731 // This should only occur in unusual situations like bitcasting to an
1732 // x86_fp80, so just turn it into a store+load
1733 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1734 }
1735
PromoteIntOp_BR_CC(SDNode * N,unsigned OpNo)1736 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1737 assert(OpNo == 2 && "Don't know how to promote this operand!");
1738
1739 SDValue LHS = N->getOperand(2);
1740 SDValue RHS = N->getOperand(3);
1741 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1742
1743 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1744 // legal types.
1745 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1746 N->getOperand(1), LHS, RHS, N->getOperand(4)),
1747 0);
1748 }
1749
PromoteIntOp_BRCOND(SDNode * N,unsigned OpNo)1750 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1751 assert(OpNo == 1 && "only know how to promote condition");
1752
1753 // Promote all the way up to the canonical SetCC type.
1754 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1755
1756 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1757 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1758 N->getOperand(2)), 0);
1759 }
1760
PromoteIntOp_BUILD_PAIR(SDNode * N)1761 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1762 // Since the result type is legal, the operands must promote to it.
1763 EVT OVT = N->getOperand(0).getValueType();
1764 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1765 SDValue Hi = GetPromotedInteger(N->getOperand(1));
1766 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1767 SDLoc dl(N);
1768
1769 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1770 DAG.getConstant(OVT.getSizeInBits(), dl,
1771 TLI.getPointerTy(DAG.getDataLayout())));
1772 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1773 }
1774
PromoteIntOp_BUILD_VECTOR(SDNode * N)1775 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1776 // The vector type is legal but the element type is not. This implies
1777 // that the vector is a power-of-two in length and that the element
1778 // type does not have a strange size (eg: it is not i1).
1779 EVT VecVT = N->getValueType(0);
1780 unsigned NumElts = VecVT.getVectorNumElements();
1781 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1782 "Legal vector of one illegal element?");
1783
1784 // Promote the inserted value. The type does not need to match the
1785 // vector element type. Check that any extra bits introduced will be
1786 // truncated away.
1787 assert(N->getOperand(0).getValueSizeInBits() >=
1788 N->getValueType(0).getScalarSizeInBits() &&
1789 "Type of inserted value narrower than vector element type!");
1790
1791 SmallVector<SDValue, 16> NewOps;
1792 for (unsigned i = 0; i < NumElts; ++i)
1793 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1794
1795 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1796 }
1797
PromoteIntOp_INSERT_VECTOR_ELT(SDNode * N,unsigned OpNo)1798 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1799 unsigned OpNo) {
1800 if (OpNo == 1) {
1801 // Promote the inserted value. This is valid because the type does not
1802 // have to match the vector element type.
1803
1804 // Check that any extra bits introduced will be truncated away.
1805 assert(N->getOperand(1).getValueSizeInBits() >=
1806 N->getValueType(0).getScalarSizeInBits() &&
1807 "Type of inserted value narrower than vector element type!");
1808 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1809 GetPromotedInteger(N->getOperand(1)),
1810 N->getOperand(2)),
1811 0);
1812 }
1813
1814 assert(OpNo == 2 && "Different operand and result vector types?");
1815
1816 // Promote the index.
1817 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1818 TLI.getVectorIdxTy(DAG.getDataLayout()));
1819 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1820 N->getOperand(1), Idx), 0);
1821 }
1822
PromoteIntOp_SCALAR_TO_VECTOR(SDNode * N)1823 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1824 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1825 // the operand in place.
1826 return SDValue(DAG.UpdateNodeOperands(N,
1827 GetPromotedInteger(N->getOperand(0))), 0);
1828 }
1829
PromoteIntOp_SPLAT_VECTOR(SDNode * N)1830 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1831 // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
1832 // operand in place.
1833 return SDValue(
1834 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1835 }
1836
PromoteIntOp_SELECT(SDNode * N,unsigned OpNo)1837 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1838 assert(OpNo == 0 && "Only know how to promote the condition!");
1839 SDValue Cond = N->getOperand(0);
1840 EVT OpTy = N->getOperand(1).getValueType();
1841
1842 if (N->getOpcode() == ISD::VSELECT)
1843 if (SDValue Res = WidenVSELECTMask(N))
1844 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1845 Res, N->getOperand(1), N->getOperand(2));
1846
1847 // Promote all the way up to the canonical SetCC type.
1848 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1849 Cond = PromoteTargetBoolean(Cond, OpVT);
1850
1851 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1852 N->getOperand(2)), 0);
1853 }
1854
PromoteIntOp_SELECT_CC(SDNode * N,unsigned OpNo)1855 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1856 assert(OpNo == 0 && "Don't know how to promote this operand!");
1857
1858 SDValue LHS = N->getOperand(0);
1859 SDValue RHS = N->getOperand(1);
1860 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1861
1862 // The CC (#4) and the possible return values (#2 and #3) have legal types.
1863 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1864 N->getOperand(3), N->getOperand(4)), 0);
1865 }
1866
PromoteIntOp_SETCC(SDNode * N,unsigned OpNo)1867 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1868 assert(OpNo == 0 && "Don't know how to promote this operand!");
1869
1870 SDValue LHS = N->getOperand(0);
1871 SDValue RHS = N->getOperand(1);
1872 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1873
1874 // The CC (#2) is always legal.
1875 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1876 }
1877
PromoteIntOp_Shift(SDNode * N)1878 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1879 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1880 ZExtPromotedInteger(N->getOperand(1))), 0);
1881 }
1882
PromoteIntOp_SIGN_EXTEND(SDNode * N)1883 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1884 SDValue Op = GetPromotedInteger(N->getOperand(0));
1885 SDLoc dl(N);
1886 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1887 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1888 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1889 }
1890
PromoteIntOp_SINT_TO_FP(SDNode * N)1891 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1892 return SDValue(DAG.UpdateNodeOperands(N,
1893 SExtPromotedInteger(N->getOperand(0))), 0);
1894 }
1895
PromoteIntOp_STRICT_SINT_TO_FP(SDNode * N)1896 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
1897 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1898 SExtPromotedInteger(N->getOperand(1))), 0);
1899 }
1900
PromoteIntOp_STORE(StoreSDNode * N,unsigned OpNo)1901 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1902 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1903 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1904 SDLoc dl(N);
1905
1906 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1907
1908 // Truncate the value and store the result.
1909 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1910 N->getMemoryVT(), N->getMemOperand());
1911 }
1912
PromoteIntOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)1913 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1914 unsigned OpNo) {
1915 SDValue DataOp = N->getValue();
1916 SDValue Mask = N->getMask();
1917
1918 if (OpNo == 4) {
1919 // The Mask. Update in place.
1920 EVT DataVT = DataOp.getValueType();
1921 Mask = PromoteTargetBoolean(Mask, DataVT);
1922 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1923 NewOps[4] = Mask;
1924 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1925 }
1926
1927 assert(OpNo == 1 && "Unexpected operand for promotion");
1928 DataOp = GetPromotedInteger(DataOp);
1929
1930 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
1931 N->getOffset(), Mask, N->getMemoryVT(),
1932 N->getMemOperand(), N->getAddressingMode(),
1933 /*IsTruncating*/ true, N->isCompressingStore());
1934 }
1935
PromoteIntOp_MLOAD(MaskedLoadSDNode * N,unsigned OpNo)1936 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1937 unsigned OpNo) {
1938 assert(OpNo == 3 && "Only know how to promote the mask!");
1939 EVT DataVT = N->getValueType(0);
1940 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1941 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1942 NewOps[OpNo] = Mask;
1943 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1944 if (Res == N)
1945 return SDValue(Res, 0);
1946
1947 // Update triggered CSE, do our own replacement since caller can't.
1948 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1949 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1950 return SDValue();
1951 }
1952
PromoteIntOp_MGATHER(MaskedGatherSDNode * N,unsigned OpNo)1953 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1954 unsigned OpNo) {
1955
1956 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1957 if (OpNo == 2) {
1958 // The Mask
1959 EVT DataVT = N->getValueType(0);
1960 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1961 } else if (OpNo == 4) {
1962 // The Index
1963 if (N->isIndexSigned())
1964 // Need to sign extend the index since the bits will likely be used.
1965 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1966 else
1967 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1968 } else
1969 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1970
1971 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1972 if (Res == N)
1973 return SDValue(Res, 0);
1974
1975 // Update triggered CSE, do our own replacement since caller can't.
1976 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1977 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1978 return SDValue();
1979 }
1980
PromoteIntOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)1981 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1982 unsigned OpNo) {
1983 bool TruncateStore = N->isTruncatingStore();
1984 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1985 if (OpNo == 2) {
1986 // The Mask
1987 EVT DataVT = N->getValue().getValueType();
1988 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1989 } else if (OpNo == 4) {
1990 // The Index
1991 if (N->isIndexSigned())
1992 // Need to sign extend the index since the bits will likely be used.
1993 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1994 else
1995 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1996
1997 N->setIndexType(TLI.getCanonicalIndexType(N->getIndexType(),
1998 N->getMemoryVT(), NewOps[OpNo]));
1999 } else {
2000 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2001 TruncateStore = true;
2002 }
2003
2004 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2005 SDLoc(N), NewOps, N->getMemOperand(),
2006 N->getIndexType(), TruncateStore);
2007 }
2008
PromoteIntOp_TRUNCATE(SDNode * N)2009 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2010 SDValue Op = GetPromotedInteger(N->getOperand(0));
2011 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2012 }
2013
PromoteIntOp_UINT_TO_FP(SDNode * N)2014 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2015 return SDValue(DAG.UpdateNodeOperands(N,
2016 ZExtPromotedInteger(N->getOperand(0))), 0);
2017 }
2018
PromoteIntOp_STRICT_UINT_TO_FP(SDNode * N)2019 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2020 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2021 ZExtPromotedInteger(N->getOperand(1))), 0);
2022 }
2023
PromoteIntOp_ZERO_EXTEND(SDNode * N)2024 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2025 SDLoc dl(N);
2026 SDValue Op = GetPromotedInteger(N->getOperand(0));
2027 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2028 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
2029 }
2030
PromoteIntOp_ADDSUBCARRY(SDNode * N,unsigned OpNo)2031 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
2032 assert(OpNo == 2 && "Don't know how to promote this operand!");
2033
2034 SDValue LHS = N->getOperand(0);
2035 SDValue RHS = N->getOperand(1);
2036 SDValue Carry = N->getOperand(2);
2037 SDLoc DL(N);
2038
2039 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
2040
2041 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
2042 }
2043
PromoteIntOp_FIX(SDNode * N)2044 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2045 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2046 return SDValue(
2047 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2048 }
2049
PromoteIntOp_FRAMERETURNADDR(SDNode * N)2050 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2051 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2052 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2053 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2054 }
2055
PromoteIntOp_PREFETCH(SDNode * N,unsigned OpNo)2056 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
2057 assert(OpNo > 1 && "Don't know how to promote this operand!");
2058 // Promote the rw, locality, and cache type arguments to a supported integer
2059 // width.
2060 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2061 SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
2062 SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
2063 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2064 Op2, Op3, Op4),
2065 0);
2066 }
2067
PromoteIntOp_FPOWI(SDNode * N)2068 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
2069 // FIXME: Support for promotion of STRICT_FPOWI is not implemented yet.
2070 assert(N->getOpcode() == ISD::FPOWI && "No STRICT_FPOWI support here yet.");
2071
2072 // The integer operand is the last operand in FPOWI (so the result and
2073 // floating point operand is already type legalized).
2074
2075 // We can't just promote the exponent type in FPOWI, since we want to lower
2076 // the node to a libcall and we if we promote to a type larger than
2077 // sizeof(int) the libcall might not be according to the targets ABI. Instead
2078 // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2079 // if the target accepts it according to shouldSignExtendTypeInLibCall.
2080 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
2081 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
2082 if (!TLI.getLibcallName(LC)) {
2083 // Some targets don't have a powi libcall; use pow instead.
2084 // FIXME: Implement this if some target needs it.
2085 DAG.getContext()->emitError("Don't know how to promote fpowi to fpow");
2086 return DAG.getUNDEF(N->getValueType(0));
2087 }
2088 // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2089 assert(DAG.getLibInfo().getIntSize() ==
2090 N->getOperand(1).getValueType().getSizeInBits() &&
2091 "POWI exponent should match with sizeof(int) when doing the libcall.");
2092 TargetLowering::MakeLibCallOptions CallOptions;
2093 CallOptions.setSExt(true);
2094 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2095 std::pair<SDValue, SDValue> Tmp =
2096 TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops,
2097 CallOptions, SDLoc(N), SDValue());
2098 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2099 return SDValue();
2100 }
2101
getExtendForIntVecReduction(SDNode * N)2102 static unsigned getExtendForIntVecReduction(SDNode *N) {
2103 switch (N->getOpcode()) {
2104 default:
2105 llvm_unreachable("Expected integer vector reduction");
2106 case ISD::VECREDUCE_ADD:
2107 case ISD::VECREDUCE_MUL:
2108 case ISD::VECREDUCE_AND:
2109 case ISD::VECREDUCE_OR:
2110 case ISD::VECREDUCE_XOR:
2111 case ISD::VP_REDUCE_ADD:
2112 case ISD::VP_REDUCE_MUL:
2113 case ISD::VP_REDUCE_AND:
2114 case ISD::VP_REDUCE_OR:
2115 case ISD::VP_REDUCE_XOR:
2116 return ISD::ANY_EXTEND;
2117 case ISD::VECREDUCE_SMAX:
2118 case ISD::VECREDUCE_SMIN:
2119 case ISD::VP_REDUCE_SMAX:
2120 case ISD::VP_REDUCE_SMIN:
2121 return ISD::SIGN_EXTEND;
2122 case ISD::VECREDUCE_UMAX:
2123 case ISD::VECREDUCE_UMIN:
2124 case ISD::VP_REDUCE_UMAX:
2125 case ISD::VP_REDUCE_UMIN:
2126 return ISD::ZERO_EXTEND;
2127 }
2128 }
2129
PromoteIntOpVectorReduction(SDNode * N,SDValue V)2130 SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2131 switch (getExtendForIntVecReduction(N)) {
2132 default:
2133 llvm_unreachable("Impossible extension kind for integer reduction");
2134 case ISD::ANY_EXTEND:
2135 return GetPromotedInteger(V);
2136 case ISD::SIGN_EXTEND:
2137 return SExtPromotedInteger(V);
2138 case ISD::ZERO_EXTEND:
2139 return ZExtPromotedInteger(V);
2140 }
2141 }
2142
PromoteIntOp_VECREDUCE(SDNode * N)2143 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2144 SDLoc dl(N);
2145 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2146
2147 EVT EltVT = Op.getValueType().getVectorElementType();
2148 EVT VT = N->getValueType(0);
2149
2150 if (VT.bitsGE(EltVT))
2151 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
2152
2153 // Result size must be >= element size. If this is not the case after
2154 // promotion, also promote the result type and then truncate.
2155 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
2156 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
2157 }
2158
PromoteIntOp_VP_REDUCE(SDNode * N,unsigned OpNo)2159 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2160 SDLoc DL(N);
2161 SDValue Op = N->getOperand(OpNo);
2162 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2163
2164 if (OpNo == 2) { // Mask
2165 // Update in place.
2166 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2167 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2168 }
2169
2170 assert(OpNo == 1 && "Unexpected operand for promotion");
2171
2172 Op = PromoteIntOpVectorReduction(N, Op);
2173
2174 NewOps[OpNo] = Op;
2175
2176 EVT VT = N->getValueType(0);
2177 EVT EltVT = Op.getValueType().getScalarType();
2178
2179 if (VT.bitsGE(EltVT))
2180 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
2181
2182 // Result size must be >= element/start-value size. If this is not the case
2183 // after promotion, also promote both the start value and result type and
2184 // then truncate.
2185 NewOps[0] =
2186 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
2187 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
2188 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2189 }
2190
PromoteIntOp_SET_ROUNDING(SDNode * N)2191 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2192 SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2193 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2194 }
2195
2196 //===----------------------------------------------------------------------===//
2197 // Integer Result Expansion
2198 //===----------------------------------------------------------------------===//
2199
2200 /// ExpandIntegerResult - This method is called when the specified result of the
2201 /// specified node is found to need expansion. At this point, the node may also
2202 /// have invalid operands or may have other results that need promotion, we just
2203 /// know that (at least) one result needs expansion.
ExpandIntegerResult(SDNode * N,unsigned ResNo)2204 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2205 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
2206 dbgs() << "\n");
2207 SDValue Lo, Hi;
2208 Lo = Hi = SDValue();
2209
2210 // See if the target wants to custom expand this node.
2211 if (CustomLowerNode(N, N->getValueType(ResNo), true))
2212 return;
2213
2214 switch (N->getOpcode()) {
2215 default:
2216 #ifndef NDEBUG
2217 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2218 N->dump(&DAG); dbgs() << "\n";
2219 #endif
2220 report_fatal_error("Do not know how to expand the result of this "
2221 "operator!");
2222
2223 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
2224 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2225 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
2226 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2227 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2228 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2229
2230 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2231 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2232 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2233 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2234 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2235
2236 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2237 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2238 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2239 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2240 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2241 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2242 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2243 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2244 case ISD::CTLZ_ZERO_UNDEF:
2245 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2246 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2247 case ISD::CTTZ_ZERO_UNDEF:
2248 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2249 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
2250 case ISD::STRICT_FP_TO_SINT:
2251 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
2252 case ISD::STRICT_FP_TO_UINT:
2253 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
2254 case ISD::FP_TO_SINT_SAT:
2255 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2256 case ISD::STRICT_LLROUND:
2257 case ISD::STRICT_LLRINT:
2258 case ISD::LLROUND:
2259 case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
2260 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2261 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2262 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
2263 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2264 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2265 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2266 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2267 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2268 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2269 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2270 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2271 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2272
2273 case ISD::ATOMIC_LOAD_ADD:
2274 case ISD::ATOMIC_LOAD_SUB:
2275 case ISD::ATOMIC_LOAD_AND:
2276 case ISD::ATOMIC_LOAD_CLR:
2277 case ISD::ATOMIC_LOAD_OR:
2278 case ISD::ATOMIC_LOAD_XOR:
2279 case ISD::ATOMIC_LOAD_NAND:
2280 case ISD::ATOMIC_LOAD_MIN:
2281 case ISD::ATOMIC_LOAD_MAX:
2282 case ISD::ATOMIC_LOAD_UMIN:
2283 case ISD::ATOMIC_LOAD_UMAX:
2284 case ISD::ATOMIC_SWAP:
2285 case ISD::ATOMIC_CMP_SWAP: {
2286 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2287 SplitInteger(Tmp.first, Lo, Hi);
2288 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2289 break;
2290 }
2291 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2292 AtomicSDNode *AN = cast<AtomicSDNode>(N);
2293 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2294 SDValue Tmp = DAG.getAtomicCmpSwap(
2295 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
2296 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2297 AN->getMemOperand());
2298
2299 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2300 // success simply by comparing the loaded value against the ingoing
2301 // comparison.
2302 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2303 N->getOperand(2), ISD::SETEQ);
2304
2305 SplitInteger(Tmp, Lo, Hi);
2306 ReplaceValueWith(SDValue(N, 1), Success);
2307 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2308 break;
2309 }
2310
2311 case ISD::AND:
2312 case ISD::OR:
2313 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2314
2315 case ISD::UMAX:
2316 case ISD::SMAX:
2317 case ISD::UMIN:
2318 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2319
2320 case ISD::ADD:
2321 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2322
2323 case ISD::ADDC:
2324 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2325
2326 case ISD::ADDE:
2327 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2328
2329 case ISD::ADDCARRY:
2330 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
2331
2332 case ISD::SADDO_CARRY:
2333 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2334
2335 case ISD::SHL:
2336 case ISD::SRA:
2337 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2338
2339 case ISD::SADDO:
2340 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2341 case ISD::UADDO:
2342 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2343 case ISD::UMULO:
2344 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2345
2346 case ISD::SADDSAT:
2347 case ISD::UADDSAT:
2348 case ISD::SSUBSAT:
2349 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2350
2351 case ISD::SSHLSAT:
2352 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2353
2354 case ISD::SMULFIX:
2355 case ISD::SMULFIXSAT:
2356 case ISD::UMULFIX:
2357 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2358
2359 case ISD::SDIVFIX:
2360 case ISD::SDIVFIXSAT:
2361 case ISD::UDIVFIX:
2362 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2363
2364 case ISD::VECREDUCE_ADD:
2365 case ISD::VECREDUCE_MUL:
2366 case ISD::VECREDUCE_AND:
2367 case ISD::VECREDUCE_OR:
2368 case ISD::VECREDUCE_XOR:
2369 case ISD::VECREDUCE_SMAX:
2370 case ISD::VECREDUCE_SMIN:
2371 case ISD::VECREDUCE_UMAX:
2372 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2373
2374 case ISD::ROTL:
2375 case ISD::ROTR:
2376 ExpandIntRes_Rotate(N, Lo, Hi);
2377 break;
2378
2379 case ISD::FSHL:
2380 case ISD::FSHR:
2381 ExpandIntRes_FunnelShift(N, Lo, Hi);
2382 break;
2383
2384 case ISD::VSCALE:
2385 ExpandIntRes_VSCALE(N, Lo, Hi);
2386 break;
2387 }
2388
2389 // If Lo/Hi is null, the sub-method took care of registering results etc.
2390 if (Lo.getNode())
2391 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2392 }
2393
2394 /// Lower an atomic node to the appropriate builtin call.
ExpandAtomic(SDNode * Node)2395 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2396 unsigned Opc = Node->getOpcode();
2397 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2398 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
2399 // Lower to outline atomic libcall if outline atomics enabled,
2400 // or to sync libcall otherwise
2401 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2402 EVT RetVT = Node->getValueType(0);
2403 TargetLowering::MakeLibCallOptions CallOptions;
2404 SmallVector<SDValue, 4> Ops;
2405 if (TLI.getLibcallName(LC)) {
2406 Ops.append(Node->op_begin() + 2, Node->op_end());
2407 Ops.push_back(Node->getOperand(1));
2408 } else {
2409 LC = RTLIB::getSYNC(Opc, VT);
2410 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2411 "Unexpected atomic op or value type!");
2412 Ops.append(Node->op_begin() + 1, Node->op_end());
2413 }
2414 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2415 Node->getOperand(0));
2416 }
2417
2418 /// N is a shift by a value that needs to be expanded,
2419 /// and the shift amount is a constant 'Amt'. Expand the operation.
ExpandShiftByConstant(SDNode * N,const APInt & Amt,SDValue & Lo,SDValue & Hi)2420 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2421 SDValue &Lo, SDValue &Hi) {
2422 SDLoc DL(N);
2423 // Expand the incoming operand to be shifted, so that we have its parts
2424 SDValue InL, InH;
2425 GetExpandedInteger(N->getOperand(0), InL, InH);
2426
2427 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2428 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2429 if (!Amt) {
2430 Lo = InL;
2431 Hi = InH;
2432 return;
2433 }
2434
2435 EVT NVT = InL.getValueType();
2436 unsigned VTBits = N->getValueType(0).getSizeInBits();
2437 unsigned NVTBits = NVT.getSizeInBits();
2438 EVT ShTy = N->getOperand(1).getValueType();
2439
2440 if (N->getOpcode() == ISD::SHL) {
2441 if (Amt.ugt(VTBits)) {
2442 Lo = Hi = DAG.getConstant(0, DL, NVT);
2443 } else if (Amt.ugt(NVTBits)) {
2444 Lo = DAG.getConstant(0, DL, NVT);
2445 Hi = DAG.getNode(ISD::SHL, DL,
2446 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2447 } else if (Amt == NVTBits) {
2448 Lo = DAG.getConstant(0, DL, NVT);
2449 Hi = InL;
2450 } else {
2451 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
2452 Hi = DAG.getNode(ISD::OR, DL, NVT,
2453 DAG.getNode(ISD::SHL, DL, NVT, InH,
2454 DAG.getConstant(Amt, DL, ShTy)),
2455 DAG.getNode(ISD::SRL, DL, NVT, InL,
2456 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2457 }
2458 return;
2459 }
2460
2461 if (N->getOpcode() == ISD::SRL) {
2462 if (Amt.ugt(VTBits)) {
2463 Lo = Hi = DAG.getConstant(0, DL, NVT);
2464 } else if (Amt.ugt(NVTBits)) {
2465 Lo = DAG.getNode(ISD::SRL, DL,
2466 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2467 Hi = DAG.getConstant(0, DL, NVT);
2468 } else if (Amt == NVTBits) {
2469 Lo = InH;
2470 Hi = DAG.getConstant(0, DL, NVT);
2471 } else {
2472 Lo = DAG.getNode(ISD::OR, DL, NVT,
2473 DAG.getNode(ISD::SRL, DL, NVT, InL,
2474 DAG.getConstant(Amt, DL, ShTy)),
2475 DAG.getNode(ISD::SHL, DL, NVT, InH,
2476 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2477 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2478 }
2479 return;
2480 }
2481
2482 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2483 if (Amt.ugt(VTBits)) {
2484 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2485 DAG.getConstant(NVTBits - 1, DL, ShTy));
2486 } else if (Amt.ugt(NVTBits)) {
2487 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2488 DAG.getConstant(Amt - NVTBits, DL, ShTy));
2489 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2490 DAG.getConstant(NVTBits - 1, DL, ShTy));
2491 } else if (Amt == NVTBits) {
2492 Lo = InH;
2493 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2494 DAG.getConstant(NVTBits - 1, DL, ShTy));
2495 } else {
2496 Lo = DAG.getNode(ISD::OR, DL, NVT,
2497 DAG.getNode(ISD::SRL, DL, NVT, InL,
2498 DAG.getConstant(Amt, DL, ShTy)),
2499 DAG.getNode(ISD::SHL, DL, NVT, InH,
2500 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2501 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2502 }
2503 }
2504
2505 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2506 /// this shift based on knowledge of the high bit of the shift amount. If we
2507 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2508 /// shift amount.
2509 bool DAGTypeLegalizer::
ExpandShiftWithKnownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2510 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2511 SDValue Amt = N->getOperand(1);
2512 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2513 EVT ShTy = Amt.getValueType();
2514 unsigned ShBits = ShTy.getScalarSizeInBits();
2515 unsigned NVTBits = NVT.getScalarSizeInBits();
2516 assert(isPowerOf2_32(NVTBits) &&
2517 "Expanded integer type size not a power of two!");
2518 SDLoc dl(N);
2519
2520 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2521 KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2522
2523 // If we don't know anything about the high bits, exit.
2524 if (((Known.Zero|Known.One) & HighBitMask) == 0)
2525 return false;
2526
2527 // Get the incoming operand to be shifted.
2528 SDValue InL, InH;
2529 GetExpandedInteger(N->getOperand(0), InL, InH);
2530
2531 // If we know that any of the high bits of the shift amount are one, then we
2532 // can do this as a couple of simple shifts.
2533 if (Known.One.intersects(HighBitMask)) {
2534 // Mask out the high bit, which we know is set.
2535 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2536 DAG.getConstant(~HighBitMask, dl, ShTy));
2537
2538 switch (N->getOpcode()) {
2539 default: llvm_unreachable("Unknown shift");
2540 case ISD::SHL:
2541 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2542 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2543 return true;
2544 case ISD::SRL:
2545 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2546 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2547 return true;
2548 case ISD::SRA:
2549 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2550 DAG.getConstant(NVTBits - 1, dl, ShTy));
2551 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2552 return true;
2553 }
2554 }
2555
2556 // If we know that all of the high bits of the shift amount are zero, then we
2557 // can do this as a couple of simple shifts.
2558 if (HighBitMask.isSubsetOf(Known.Zero)) {
2559 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2560 // shift if x is zero. We can use XOR here because x is known to be smaller
2561 // than 32.
2562 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2563 DAG.getConstant(NVTBits - 1, dl, ShTy));
2564
2565 unsigned Op1, Op2;
2566 switch (N->getOpcode()) {
2567 default: llvm_unreachable("Unknown shift");
2568 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2569 case ISD::SRL:
2570 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2571 }
2572
2573 // When shifting right the arithmetic for Lo and Hi is swapped.
2574 if (N->getOpcode() != ISD::SHL)
2575 std::swap(InL, InH);
2576
2577 // Use a little trick to get the bits that move from Lo to Hi. First
2578 // shift by one bit.
2579 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2580 // Then compute the remaining shift with amount-1.
2581 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2582
2583 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2584 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2585
2586 if (N->getOpcode() != ISD::SHL)
2587 std::swap(Hi, Lo);
2588 return true;
2589 }
2590
2591 return false;
2592 }
2593
2594 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2595 /// of any size.
2596 bool DAGTypeLegalizer::
ExpandShiftWithUnknownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2597 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2598 SDValue Amt = N->getOperand(1);
2599 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2600 EVT ShTy = Amt.getValueType();
2601 unsigned NVTBits = NVT.getSizeInBits();
2602 assert(isPowerOf2_32(NVTBits) &&
2603 "Expanded integer type size not a power of two!");
2604 SDLoc dl(N);
2605
2606 // Get the incoming operand to be shifted.
2607 SDValue InL, InH;
2608 GetExpandedInteger(N->getOperand(0), InL, InH);
2609
2610 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2611 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2612 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2613 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2614 Amt, NVBitsNode, ISD::SETULT);
2615 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2616 Amt, DAG.getConstant(0, dl, ShTy),
2617 ISD::SETEQ);
2618
2619 SDValue LoS, HiS, LoL, HiL;
2620 switch (N->getOpcode()) {
2621 default: llvm_unreachable("Unknown shift");
2622 case ISD::SHL:
2623 // Short: ShAmt < NVTBits
2624 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2625 HiS = DAG.getNode(ISD::OR, dl, NVT,
2626 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2627 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2628
2629 // Long: ShAmt >= NVTBits
2630 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2631 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2632
2633 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2634 Hi = DAG.getSelect(dl, NVT, isZero, InH,
2635 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2636 return true;
2637 case ISD::SRL:
2638 // Short: ShAmt < NVTBits
2639 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2640 LoS = DAG.getNode(ISD::OR, dl, NVT,
2641 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2642 // FIXME: If Amt is zero, the following shift generates an undefined result
2643 // on some architectures.
2644 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2645
2646 // Long: ShAmt >= NVTBits
2647 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2648 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2649
2650 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2651 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2652 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2653 return true;
2654 case ISD::SRA:
2655 // Short: ShAmt < NVTBits
2656 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2657 LoS = DAG.getNode(ISD::OR, dl, NVT,
2658 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2659 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2660
2661 // Long: ShAmt >= NVTBits
2662 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2663 DAG.getConstant(NVTBits - 1, dl, ShTy));
2664 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2665
2666 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2667 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2668 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2669 return true;
2670 }
2671 }
2672
getExpandedMinMaxOps(int Op)2673 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2674
2675 switch (Op) {
2676 default: llvm_unreachable("invalid min/max opcode");
2677 case ISD::SMAX:
2678 return std::make_pair(ISD::SETGT, ISD::UMAX);
2679 case ISD::UMAX:
2680 return std::make_pair(ISD::SETUGT, ISD::UMAX);
2681 case ISD::SMIN:
2682 return std::make_pair(ISD::SETLT, ISD::UMIN);
2683 case ISD::UMIN:
2684 return std::make_pair(ISD::SETULT, ISD::UMIN);
2685 }
2686 }
2687
ExpandIntRes_MINMAX(SDNode * N,SDValue & Lo,SDValue & Hi)2688 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2689 SDValue &Lo, SDValue &Hi) {
2690 SDLoc DL(N);
2691 ISD::NodeType LoOpc;
2692 ISD::CondCode CondC;
2693 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2694
2695 // Expand the subcomponents.
2696 SDValue LHSL, LHSH, RHSL, RHSH;
2697 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2698 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2699
2700 // Value types
2701 EVT NVT = LHSL.getValueType();
2702 EVT CCT = getSetCCResultType(NVT);
2703
2704 // Hi part is always the same op
2705 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2706
2707 // We need to know whether to select Lo part that corresponds to 'winning'
2708 // Hi part or if Hi parts are equal.
2709 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2710 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2711
2712 // Lo part corresponding to the 'winning' Hi part
2713 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2714
2715 // Recursed Lo part if Hi parts are equal, this uses unsigned version
2716 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2717
2718 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2719 }
2720
ExpandIntRes_ADDSUB(SDNode * N,SDValue & Lo,SDValue & Hi)2721 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2722 SDValue &Lo, SDValue &Hi) {
2723 SDLoc dl(N);
2724 // Expand the subcomponents.
2725 SDValue LHSL, LHSH, RHSL, RHSH;
2726 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2727 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2728
2729 EVT NVT = LHSL.getValueType();
2730 SDValue LoOps[2] = { LHSL, RHSL };
2731 SDValue HiOps[3] = { LHSH, RHSH };
2732
2733 bool HasOpCarry = TLI.isOperationLegalOrCustom(
2734 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2735 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2736 if (HasOpCarry) {
2737 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2738 if (N->getOpcode() == ISD::ADD) {
2739 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2740 HiOps[2] = Lo.getValue(1);
2741 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2742 } else {
2743 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2744 HiOps[2] = Lo.getValue(1);
2745 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2746 }
2747 return;
2748 }
2749
2750 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2751 // them. TODO: Teach operation legalization how to expand unsupported
2752 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2753 // a carry of type MVT::Glue, but there doesn't seem to be any way to
2754 // generate a value of this type in the expanded code sequence.
2755 bool hasCarry =
2756 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2757 ISD::ADDC : ISD::SUBC,
2758 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2759
2760 if (hasCarry) {
2761 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2762 if (N->getOpcode() == ISD::ADD) {
2763 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2764 HiOps[2] = Lo.getValue(1);
2765 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2766 } else {
2767 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2768 HiOps[2] = Lo.getValue(1);
2769 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2770 }
2771 return;
2772 }
2773
2774 bool hasOVF =
2775 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2776 ISD::UADDO : ISD::USUBO,
2777 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2778 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
2779
2780 if (hasOVF) {
2781 EVT OvfVT = getSetCCResultType(NVT);
2782 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2783 int RevOpc;
2784 if (N->getOpcode() == ISD::ADD) {
2785 RevOpc = ISD::SUB;
2786 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2787 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2788 } else {
2789 RevOpc = ISD::ADD;
2790 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2791 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2792 }
2793 SDValue OVF = Lo.getValue(1);
2794
2795 switch (BoolType) {
2796 case TargetLoweringBase::UndefinedBooleanContent:
2797 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2798 LLVM_FALLTHROUGH;
2799 case TargetLoweringBase::ZeroOrOneBooleanContent:
2800 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2801 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2802 break;
2803 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2804 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2805 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2806 }
2807 return;
2808 }
2809
2810 if (N->getOpcode() == ISD::ADD) {
2811 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2812 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2813 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2814 ISD::SETULT);
2815
2816 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
2817 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2818 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2819 return;
2820 }
2821
2822 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2823 DAG.getConstant(1, dl, NVT),
2824 DAG.getConstant(0, dl, NVT));
2825 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2826 ISD::SETULT);
2827 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2828 DAG.getConstant(1, dl, NVT), Carry1);
2829 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2830 } else {
2831 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2832 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2833 SDValue Cmp =
2834 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2835 LoOps[0], LoOps[1], ISD::SETULT);
2836
2837 SDValue Borrow;
2838 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
2839 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2840 else
2841 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2842 DAG.getConstant(0, dl, NVT));
2843
2844 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2845 }
2846 }
2847
ExpandIntRes_ADDSUBC(SDNode * N,SDValue & Lo,SDValue & Hi)2848 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2849 SDValue &Lo, SDValue &Hi) {
2850 // Expand the subcomponents.
2851 SDValue LHSL, LHSH, RHSL, RHSH;
2852 SDLoc dl(N);
2853 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2854 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2855 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2856 SDValue LoOps[2] = { LHSL, RHSL };
2857 SDValue HiOps[3] = { LHSH, RHSH };
2858
2859 if (N->getOpcode() == ISD::ADDC) {
2860 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2861 HiOps[2] = Lo.getValue(1);
2862 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2863 } else {
2864 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2865 HiOps[2] = Lo.getValue(1);
2866 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2867 }
2868
2869 // Legalized the flag result - switch anything that used the old flag to
2870 // use the new one.
2871 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2872 }
2873
ExpandIntRes_ADDSUBE(SDNode * N,SDValue & Lo,SDValue & Hi)2874 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2875 SDValue &Lo, SDValue &Hi) {
2876 // Expand the subcomponents.
2877 SDValue LHSL, LHSH, RHSL, RHSH;
2878 SDLoc dl(N);
2879 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2880 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2881 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2882 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2883 SDValue HiOps[3] = { LHSH, RHSH };
2884
2885 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2886 HiOps[2] = Lo.getValue(1);
2887 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2888
2889 // Legalized the flag result - switch anything that used the old flag to
2890 // use the new one.
2891 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2892 }
2893
ExpandIntRes_UADDSUBO(SDNode * N,SDValue & Lo,SDValue & Hi)2894 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2895 SDValue &Lo, SDValue &Hi) {
2896 SDValue LHS = N->getOperand(0);
2897 SDValue RHS = N->getOperand(1);
2898 SDLoc dl(N);
2899
2900 SDValue Ovf;
2901
2902 unsigned CarryOp, NoCarryOp;
2903 ISD::CondCode Cond;
2904 switch(N->getOpcode()) {
2905 case ISD::UADDO:
2906 CarryOp = ISD::ADDCARRY;
2907 NoCarryOp = ISD::ADD;
2908 Cond = ISD::SETULT;
2909 break;
2910 case ISD::USUBO:
2911 CarryOp = ISD::SUBCARRY;
2912 NoCarryOp = ISD::SUB;
2913 Cond = ISD::SETUGT;
2914 break;
2915 default:
2916 llvm_unreachable("Node has unexpected Opcode");
2917 }
2918
2919 bool HasCarryOp = TLI.isOperationLegalOrCustom(
2920 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2921
2922 if (HasCarryOp) {
2923 // Expand the subcomponents.
2924 SDValue LHSL, LHSH, RHSL, RHSH;
2925 GetExpandedInteger(LHS, LHSL, LHSH);
2926 GetExpandedInteger(RHS, RHSL, RHSH);
2927 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2928 SDValue LoOps[2] = { LHSL, RHSL };
2929 SDValue HiOps[3] = { LHSH, RHSH };
2930
2931 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2932 HiOps[2] = Lo.getValue(1);
2933 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
2934
2935 Ovf = Hi.getValue(1);
2936 } else {
2937 // Expand the result by simply replacing it with the equivalent
2938 // non-overflow-checking operation.
2939 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2940 SplitInteger(Sum, Lo, Hi);
2941
2942 // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2943 // overflows iff a - b > a.
2944 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2945 }
2946
2947 // Legalized the flag result - switch anything that used the old flag to
2948 // use the new one.
2949 ReplaceValueWith(SDValue(N, 1), Ovf);
2950 }
2951
ExpandIntRes_ADDSUBCARRY(SDNode * N,SDValue & Lo,SDValue & Hi)2952 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2953 SDValue &Lo, SDValue &Hi) {
2954 // Expand the subcomponents.
2955 SDValue LHSL, LHSH, RHSL, RHSH;
2956 SDLoc dl(N);
2957 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2958 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2959 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2960 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2961 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2962
2963 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2964 HiOps[2] = Lo.getValue(1);
2965 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2966
2967 // Legalized the flag result - switch anything that used the old flag to
2968 // use the new one.
2969 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2970 }
2971
ExpandIntRes_SADDSUBO_CARRY(SDNode * N,SDValue & Lo,SDValue & Hi)2972 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
2973 SDValue &Lo, SDValue &Hi) {
2974 // Expand the subcomponents.
2975 SDValue LHSL, LHSH, RHSL, RHSH;
2976 SDLoc dl(N);
2977 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2978 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2979 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2980
2981 // We need to use an unsigned carry op for the lo part.
2982 unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY
2983 : ISD::SUBCARRY;
2984 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
2985 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
2986
2987 // Legalized the flag result - switch anything that used the old flag to
2988 // use the new one.
2989 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2990 }
2991
ExpandIntRes_ANY_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)2992 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2993 SDValue &Lo, SDValue &Hi) {
2994 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2995 SDLoc dl(N);
2996 SDValue Op = N->getOperand(0);
2997 if (Op.getValueType().bitsLE(NVT)) {
2998 // The low part is any extension of the input (which degenerates to a copy).
2999 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
3000 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
3001 } else {
3002 // For example, extension of an i48 to an i64. The operand type necessarily
3003 // promotes to the result type, so will end up being expanded too.
3004 assert(getTypeAction(Op.getValueType()) ==
3005 TargetLowering::TypePromoteInteger &&
3006 "Only know how to promote this result!");
3007 SDValue Res = GetPromotedInteger(Op);
3008 assert(Res.getValueType() == N->getValueType(0) &&
3009 "Operand over promoted?");
3010 // Split the promoted operand. This will simplify when it is expanded.
3011 SplitInteger(Res, Lo, Hi);
3012 }
3013 }
3014
ExpandIntRes_AssertSext(SDNode * N,SDValue & Lo,SDValue & Hi)3015 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3016 SDValue &Lo, SDValue &Hi) {
3017 SDLoc dl(N);
3018 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3019 EVT NVT = Lo.getValueType();
3020 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3021 unsigned NVTBits = NVT.getSizeInBits();
3022 unsigned EVTBits = EVT.getSizeInBits();
3023
3024 if (NVTBits < EVTBits) {
3025 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
3026 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3027 EVTBits - NVTBits)));
3028 } else {
3029 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
3030 // The high part replicates the sign bit of Lo, make it explicit.
3031 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3032 DAG.getConstant(NVTBits - 1, dl,
3033 TLI.getPointerTy(DAG.getDataLayout())));
3034 }
3035 }
3036
ExpandIntRes_AssertZext(SDNode * N,SDValue & Lo,SDValue & Hi)3037 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
3038 SDValue &Lo, SDValue &Hi) {
3039 SDLoc dl(N);
3040 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3041 EVT NVT = Lo.getValueType();
3042 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3043 unsigned NVTBits = NVT.getSizeInBits();
3044 unsigned EVTBits = EVT.getSizeInBits();
3045
3046 if (NVTBits < EVTBits) {
3047 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
3048 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3049 EVTBits - NVTBits)));
3050 } else {
3051 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
3052 // The high part must be zero, make it explicit.
3053 Hi = DAG.getConstant(0, dl, NVT);
3054 }
3055 }
3056
ExpandIntRes_BITREVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)3057 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
3058 SDValue &Lo, SDValue &Hi) {
3059 SDLoc dl(N);
3060 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3061 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
3062 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3063 }
3064
ExpandIntRes_BSWAP(SDNode * N,SDValue & Lo,SDValue & Hi)3065 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
3066 SDValue &Lo, SDValue &Hi) {
3067 SDLoc dl(N);
3068 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3069 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
3070 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3071 }
3072
ExpandIntRes_PARITY(SDNode * N,SDValue & Lo,SDValue & Hi)3073 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
3074 SDValue &Hi) {
3075 SDLoc dl(N);
3076 // parity(HiLo) -> parity(Lo^Hi)
3077 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3078 EVT NVT = Lo.getValueType();
3079 Lo =
3080 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
3081 Hi = DAG.getConstant(0, dl, NVT);
3082 }
3083
ExpandIntRes_Constant(SDNode * N,SDValue & Lo,SDValue & Hi)3084 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
3085 SDValue &Lo, SDValue &Hi) {
3086 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3087 unsigned NBitWidth = NVT.getSizeInBits();
3088 auto Constant = cast<ConstantSDNode>(N);
3089 const APInt &Cst = Constant->getAPIntValue();
3090 bool IsTarget = Constant->isTargetOpcode();
3091 bool IsOpaque = Constant->isOpaque();
3092 SDLoc dl(N);
3093 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
3094 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
3095 IsOpaque);
3096 }
3097
ExpandIntRes_ABS(SDNode * N,SDValue & Lo,SDValue & Hi)3098 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
3099 SDLoc dl(N);
3100
3101 SDValue N0 = N->getOperand(0);
3102 GetExpandedInteger(N0, Lo, Hi);
3103 EVT NVT = Lo.getValueType();
3104
3105 // If we have ADDCARRY, use the expanded form of the sra+add+xor sequence we
3106 // use in LegalizeDAG. The ADD part of the expansion is based on
3107 // ExpandIntRes_ADDSUB which also uses ADDCARRY/UADDO after checking that
3108 // ADDCARRY is LegalOrCustom. Each of the pieces here can be further expanded
3109 // if needed. Shift expansion has a special case for filling with sign bits
3110 // so that we will only end up with one SRA.
3111 bool HasAddCarry = TLI.isOperationLegalOrCustom(
3112 ISD::ADDCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3113 if (HasAddCarry) {
3114 EVT ShiftAmtTy = getShiftAmountTyForConstant(NVT, TLI, DAG);
3115 SDValue Sign =
3116 DAG.getNode(ISD::SRA, dl, NVT, Hi,
3117 DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy));
3118 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3119 Lo = DAG.getNode(ISD::UADDO, dl, VTList, Lo, Sign);
3120 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
3121 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
3122 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
3123 return;
3124 }
3125
3126 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
3127 EVT VT = N->getValueType(0);
3128 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
3129 DAG.getConstant(0, dl, VT), N0);
3130 SDValue NegLo, NegHi;
3131 SplitInteger(Neg, NegLo, NegHi);
3132
3133 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
3134 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
3135 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
3136 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
3137 }
3138
ExpandIntRes_CTLZ(SDNode * N,SDValue & Lo,SDValue & Hi)3139 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
3140 SDValue &Lo, SDValue &Hi) {
3141 SDLoc dl(N);
3142 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
3143 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3144 EVT NVT = Lo.getValueType();
3145
3146 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3147 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3148
3149 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
3150 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
3151
3152 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
3153 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
3154 DAG.getConstant(NVT.getSizeInBits(), dl,
3155 NVT)));
3156 Hi = DAG.getConstant(0, dl, NVT);
3157 }
3158
ExpandIntRes_CTPOP(SDNode * N,SDValue & Lo,SDValue & Hi)3159 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
3160 SDValue &Lo, SDValue &Hi) {
3161 SDLoc dl(N);
3162 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
3163 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3164 EVT NVT = Lo.getValueType();
3165 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
3166 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
3167 Hi = DAG.getConstant(0, dl, NVT);
3168 }
3169
ExpandIntRes_CTTZ(SDNode * N,SDValue & Lo,SDValue & Hi)3170 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
3171 SDValue &Lo, SDValue &Hi) {
3172 SDLoc dl(N);
3173 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
3174 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3175 EVT NVT = Lo.getValueType();
3176
3177 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3178 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3179
3180 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
3181 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
3182
3183 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
3184 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
3185 DAG.getConstant(NVT.getSizeInBits(), dl,
3186 NVT)));
3187 Hi = DAG.getConstant(0, dl, NVT);
3188 }
3189
ExpandIntRes_FLT_ROUNDS(SDNode * N,SDValue & Lo,SDValue & Hi)3190 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
3191 SDValue &Hi) {
3192 SDLoc dl(N);
3193 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3194 unsigned NBitWidth = NVT.getSizeInBits();
3195
3196 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3197 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0));
3198 SDValue Chain = Lo.getValue(1);
3199 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
3200 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3201 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
3202
3203 // Legalize the chain result - switch anything that used the old chain to
3204 // use the new one.
3205 ReplaceValueWith(SDValue(N, 1), Chain);
3206 }
3207
ExpandIntRes_FP_TO_SINT(SDNode * N,SDValue & Lo,SDValue & Hi)3208 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
3209 SDValue &Hi) {
3210 SDLoc dl(N);
3211 EVT VT = N->getValueType(0);
3212
3213 bool IsStrict = N->isStrictFPOpcode();
3214 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3215 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3216 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3217 Op = GetPromotedFloat(Op);
3218
3219 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3220 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3221 Op = GetSoftPromotedHalf(Op);
3222 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3223 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
3224 SplitInteger(Op, Lo, Hi);
3225 return;
3226 }
3227
3228 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
3229 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
3230 TargetLowering::MakeLibCallOptions CallOptions;
3231 CallOptions.setSExt(true);
3232 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3233 CallOptions, dl, Chain);
3234 SplitInteger(Tmp.first, Lo, Hi);
3235
3236 if (IsStrict)
3237 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3238 }
3239
ExpandIntRes_FP_TO_UINT(SDNode * N,SDValue & Lo,SDValue & Hi)3240 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
3241 SDValue &Hi) {
3242 SDLoc dl(N);
3243 EVT VT = N->getValueType(0);
3244
3245 bool IsStrict = N->isStrictFPOpcode();
3246 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3247 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3248 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3249 Op = GetPromotedFloat(Op);
3250
3251 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3252 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3253 Op = GetSoftPromotedHalf(Op);
3254 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3255 Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op);
3256 SplitInteger(Op, Lo, Hi);
3257 return;
3258 }
3259
3260 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
3261 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
3262 TargetLowering::MakeLibCallOptions CallOptions;
3263 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3264 CallOptions, dl, Chain);
3265 SplitInteger(Tmp.first, Lo, Hi);
3266
3267 if (IsStrict)
3268 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3269 }
3270
ExpandIntRes_FP_TO_XINT_SAT(SDNode * N,SDValue & Lo,SDValue & Hi)3271 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3272 SDValue &Hi) {
3273 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3274 SplitInteger(Res, Lo, Hi);
3275 }
3276
ExpandIntRes_LLROUND_LLRINT(SDNode * N,SDValue & Lo,SDValue & Hi)3277 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
3278 SDValue &Hi) {
3279 SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
3280
3281 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
3282 "Input type needs to be promoted!");
3283
3284 EVT VT = Op.getValueType();
3285
3286 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3287 if (N->getOpcode() == ISD::LLROUND ||
3288 N->getOpcode() == ISD::STRICT_LLROUND) {
3289 if (VT == MVT::f32)
3290 LC = RTLIB::LLROUND_F32;
3291 else if (VT == MVT::f64)
3292 LC = RTLIB::LLROUND_F64;
3293 else if (VT == MVT::f80)
3294 LC = RTLIB::LLROUND_F80;
3295 else if (VT == MVT::f128)
3296 LC = RTLIB::LLROUND_F128;
3297 else if (VT == MVT::ppcf128)
3298 LC = RTLIB::LLROUND_PPCF128;
3299 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
3300 } else if (N->getOpcode() == ISD::LLRINT ||
3301 N->getOpcode() == ISD::STRICT_LLRINT) {
3302 if (VT == MVT::f32)
3303 LC = RTLIB::LLRINT_F32;
3304 else if (VT == MVT::f64)
3305 LC = RTLIB::LLRINT_F64;
3306 else if (VT == MVT::f80)
3307 LC = RTLIB::LLRINT_F80;
3308 else if (VT == MVT::f128)
3309 LC = RTLIB::LLRINT_F128;
3310 else if (VT == MVT::ppcf128)
3311 LC = RTLIB::LLRINT_PPCF128;
3312 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
3313 } else
3314 llvm_unreachable("Unexpected opcode!");
3315
3316 SDLoc dl(N);
3317 EVT RetVT = N->getValueType(0);
3318 SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();
3319
3320 TargetLowering::MakeLibCallOptions CallOptions;
3321 CallOptions.setSExt(true);
3322 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3323 Op, CallOptions, dl,
3324 Chain);
3325 SplitInteger(Tmp.first, Lo, Hi);
3326
3327 if (N->isStrictFPOpcode())
3328 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3329 }
3330
ExpandIntRes_LOAD(LoadSDNode * N,SDValue & Lo,SDValue & Hi)3331 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
3332 SDValue &Lo, SDValue &Hi) {
3333 if (N->isAtomic()) {
3334 // It's typical to have larger CAS than atomic load instructions.
3335 SDLoc dl(N);
3336 EVT VT = N->getMemoryVT();
3337 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3338 SDValue Zero = DAG.getConstant(0, dl, VT);
3339 SDValue Swap = DAG.getAtomicCmpSwap(
3340 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
3341 VT, VTs, N->getOperand(0),
3342 N->getOperand(1), Zero, Zero, N->getMemOperand());
3343 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3344 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3345 return;
3346 }
3347
3348 if (ISD::isNormalLoad(N)) {
3349 ExpandRes_NormalLoad(N, Lo, Hi);
3350 return;
3351 }
3352
3353 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
3354
3355 EVT VT = N->getValueType(0);
3356 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3357 SDValue Ch = N->getChain();
3358 SDValue Ptr = N->getBasePtr();
3359 ISD::LoadExtType ExtType = N->getExtensionType();
3360 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3361 AAMDNodes AAInfo = N->getAAInfo();
3362 SDLoc dl(N);
3363
3364 assert(NVT.isByteSized() && "Expanded type not byte sized!");
3365
3366 if (N->getMemoryVT().bitsLE(NVT)) {
3367 EVT MemVT = N->getMemoryVT();
3368
3369 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
3370 N->getOriginalAlign(), MMOFlags, AAInfo);
3371
3372 // Remember the chain.
3373 Ch = Lo.getValue(1);
3374
3375 if (ExtType == ISD::SEXTLOAD) {
3376 // The high part is obtained by SRA'ing all but one of the bits of the
3377 // lo part.
3378 unsigned LoSize = Lo.getValueSizeInBits();
3379 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3380 DAG.getConstant(LoSize - 1, dl,
3381 TLI.getPointerTy(DAG.getDataLayout())));
3382 } else if (ExtType == ISD::ZEXTLOAD) {
3383 // The high part is just a zero.
3384 Hi = DAG.getConstant(0, dl, NVT);
3385 } else {
3386 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
3387 // The high part is undefined.
3388 Hi = DAG.getUNDEF(NVT);
3389 }
3390 } else if (DAG.getDataLayout().isLittleEndian()) {
3391 // Little-endian - low bits are at low addresses.
3392 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
3393 N->getOriginalAlign(), MMOFlags, AAInfo);
3394
3395 unsigned ExcessBits =
3396 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3397 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3398
3399 // Increment the pointer to the other half.
3400 unsigned IncrementSize = NVT.getSizeInBits()/8;
3401 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3402 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3403 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3404 N->getOriginalAlign(), MMOFlags, AAInfo);
3405
3406 // Build a factor node to remember that this load is independent of the
3407 // other one.
3408 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3409 Hi.getValue(1));
3410 } else {
3411 // Big-endian - high bits are at low addresses. Favor aligned loads at
3412 // the cost of some bit-fiddling.
3413 EVT MemVT = N->getMemoryVT();
3414 unsigned EBytes = MemVT.getStoreSize();
3415 unsigned IncrementSize = NVT.getSizeInBits()/8;
3416 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3417
3418 // Load both the high bits and maybe some of the low bits.
3419 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3420 EVT::getIntegerVT(*DAG.getContext(),
3421 MemVT.getSizeInBits() - ExcessBits),
3422 N->getOriginalAlign(), MMOFlags, AAInfo);
3423
3424 // Increment the pointer to the other half.
3425 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3426 // Load the rest of the low bits.
3427 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3428 N->getPointerInfo().getWithOffset(IncrementSize),
3429 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3430 N->getOriginalAlign(), MMOFlags, AAInfo);
3431
3432 // Build a factor node to remember that this load is independent of the
3433 // other one.
3434 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3435 Hi.getValue(1));
3436
3437 if (ExcessBits < NVT.getSizeInBits()) {
3438 // Transfer low bits from the bottom of Hi to the top of Lo.
3439 Lo = DAG.getNode(
3440 ISD::OR, dl, NVT, Lo,
3441 DAG.getNode(ISD::SHL, dl, NVT, Hi,
3442 DAG.getConstant(ExcessBits, dl,
3443 TLI.getPointerTy(DAG.getDataLayout()))));
3444 // Move high bits to the right position in Hi.
3445 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
3446 Hi,
3447 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3448 TLI.getPointerTy(DAG.getDataLayout())));
3449 }
3450 }
3451
3452 // Legalize the chain result - switch anything that used the old chain to
3453 // use the new one.
3454 ReplaceValueWith(SDValue(N, 1), Ch);
3455 }
3456
ExpandIntRes_Logical(SDNode * N,SDValue & Lo,SDValue & Hi)3457 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
3458 SDValue &Lo, SDValue &Hi) {
3459 SDLoc dl(N);
3460 SDValue LL, LH, RL, RH;
3461 GetExpandedInteger(N->getOperand(0), LL, LH);
3462 GetExpandedInteger(N->getOperand(1), RL, RH);
3463 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
3464 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3465 }
3466
ExpandIntRes_MUL(SDNode * N,SDValue & Lo,SDValue & Hi)3467 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
3468 SDValue &Lo, SDValue &Hi) {
3469 EVT VT = N->getValueType(0);
3470 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3471 SDLoc dl(N);
3472
3473 SDValue LL, LH, RL, RH;
3474 GetExpandedInteger(N->getOperand(0), LL, LH);
3475 GetExpandedInteger(N->getOperand(1), RL, RH);
3476
3477 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
3478 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3479 LL, LH, RL, RH))
3480 return;
3481
3482 // If nothing else, we can make a libcall.
3483 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3484 if (VT == MVT::i16)
3485 LC = RTLIB::MUL_I16;
3486 else if (VT == MVT::i32)
3487 LC = RTLIB::MUL_I32;
3488 else if (VT == MVT::i64)
3489 LC = RTLIB::MUL_I64;
3490 else if (VT == MVT::i128)
3491 LC = RTLIB::MUL_I128;
3492
3493 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
3494 // We'll expand the multiplication by brute force because we have no other
3495 // options. This is a trivially-generalized version of the code from
3496 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
3497 // 4.3.1).
3498 unsigned Bits = NVT.getSizeInBits();
3499 unsigned HalfBits = Bits >> 1;
3500 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
3501 NVT);
3502 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
3503 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
3504
3505 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
3506 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
3507
3508 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3509 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
3510 // The type from TLI is too small to fit the shift amount we want.
3511 // Override it with i32. The shift will have to be legalized.
3512 ShiftAmtTy = MVT::i32;
3513 }
3514 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
3515 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
3516 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
3517 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
3518
3519 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
3520 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
3521 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
3522 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
3523
3524 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
3525 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
3526 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
3527
3528 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
3529 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
3530 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
3531 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
3532 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
3533
3534 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
3535 DAG.getNode(ISD::ADD, dl, NVT,
3536 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
3537 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
3538 return;
3539 }
3540
3541 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3542 TargetLowering::MakeLibCallOptions CallOptions;
3543 CallOptions.setSExt(true);
3544 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3545 Lo, Hi);
3546 }
3547
ExpandIntRes_READCYCLECOUNTER(SDNode * N,SDValue & Lo,SDValue & Hi)3548 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
3549 SDValue &Hi) {
3550 SDLoc DL(N);
3551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3552 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
3553 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
3554 Lo = R.getValue(0);
3555 Hi = R.getValue(1);
3556 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3557 }
3558
ExpandIntRes_ADDSUBSAT(SDNode * N,SDValue & Lo,SDValue & Hi)3559 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
3560 SDValue &Hi) {
3561 SDValue Result = TLI.expandAddSubSat(N, DAG);
3562 SplitInteger(Result, Lo, Hi);
3563 }
3564
ExpandIntRes_SHLSAT(SDNode * N,SDValue & Lo,SDValue & Hi)3565 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
3566 SDValue &Hi) {
3567 SDValue Result = TLI.expandShlSat(N, DAG);
3568 SplitInteger(Result, Lo, Hi);
3569 }
3570
3571 /// This performs an expansion of the integer result for a fixed point
3572 /// multiplication. The default expansion performs rounding down towards
3573 /// negative infinity, though targets that do care about rounding should specify
3574 /// a target hook for rounding and provide their own expansion or lowering of
3575 /// fixed point multiplication to be consistent with rounding.
ExpandIntRes_MULFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3576 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
3577 SDValue &Hi) {
3578 SDLoc dl(N);
3579 EVT VT = N->getValueType(0);
3580 unsigned VTSize = VT.getScalarSizeInBits();
3581 SDValue LHS = N->getOperand(0);
3582 SDValue RHS = N->getOperand(1);
3583 uint64_t Scale = N->getConstantOperandVal(2);
3584 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
3585 N->getOpcode() == ISD::UMULFIXSAT);
3586 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
3587 N->getOpcode() == ISD::SMULFIXSAT);
3588
3589 // Handle special case when scale is equal to zero.
3590 if (!Scale) {
3591 SDValue Result;
3592 if (!Saturating) {
3593 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3594 } else {
3595 EVT BoolVT = getSetCCResultType(VT);
3596 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
3597 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
3598 SDValue Product = Result.getValue(0);
3599 SDValue Overflow = Result.getValue(1);
3600 if (Signed) {
3601 APInt MinVal = APInt::getSignedMinValue(VTSize);
3602 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
3603 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
3604 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3605 SDValue Zero = DAG.getConstant(0, dl, VT);
3606 // Xor the inputs, if resulting sign bit is 0 the product will be
3607 // positive, else negative.
3608 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
3609 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
3610 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
3611 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
3612 } else {
3613 // For unsigned multiplication, we only need to check the max since we
3614 // can't really overflow towards zero.
3615 APInt MaxVal = APInt::getMaxValue(VTSize);
3616 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3617 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
3618 }
3619 }
3620 SplitInteger(Result, Lo, Hi);
3621 return;
3622 }
3623
3624 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
3625 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
3626 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
3627
3628 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3629 SDValue LL, LH, RL, RH;
3630 GetExpandedInteger(LHS, LL, LH);
3631 GetExpandedInteger(RHS, RL, RH);
3632 SmallVector<SDValue, 4> Result;
3633
3634 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
3635 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
3636 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3637 LL, LH, RL, RH)) {
3638 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
3639 return;
3640 }
3641
3642 unsigned NVTSize = NVT.getScalarSizeInBits();
3643 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
3644 "the size of the current value type");
3645 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3646
3647 // After getting the multiplication result in 4 parts, we need to perform a
3648 // shift right by the amount of the scale to get the result in that scale.
3649 //
3650 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
3651 // 128 bits that are cut into 4 32-bit parts:
3652 //
3653 // HH HL LH LL
3654 // |---32---|---32---|---32---|---32---|
3655 // 128 96 64 32 0
3656 //
3657 // |------VTSize-----|
3658 //
3659 // |NVTSize-|
3660 //
3661 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
3662 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
3663 // amount and get Lo and Hi using two funnel shifts. Or for the special case
3664 // when Scale is a multiple of NVTSize we can just pick the result without
3665 // shifting.
3666 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
3667 if (Scale % NVTSize) {
3668 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
3669 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
3670 ShiftAmount);
3671 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
3672 ShiftAmount);
3673 } else {
3674 Lo = Result[Part0];
3675 Hi = Result[Part0 + 1];
3676 }
3677
3678 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
3679 if (!Saturating)
3680 return;
3681
3682 // Can not overflow when there is no integer part.
3683 if (Scale == VTSize)
3684 return;
3685
3686 // To handle saturation we must check for overflow in the multiplication.
3687 //
3688 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
3689 // aren't all zeroes.
3690 //
3691 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
3692 // aren't all ones or all zeroes.
3693 //
3694 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
3695 // highest bit of HH determines saturation direction in the event of signed
3696 // saturation.
3697
3698 SDValue ResultHL = Result[2];
3699 SDValue ResultHH = Result[3];
3700
3701 SDValue SatMax, SatMin;
3702 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
3703 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3704 EVT BoolNVT = getSetCCResultType(NVT);
3705
3706 if (!Signed) {
3707 if (Scale < NVTSize) {
3708 // Overflow happened if ((HH | (HL >> Scale)) != 0).
3709 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3710 DAG.getConstant(Scale, dl, ShiftTy));
3711 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3712 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3713 } else if (Scale == NVTSize) {
3714 // Overflow happened if (HH != 0).
3715 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3716 } else if (Scale < VTSize) {
3717 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
3718 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3719 DAG.getConstant(Scale - NVTSize, dl,
3720 ShiftTy));
3721 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3722 } else
3723 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
3724 "(and saturation can't happen with Scale==VTSize).");
3725
3726 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3727 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3728 return;
3729 }
3730
3731 if (Scale < NVTSize) {
3732 // The number of overflow bits we can check are VTSize - Scale + 1 (we
3733 // include the sign bit). If these top bits are > 0, then we overflowed past
3734 // the max value. If these top bits are < -1, then we overflowed past the
3735 // min value. Otherwise, we did not overflow.
3736 unsigned OverflowBits = VTSize - Scale + 1;
3737 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3738 "Extent of overflow bits must start within HL");
3739 SDValue HLHiMask = DAG.getConstant(
3740 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3741 SDValue HLLoMask = DAG.getConstant(
3742 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3743 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3744 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3745 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3746 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3747 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3748 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3749 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3750 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3751 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3752 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3753 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3754 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3755 } else if (Scale == NVTSize) {
3756 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3757 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3758 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3759 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3760 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3761 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3762 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3763 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3764 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3765 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3766 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3767 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3768 } else if (Scale < VTSize) {
3769 // This is similar to the case when we saturate if Scale < NVTSize, but we
3770 // only need to check HH.
3771 unsigned OverflowBits = VTSize - Scale + 1;
3772 SDValue HHHiMask = DAG.getConstant(
3773 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3774 SDValue HHLoMask = DAG.getConstant(
3775 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3776 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3777 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3778 } else
3779 llvm_unreachable("Illegal scale for signed fixed point mul.");
3780
3781 // Saturate to signed maximum.
3782 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3783 APInt MaxLo = APInt::getAllOnes(NVTSize);
3784 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3785 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3786 // Saturate to signed minimum.
3787 APInt MinHi = APInt::getSignedMinValue(NVTSize);
3788 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3789 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3790 }
3791
ExpandIntRes_DIVFIX(SDNode * N,SDValue & Lo,SDValue & Hi)3792 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
3793 SDValue &Hi) {
3794 SDLoc dl(N);
3795 // Try expanding in the existing type first.
3796 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
3797 N->getOperand(1),
3798 N->getConstantOperandVal(2), DAG);
3799
3800 if (!Res)
3801 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
3802 N->getConstantOperandVal(2), TLI, DAG);
3803 SplitInteger(Res, Lo, Hi);
3804 }
3805
ExpandIntRes_SADDSUBO(SDNode * Node,SDValue & Lo,SDValue & Hi)3806 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3807 SDValue &Lo, SDValue &Hi) {
3808 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
3809 "Node has unexpected Opcode");
3810 SDValue LHS = Node->getOperand(0);
3811 SDValue RHS = Node->getOperand(1);
3812 SDLoc dl(Node);
3813
3814 SDValue Ovf;
3815
3816 bool IsAdd = Node->getOpcode() == ISD::SADDO;
3817 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
3818
3819 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3820 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3821
3822 if (HasCarryOp) {
3823 // Expand the subcomponents.
3824 SDValue LHSL, LHSH, RHSL, RHSH;
3825 GetExpandedInteger(LHS, LHSL, LHSH);
3826 GetExpandedInteger(RHS, RHSL, RHSH);
3827 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
3828
3829 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
3830 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3831
3832 Ovf = Hi.getValue(1);
3833 } else {
3834 // Expand the result by simply replacing it with the equivalent
3835 // non-overflow-checking operation.
3836 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3837 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3838 LHS, RHS);
3839 SplitInteger(Sum, Lo, Hi);
3840
3841 // Compute the overflow.
3842 //
3843 // LHSSign -> LHS < 0
3844 // RHSSign -> RHS < 0
3845 // SumSign -> Sum < 0
3846 //
3847 // Add:
3848 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3849 // Sub:
3850 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3851 //
3852 // To get better codegen we can rewrite this by doing bitwise math on
3853 // the integers and extract the final sign bit at the end. So the
3854 // above becomes:
3855 //
3856 // Add:
3857 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
3858 // Sub:
3859 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
3860 //
3861 // NOTE: This is different than the expansion we do in expandSADDSUBO
3862 // because it is more costly to determine the RHS is > 0 for SSUBO with the
3863 // integers split.
3864 EVT VT = LHS.getValueType();
3865 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
3866 if (IsAdd)
3867 SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
3868
3869 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
3870 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
3871 EVT OType = Node->getValueType(1);
3872 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
3873 }
3874
3875 // Use the calculated overflow everywhere.
3876 ReplaceValueWith(SDValue(Node, 1), Ovf);
3877 }
3878
ExpandIntRes_SDIV(SDNode * N,SDValue & Lo,SDValue & Hi)3879 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3880 SDValue &Lo, SDValue &Hi) {
3881 EVT VT = N->getValueType(0);
3882 SDLoc dl(N);
3883 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3884
3885 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3886 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3887 SplitInteger(Res.getValue(0), Lo, Hi);
3888 return;
3889 }
3890
3891 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3892 if (VT == MVT::i16)
3893 LC = RTLIB::SDIV_I16;
3894 else if (VT == MVT::i32)
3895 LC = RTLIB::SDIV_I32;
3896 else if (VT == MVT::i64)
3897 LC = RTLIB::SDIV_I64;
3898 else if (VT == MVT::i128)
3899 LC = RTLIB::SDIV_I128;
3900 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3901
3902 TargetLowering::MakeLibCallOptions CallOptions;
3903 CallOptions.setSExt(true);
3904 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3905 }
3906
ExpandIntRes_Shift(SDNode * N,SDValue & Lo,SDValue & Hi)3907 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3908 SDValue &Lo, SDValue &Hi) {
3909 EVT VT = N->getValueType(0);
3910 SDLoc dl(N);
3911
3912 // If we can emit an efficient shift operation, do so now. Check to see if
3913 // the RHS is a constant.
3914 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3915 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3916
3917 // If we can determine that the high bit of the shift is zero or one, even if
3918 // the low bits are variable, emit this shift in an optimized form.
3919 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3920 return;
3921
3922 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3923 unsigned PartsOpc;
3924 if (N->getOpcode() == ISD::SHL) {
3925 PartsOpc = ISD::SHL_PARTS;
3926 } else if (N->getOpcode() == ISD::SRL) {
3927 PartsOpc = ISD::SRL_PARTS;
3928 } else {
3929 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3930 PartsOpc = ISD::SRA_PARTS;
3931 }
3932
3933 // Next check to see if the target supports this SHL_PARTS operation or if it
3934 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3935 // size, but create a libcall instead.
3936 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3937 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3938 const bool LegalOrCustom =
3939 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3940 Action == TargetLowering::Custom;
3941
3942 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3943 // Expand the subcomponents.
3944 SDValue LHSL, LHSH;
3945 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3946 EVT VT = LHSL.getValueType();
3947
3948 // If the shift amount operand is coming from a vector legalization it may
3949 // have an illegal type. Fix that first by casting the operand, otherwise
3950 // the new SHL_PARTS operation would need further legalization.
3951 SDValue ShiftOp = N->getOperand(1);
3952 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3953 assert(ShiftTy.getScalarSizeInBits() >=
3954 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3955 "ShiftAmountTy is too small to cover the range of this type!");
3956 if (ShiftOp.getValueType() != ShiftTy)
3957 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3958
3959 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3960 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3961 Hi = Lo.getValue(1);
3962 return;
3963 }
3964
3965 // Otherwise, emit a libcall.
3966 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3967 bool isSigned;
3968 if (N->getOpcode() == ISD::SHL) {
3969 isSigned = false; /*sign irrelevant*/
3970 if (VT == MVT::i16)
3971 LC = RTLIB::SHL_I16;
3972 else if (VT == MVT::i32)
3973 LC = RTLIB::SHL_I32;
3974 else if (VT == MVT::i64)
3975 LC = RTLIB::SHL_I64;
3976 else if (VT == MVT::i128)
3977 LC = RTLIB::SHL_I128;
3978 } else if (N->getOpcode() == ISD::SRL) {
3979 isSigned = false;
3980 if (VT == MVT::i16)
3981 LC = RTLIB::SRL_I16;
3982 else if (VT == MVT::i32)
3983 LC = RTLIB::SRL_I32;
3984 else if (VT == MVT::i64)
3985 LC = RTLIB::SRL_I64;
3986 else if (VT == MVT::i128)
3987 LC = RTLIB::SRL_I128;
3988 } else {
3989 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3990 isSigned = true;
3991 if (VT == MVT::i16)
3992 LC = RTLIB::SRA_I16;
3993 else if (VT == MVT::i32)
3994 LC = RTLIB::SRA_I32;
3995 else if (VT == MVT::i64)
3996 LC = RTLIB::SRA_I64;
3997 else if (VT == MVT::i128)
3998 LC = RTLIB::SRA_I128;
3999 }
4000
4001 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
4002 EVT ShAmtTy =
4003 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());
4004 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy);
4005 SDValue Ops[2] = {N->getOperand(0), ShAmt};
4006 TargetLowering::MakeLibCallOptions CallOptions;
4007 CallOptions.setSExt(isSigned);
4008 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4009 return;
4010 }
4011
4012 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
4013 llvm_unreachable("Unsupported shift!");
4014 }
4015
ExpandIntRes_SIGN_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)4016 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
4017 SDValue &Lo, SDValue &Hi) {
4018 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4019 SDLoc dl(N);
4020 SDValue Op = N->getOperand(0);
4021 if (Op.getValueType().bitsLE(NVT)) {
4022 // The low part is sign extension of the input (degenerates to a copy).
4023 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
4024 // The high part is obtained by SRA'ing all but one of the bits of low part.
4025 unsigned LoSize = NVT.getSizeInBits();
4026 Hi = DAG.getNode(
4027 ISD::SRA, dl, NVT, Lo,
4028 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
4029 } else {
4030 // For example, extension of an i48 to an i64. The operand type necessarily
4031 // promotes to the result type, so will end up being expanded too.
4032 assert(getTypeAction(Op.getValueType()) ==
4033 TargetLowering::TypePromoteInteger &&
4034 "Only know how to promote this result!");
4035 SDValue Res = GetPromotedInteger(Op);
4036 assert(Res.getValueType() == N->getValueType(0) &&
4037 "Operand over promoted?");
4038 // Split the promoted operand. This will simplify when it is expanded.
4039 SplitInteger(Res, Lo, Hi);
4040 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4041 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4042 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
4043 ExcessBits)));
4044 }
4045 }
4046
4047 void DAGTypeLegalizer::
ExpandIntRes_SIGN_EXTEND_INREG(SDNode * N,SDValue & Lo,SDValue & Hi)4048 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4049 SDLoc dl(N);
4050 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4051 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
4052
4053 if (EVT.bitsLE(Lo.getValueType())) {
4054 // sext_inreg the low part if needed.
4055 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
4056 N->getOperand(1));
4057
4058 // The high part gets the sign extension from the lo-part. This handles
4059 // things like sextinreg V:i64 from i8.
4060 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
4061 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
4062 TLI.getPointerTy(DAG.getDataLayout())));
4063 } else {
4064 // For example, extension of an i48 to an i64. Leave the low part alone,
4065 // sext_inreg the high part.
4066 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
4067 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4068 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
4069 ExcessBits)));
4070 }
4071 }
4072
ExpandIntRes_SREM(SDNode * N,SDValue & Lo,SDValue & Hi)4073 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
4074 SDValue &Lo, SDValue &Hi) {
4075 EVT VT = N->getValueType(0);
4076 SDLoc dl(N);
4077 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4078
4079 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
4080 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4081 SplitInteger(Res.getValue(1), Lo, Hi);
4082 return;
4083 }
4084
4085 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4086 if (VT == MVT::i16)
4087 LC = RTLIB::SREM_I16;
4088 else if (VT == MVT::i32)
4089 LC = RTLIB::SREM_I32;
4090 else if (VT == MVT::i64)
4091 LC = RTLIB::SREM_I64;
4092 else if (VT == MVT::i128)
4093 LC = RTLIB::SREM_I128;
4094 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
4095
4096 TargetLowering::MakeLibCallOptions CallOptions;
4097 CallOptions.setSExt(true);
4098 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4099 }
4100
ExpandIntRes_TRUNCATE(SDNode * N,SDValue & Lo,SDValue & Hi)4101 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
4102 SDValue &Lo, SDValue &Hi) {
4103 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4104 SDLoc dl(N);
4105 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
4106 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
4107 N->getOperand(0),
4108 DAG.getConstant(NVT.getSizeInBits(), dl,
4109 TLI.getPointerTy(DAG.getDataLayout())));
4110 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
4111 }
4112
ExpandIntRes_XMULO(SDNode * N,SDValue & Lo,SDValue & Hi)4113 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
4114 SDValue &Lo, SDValue &Hi) {
4115 EVT VT = N->getValueType(0);
4116 SDLoc dl(N);
4117
4118 if (N->getOpcode() == ISD::UMULO) {
4119 // This section expands the operation into the following sequence of
4120 // instructions. `iNh` here refers to a type which has half the bit width of
4121 // the type the original operation operated on.
4122 //
4123 // %0 = %LHS.HI != 0 && %RHS.HI != 0
4124 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
4125 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
4126 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
4127 // %4 = add iNh %1.0, %2.0 as iN
4128 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
4129 //
4130 // %lo = %3.LO
4131 // %hi = %5.0
4132 // %ovf = %0 || %1.1 || %2.1 || %5.1
4133 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
4134 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4135 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4136 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4137 EVT HalfVT = LHSLow.getValueType();
4138 EVT BitVT = N->getValueType(1);
4139 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
4140
4141 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
4142 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
4143 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
4144 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
4145
4146 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
4147 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
4148
4149 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
4150 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
4151
4152 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
4153
4154 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
4155 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
4156 // operation recursively legalized?).
4157 //
4158 // Many backends understand this pattern and will convert into LOHI
4159 // themselves, if applicable.
4160 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
4161 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
4162 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
4163 SplitInteger(Three, Lo, Hi);
4164
4165 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
4166 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
4167 ReplaceValueWith(SDValue(N, 1), Overflow);
4168 return;
4169 }
4170
4171 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
4172 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
4173 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
4174
4175 // Replace this with a libcall that will check overflow.
4176 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4177 if (VT == MVT::i32)
4178 LC = RTLIB::MULO_I32;
4179 else if (VT == MVT::i64)
4180 LC = RTLIB::MULO_I64;
4181 else if (VT == MVT::i128)
4182 LC = RTLIB::MULO_I128;
4183
4184 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
4185 // FIXME: This is not an optimal expansion, but better than crashing.
4186 EVT WideVT =
4187 EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
4188 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0));
4189 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1));
4190 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
4191 SDValue MulLo, MulHi;
4192 SplitInteger(Mul, MulLo, MulHi);
4193 SDValue SRA =
4194 DAG.getNode(ISD::SRA, dl, VT, MulLo,
4195 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT));
4196 SDValue Overflow =
4197 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE);
4198 SplitInteger(MulLo, Lo, Hi);
4199 ReplaceValueWith(SDValue(N, 1), Overflow);
4200 return;
4201 }
4202
4203 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
4204 // Temporary for the overflow value, default it to zero.
4205 SDValue Chain =
4206 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
4207 MachinePointerInfo());
4208
4209 TargetLowering::ArgListTy Args;
4210 TargetLowering::ArgListEntry Entry;
4211 for (const SDValue &Op : N->op_values()) {
4212 EVT ArgVT = Op.getValueType();
4213 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
4214 Entry.Node = Op;
4215 Entry.Ty = ArgTy;
4216 Entry.IsSExt = true;
4217 Entry.IsZExt = false;
4218 Args.push_back(Entry);
4219 }
4220
4221 // Also pass the address of the overflow check.
4222 Entry.Node = Temp;
4223 Entry.Ty = PtrTy->getPointerTo();
4224 Entry.IsSExt = true;
4225 Entry.IsZExt = false;
4226 Args.push_back(Entry);
4227
4228 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
4229
4230 TargetLowering::CallLoweringInfo CLI(DAG);
4231 CLI.setDebugLoc(dl)
4232 .setChain(Chain)
4233 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
4234 .setSExtResult();
4235
4236 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
4237
4238 SplitInteger(CallInfo.first, Lo, Hi);
4239 SDValue Temp2 =
4240 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
4241 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
4242 DAG.getConstant(0, dl, PtrVT),
4243 ISD::SETNE);
4244 // Use the overflow from the libcall everywhere.
4245 ReplaceValueWith(SDValue(N, 1), Ofl);
4246 }
4247
ExpandIntRes_UDIV(SDNode * N,SDValue & Lo,SDValue & Hi)4248 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
4249 SDValue &Lo, SDValue &Hi) {
4250 EVT VT = N->getValueType(0);
4251 SDLoc dl(N);
4252 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4253
4254 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4255 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4256 SplitInteger(Res.getValue(0), Lo, Hi);
4257 return;
4258 }
4259
4260 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4261 if (VT == MVT::i16)
4262 LC = RTLIB::UDIV_I16;
4263 else if (VT == MVT::i32)
4264 LC = RTLIB::UDIV_I32;
4265 else if (VT == MVT::i64)
4266 LC = RTLIB::UDIV_I64;
4267 else if (VT == MVT::i128)
4268 LC = RTLIB::UDIV_I128;
4269 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
4270
4271 TargetLowering::MakeLibCallOptions CallOptions;
4272 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4273 }
4274
ExpandIntRes_UREM(SDNode * N,SDValue & Lo,SDValue & Hi)4275 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
4276 SDValue &Lo, SDValue &Hi) {
4277 EVT VT = N->getValueType(0);
4278 SDLoc dl(N);
4279 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4280
4281 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4282 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4283 SplitInteger(Res.getValue(1), Lo, Hi);
4284 return;
4285 }
4286
4287 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4288 if (VT == MVT::i16)
4289 LC = RTLIB::UREM_I16;
4290 else if (VT == MVT::i32)
4291 LC = RTLIB::UREM_I32;
4292 else if (VT == MVT::i64)
4293 LC = RTLIB::UREM_I64;
4294 else if (VT == MVT::i128)
4295 LC = RTLIB::UREM_I128;
4296 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
4297
4298 TargetLowering::MakeLibCallOptions CallOptions;
4299 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4300 }
4301
ExpandIntRes_ZERO_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)4302 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
4303 SDValue &Lo, SDValue &Hi) {
4304 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4305 SDLoc dl(N);
4306 SDValue Op = N->getOperand(0);
4307 if (Op.getValueType().bitsLE(NVT)) {
4308 // The low part is zero extension of the input (degenerates to a copy).
4309 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
4310 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
4311 } else {
4312 // For example, extension of an i48 to an i64. The operand type necessarily
4313 // promotes to the result type, so will end up being expanded too.
4314 assert(getTypeAction(Op.getValueType()) ==
4315 TargetLowering::TypePromoteInteger &&
4316 "Only know how to promote this result!");
4317 SDValue Res = GetPromotedInteger(Op);
4318 assert(Res.getValueType() == N->getValueType(0) &&
4319 "Operand over promoted?");
4320 // Split the promoted operand. This will simplify when it is expanded.
4321 SplitInteger(Res, Lo, Hi);
4322 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4323 Hi = DAG.getZeroExtendInReg(Hi, dl,
4324 EVT::getIntegerVT(*DAG.getContext(),
4325 ExcessBits));
4326 }
4327 }
4328
ExpandIntRes_ATOMIC_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)4329 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
4330 SDValue &Lo, SDValue &Hi) {
4331 SDLoc dl(N);
4332 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
4333 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
4334 SDValue Zero = DAG.getConstant(0, dl, VT);
4335 SDValue Swap = DAG.getAtomicCmpSwap(
4336 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
4337 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
4338 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
4339
4340 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
4341 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
4342 }
4343
ExpandIntRes_VECREDUCE(SDNode * N,SDValue & Lo,SDValue & Hi)4344 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
4345 SDValue &Lo, SDValue &Hi) {
4346 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
4347 // both halves independently.
4348 SDValue Res = TLI.expandVecReduce(N, DAG);
4349 SplitInteger(Res, Lo, Hi);
4350 }
4351
ExpandIntRes_Rotate(SDNode * N,SDValue & Lo,SDValue & Hi)4352 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
4353 SDValue &Lo, SDValue &Hi) {
4354 // Lower the rotate to shifts and ORs which can be expanded.
4355 SDValue Res;
4356 TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
4357 SplitInteger(Res, Lo, Hi);
4358 }
4359
ExpandIntRes_FunnelShift(SDNode * N,SDValue & Lo,SDValue & Hi)4360 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N,
4361 SDValue &Lo, SDValue &Hi) {
4362 // Lower the funnel shift to shifts and ORs which can be expanded.
4363 SDValue Res;
4364 TLI.expandFunnelShift(N, Res, DAG);
4365 SplitInteger(Res, Lo, Hi);
4366 }
4367
ExpandIntRes_VSCALE(SDNode * N,SDValue & Lo,SDValue & Hi)4368 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
4369 SDValue &Hi) {
4370 EVT VT = N->getValueType(0);
4371 EVT HalfVT =
4372 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2);
4373 SDLoc dl(N);
4374
4375 // We assume VSCALE(1) fits into a legal integer.
4376 APInt One(HalfVT.getSizeInBits(), 1);
4377 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One);
4378 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase);
4379 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0));
4380 SplitInteger(Res, Lo, Hi);
4381 }
4382
4383 //===----------------------------------------------------------------------===//
4384 // Integer Operand Expansion
4385 //===----------------------------------------------------------------------===//
4386
4387 /// ExpandIntegerOperand - This method is called when the specified operand of
4388 /// the specified node is found to need expansion. At this point, all of the
4389 /// result types of the node are known to be legal, but other operands of the
4390 /// node may need promotion or expansion as well as the specified one.
ExpandIntegerOperand(SDNode * N,unsigned OpNo)4391 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
4392 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
4393 dbgs() << "\n");
4394 SDValue Res = SDValue();
4395
4396 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4397 return false;
4398
4399 switch (N->getOpcode()) {
4400 default:
4401 #ifndef NDEBUG
4402 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
4403 N->dump(&DAG); dbgs() << "\n";
4404 #endif
4405 report_fatal_error("Do not know how to expand this operator's operand!");
4406
4407 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
4408 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
4409 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
4410 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
4411 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
4412 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
4413 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
4414 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
4415 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
4416 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
4417 case ISD::STRICT_SINT_TO_FP:
4418 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
4419 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
4420 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
4421 case ISD::STRICT_UINT_TO_FP:
4422 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
4423
4424 case ISD::SHL:
4425 case ISD::SRA:
4426 case ISD::SRL:
4427 case ISD::ROTL:
4428 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
4429 case ISD::RETURNADDR:
4430 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
4431
4432 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
4433 }
4434
4435 // If the result is null, the sub-method took care of registering results etc.
4436 if (!Res.getNode()) return false;
4437
4438 // If the result is N, the sub-method updated N in place. Tell the legalizer
4439 // core about this.
4440 if (Res.getNode() == N)
4441 return true;
4442
4443 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4444 "Invalid operand expansion");
4445
4446 ReplaceValueWith(SDValue(N, 0), Res);
4447 return false;
4448 }
4449
4450 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
4451 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
IntegerExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl)4452 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
4453 SDValue &NewRHS,
4454 ISD::CondCode &CCCode,
4455 const SDLoc &dl) {
4456 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4457 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
4458 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
4459
4460 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
4461 if (RHSLo == RHSHi) {
4462 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
4463 if (RHSCST->isAllOnes()) {
4464 // Equality comparison to -1.
4465 NewLHS = DAG.getNode(ISD::AND, dl,
4466 LHSLo.getValueType(), LHSLo, LHSHi);
4467 NewRHS = RHSLo;
4468 return;
4469 }
4470 }
4471 }
4472
4473 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
4474 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
4475 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
4476 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
4477 return;
4478 }
4479
4480 // If this is a comparison of the sign bit, just look at the top part.
4481 // X > -1, x < 0
4482 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
4483 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0
4484 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1
4485 NewLHS = LHSHi;
4486 NewRHS = RHSHi;
4487 return;
4488 }
4489
4490 // FIXME: This generated code sucks.
4491 ISD::CondCode LowCC;
4492 switch (CCCode) {
4493 default: llvm_unreachable("Unknown integer setcc!");
4494 case ISD::SETLT:
4495 case ISD::SETULT: LowCC = ISD::SETULT; break;
4496 case ISD::SETGT:
4497 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
4498 case ISD::SETLE:
4499 case ISD::SETULE: LowCC = ISD::SETULE; break;
4500 case ISD::SETGE:
4501 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
4502 }
4503
4504 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
4505 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
4506 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
4507
4508 // NOTE: on targets without efficient SELECT of bools, we can always use
4509 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
4510 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
4511 nullptr);
4512 SDValue LoCmp, HiCmp;
4513 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
4514 TLI.isTypeLegal(RHSLo.getValueType()))
4515 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
4516 RHSLo, LowCC, false, DagCombineInfo, dl);
4517 if (!LoCmp.getNode())
4518 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
4519 RHSLo, LowCC);
4520 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
4521 TLI.isTypeLegal(RHSHi.getValueType()))
4522 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
4523 RHSHi, CCCode, false, DagCombineInfo, dl);
4524 if (!HiCmp.getNode())
4525 HiCmp =
4526 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
4527 LHSHi, RHSHi, DAG.getCondCode(CCCode));
4528
4529 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
4530 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
4531
4532 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
4533 CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
4534
4535 // FIXME: Is the HiCmpC->isOne() here correct for
4536 // ZeroOrNegativeOneBooleanContent.
4537 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) ||
4538 (!EqAllowed &&
4539 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) {
4540 // For LE / GE, if high part is known false, ignore the low part.
4541 // For LT / GT: if low part is known false, return the high part.
4542 // if high part is known true, ignore the low part.
4543 NewLHS = HiCmp;
4544 NewRHS = SDValue();
4545 return;
4546 }
4547
4548 if (LHSHi == RHSHi) {
4549 // Comparing the low bits is enough.
4550 NewLHS = LoCmp;
4551 NewRHS = SDValue();
4552 return;
4553 }
4554
4555 // Lower with SETCCCARRY if the target supports it.
4556 EVT HiVT = LHSHi.getValueType();
4557 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
4558 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
4559
4560 // FIXME: Make all targets support this, then remove the other lowering.
4561 if (HasSETCCCARRY) {
4562 // SETCCCARRY can detect < and >= directly. For > and <=, flip
4563 // operands and condition code.
4564 bool FlipOperands = false;
4565 switch (CCCode) {
4566 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
4567 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
4568 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
4569 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
4570 default: break;
4571 }
4572 if (FlipOperands) {
4573 std::swap(LHSLo, RHSLo);
4574 std::swap(LHSHi, RHSHi);
4575 }
4576 // Perform a wide subtraction, feeding the carry from the low part into
4577 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
4578 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
4579 // zero or positive iff LHS >= RHS.
4580 EVT LoVT = LHSLo.getValueType();
4581 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
4582 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
4583 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
4584 LHSHi, RHSHi, LowCmp.getValue(1),
4585 DAG.getCondCode(CCCode));
4586 NewLHS = Res;
4587 NewRHS = SDValue();
4588 return;
4589 }
4590
4591 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
4592 false, DagCombineInfo, dl);
4593 if (!NewLHS.getNode())
4594 NewLHS =
4595 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
4596 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
4597 NewRHS = SDValue();
4598 }
4599
ExpandIntOp_BR_CC(SDNode * N)4600 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
4601 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
4602 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
4603 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4604
4605 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4606 // against zero to select between true and false values.
4607 if (!NewRHS.getNode()) {
4608 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4609 CCCode = ISD::SETNE;
4610 }
4611
4612 // Update N to have the operands specified.
4613 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
4614 DAG.getCondCode(CCCode), NewLHS, NewRHS,
4615 N->getOperand(4)), 0);
4616 }
4617
ExpandIntOp_SELECT_CC(SDNode * N)4618 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
4619 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4620 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
4621 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4622
4623 // If ExpandSetCCOperands returned a scalar, we need to compare the result
4624 // against zero to select between true and false values.
4625 if (!NewRHS.getNode()) {
4626 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
4627 CCCode = ISD::SETNE;
4628 }
4629
4630 // Update N to have the operands specified.
4631 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
4632 N->getOperand(2), N->getOperand(3),
4633 DAG.getCondCode(CCCode)), 0);
4634 }
4635
ExpandIntOp_SETCC(SDNode * N)4636 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
4637 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
4638 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
4639 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
4640
4641 // If ExpandSetCCOperands returned a scalar, use it.
4642 if (!NewRHS.getNode()) {
4643 assert(NewLHS.getValueType() == N->getValueType(0) &&
4644 "Unexpected setcc expansion!");
4645 return NewLHS;
4646 }
4647
4648 // Otherwise, update N to have the operands specified.
4649 return SDValue(
4650 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
4651 }
4652
ExpandIntOp_SETCCCARRY(SDNode * N)4653 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
4654 SDValue LHS = N->getOperand(0);
4655 SDValue RHS = N->getOperand(1);
4656 SDValue Carry = N->getOperand(2);
4657 SDValue Cond = N->getOperand(3);
4658 SDLoc dl = SDLoc(N);
4659
4660 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
4661 GetExpandedInteger(LHS, LHSLo, LHSHi);
4662 GetExpandedInteger(RHS, RHSLo, RHSHi);
4663
4664 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
4665 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
4666 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
4667 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
4668 LowCmp.getValue(1), Cond);
4669 }
4670
ExpandIntOp_SPLAT_VECTOR(SDNode * N)4671 SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
4672 // Split the operand and replace with SPLAT_VECTOR_PARTS.
4673 SDValue Lo, Hi;
4674 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4675 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo,
4676 Hi);
4677 }
4678
ExpandIntOp_Shift(SDNode * N)4679 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
4680 // The value being shifted is legal, but the shift amount is too big.
4681 // It follows that either the result of the shift is undefined, or the
4682 // upper half of the shift amount is zero. Just use the lower half.
4683 SDValue Lo, Hi;
4684 GetExpandedInteger(N->getOperand(1), Lo, Hi);
4685 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
4686 }
4687
ExpandIntOp_RETURNADDR(SDNode * N)4688 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
4689 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
4690 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
4691 // constant to valid type.
4692 SDValue Lo, Hi;
4693 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4694 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
4695 }
4696
ExpandIntOp_SINT_TO_FP(SDNode * N)4697 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
4698 bool IsStrict = N->isStrictFPOpcode();
4699 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4700 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4701 EVT DstVT = N->getValueType(0);
4702 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
4703 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4704 "Don't know how to expand this SINT_TO_FP!");
4705 TargetLowering::MakeLibCallOptions CallOptions;
4706 CallOptions.setSExt(true);
4707 std::pair<SDValue, SDValue> Tmp =
4708 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4709
4710 if (!IsStrict)
4711 return Tmp.first;
4712
4713 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4714 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4715 return SDValue();
4716 }
4717
ExpandIntOp_STORE(StoreSDNode * N,unsigned OpNo)4718 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
4719 if (N->isAtomic()) {
4720 // It's typical to have larger CAS than atomic store instructions.
4721 SDLoc dl(N);
4722 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4723 N->getMemoryVT(),
4724 N->getOperand(0), N->getOperand(2),
4725 N->getOperand(1),
4726 N->getMemOperand());
4727 return Swap.getValue(1);
4728 }
4729 if (ISD::isNormalStore(N))
4730 return ExpandOp_NormalStore(N, OpNo);
4731
4732 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
4733 assert(OpNo == 1 && "Can only expand the stored value so far");
4734
4735 EVT VT = N->getOperand(1).getValueType();
4736 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4737 SDValue Ch = N->getChain();
4738 SDValue Ptr = N->getBasePtr();
4739 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4740 AAMDNodes AAInfo = N->getAAInfo();
4741 SDLoc dl(N);
4742 SDValue Lo, Hi;
4743
4744 assert(NVT.isByteSized() && "Expanded type not byte sized!");
4745
4746 if (N->getMemoryVT().bitsLE(NVT)) {
4747 GetExpandedInteger(N->getValue(), Lo, Hi);
4748 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4749 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags,
4750 AAInfo);
4751 }
4752
4753 if (DAG.getDataLayout().isLittleEndian()) {
4754 // Little-endian - low bits are at low addresses.
4755 GetExpandedInteger(N->getValue(), Lo, Hi);
4756
4757 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
4758 N->getOriginalAlign(), MMOFlags, AAInfo);
4759
4760 unsigned ExcessBits =
4761 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
4762 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
4763
4764 // Increment the pointer to the other half.
4765 unsigned IncrementSize = NVT.getSizeInBits()/8;
4766 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
4767 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
4768 N->getPointerInfo().getWithOffset(IncrementSize),
4769 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo);
4770 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4771 }
4772
4773 // Big-endian - high bits are at low addresses. Favor aligned stores at
4774 // the cost of some bit-fiddling.
4775 GetExpandedInteger(N->getValue(), Lo, Hi);
4776
4777 EVT ExtVT = N->getMemoryVT();
4778 unsigned EBytes = ExtVT.getStoreSize();
4779 unsigned IncrementSize = NVT.getSizeInBits()/8;
4780 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4781 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
4782 ExtVT.getSizeInBits() - ExcessBits);
4783
4784 if (ExcessBits < NVT.getSizeInBits()) {
4785 // Transfer high bits from the top of Lo to the bottom of Hi.
4786 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
4787 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
4788 TLI.getPointerTy(DAG.getDataLayout())));
4789 Hi = DAG.getNode(
4790 ISD::OR, dl, NVT, Hi,
4791 DAG.getNode(ISD::SRL, dl, NVT, Lo,
4792 DAG.getConstant(ExcessBits, dl,
4793 TLI.getPointerTy(DAG.getDataLayout()))));
4794 }
4795
4796 // Store both the high bits and maybe some of the low bits.
4797 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
4798 N->getOriginalAlign(), MMOFlags, AAInfo);
4799
4800 // Increment the pointer to the other half.
4801 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
4802 // Store the lowest ExcessBits bits in the second half.
4803 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
4804 N->getPointerInfo().getWithOffset(IncrementSize),
4805 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
4806 N->getOriginalAlign(), MMOFlags, AAInfo);
4807 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4808 }
4809
ExpandIntOp_TRUNCATE(SDNode * N)4810 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
4811 SDValue InL, InH;
4812 GetExpandedInteger(N->getOperand(0), InL, InH);
4813 // Just truncate the low part of the source.
4814 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
4815 }
4816
ExpandIntOp_UINT_TO_FP(SDNode * N)4817 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
4818 bool IsStrict = N->isStrictFPOpcode();
4819 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4820 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
4821 EVT DstVT = N->getValueType(0);
4822 RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
4823 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4824 "Don't know how to expand this UINT_TO_FP!");
4825 TargetLowering::MakeLibCallOptions CallOptions;
4826 CallOptions.setSExt(true);
4827 std::pair<SDValue, SDValue> Tmp =
4828 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
4829
4830 if (!IsStrict)
4831 return Tmp.first;
4832
4833 ReplaceValueWith(SDValue(N, 1), Tmp.second);
4834 ReplaceValueWith(SDValue(N, 0), Tmp.first);
4835 return SDValue();
4836 }
4837
ExpandIntOp_ATOMIC_STORE(SDNode * N)4838 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
4839 SDLoc dl(N);
4840 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4841 cast<AtomicSDNode>(N)->getMemoryVT(),
4842 N->getOperand(0),
4843 N->getOperand(1), N->getOperand(2),
4844 cast<AtomicSDNode>(N)->getMemOperand());
4845 return Swap.getValue(1);
4846 }
4847
PromoteIntRes_VECTOR_SPLICE(SDNode * N)4848 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
4849 SDLoc dl(N);
4850
4851 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4852 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4853 EVT OutVT = V0.getValueType();
4854
4855 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2));
4856 }
4857
PromoteIntRes_EXTRACT_SUBVECTOR(SDNode * N)4858 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4859
4860 EVT OutVT = N->getValueType(0);
4861 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4862 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4863 EVT NOutVTElem = NOutVT.getVectorElementType();
4864
4865 SDLoc dl(N);
4866 SDValue BaseIdx = N->getOperand(1);
4867
4868 // TODO: We may be able to use this for types other than scalable
4869 // vectors and fix those tests that expect BUILD_VECTOR to be used
4870 if (OutVT.isScalableVector()) {
4871 SDValue InOp0 = N->getOperand(0);
4872 EVT InVT = InOp0.getValueType();
4873
4874 // Try and extract from a smaller type so that it eventually falls
4875 // into the promotion code below.
4876 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector ||
4877 getTypeAction(InVT) == TargetLowering::TypeLegal) {
4878 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext());
4879 unsigned NElts = NInVT.getVectorMinNumElements();
4880 uint64_t IdxVal = cast<ConstantSDNode>(BaseIdx)->getZExtValue();
4881
4882 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0,
4883 DAG.getConstant(alignDown(IdxVal, NElts), dl,
4884 BaseIdx.getValueType()));
4885 SDValue Step2 = DAG.getNode(
4886 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1,
4887 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType()));
4888 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2);
4889 }
4890
4891 // Try and extract from a widened type.
4892 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4893 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
4894 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops);
4895 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
4896 }
4897
4898 // Promote operands and see if this is handled by target lowering,
4899 // Otherwise, use the BUILD_VECTOR approach below
4900 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
4901 // Collect the (promoted) operands
4902 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
4903
4904 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
4905 assert(PromEltVT.bitsLE(NOutVTElem) &&
4906 "Promoted operand has an element type greater than result");
4907
4908 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
4909 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
4910 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
4911 }
4912 }
4913
4914 if (OutVT.isScalableVector())
4915 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
4916
4917 SDValue InOp0 = N->getOperand(0);
4918 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
4919 InOp0 = GetPromotedInteger(N->getOperand(0));
4920
4921 EVT InVT = InOp0.getValueType();
4922
4923 unsigned OutNumElems = OutVT.getVectorNumElements();
4924 SmallVector<SDValue, 8> Ops;
4925 Ops.reserve(OutNumElems);
4926 for (unsigned i = 0; i != OutNumElems; ++i) {
4927
4928 // Extract the element from the original vector.
4929 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4930 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4931 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4932 InVT.getVectorElementType(), N->getOperand(0), Index);
4933
4934 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4935 // Insert the converted element to the new vector.
4936 Ops.push_back(Op);
4937 }
4938
4939 return DAG.getBuildVector(NOutVT, dl, Ops);
4940 }
4941
PromoteIntRes_INSERT_SUBVECTOR(SDNode * N)4942 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) {
4943 EVT OutVT = N->getValueType(0);
4944 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4945 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4946
4947 SDLoc dl(N);
4948 SDValue Vec = N->getOperand(0);
4949 SDValue SubVec = N->getOperand(1);
4950 SDValue Idx = N->getOperand(2);
4951
4952 EVT SubVecVT = SubVec.getValueType();
4953 EVT NSubVT =
4954 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(),
4955 SubVecVT.getVectorElementCount());
4956
4957 Vec = GetPromotedInteger(Vec);
4958 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec);
4959
4960 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx);
4961 }
4962
PromoteIntRes_VECTOR_REVERSE(SDNode * N)4963 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
4964 SDLoc dl(N);
4965
4966 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4967 EVT OutVT = V0.getValueType();
4968
4969 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0);
4970 }
4971
PromoteIntRes_VECTOR_SHUFFLE(SDNode * N)4972 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4973 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4974 EVT VT = N->getValueType(0);
4975 SDLoc dl(N);
4976
4977 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4978
4979 SDValue V0 = GetPromotedInteger(N->getOperand(0));
4980 SDValue V1 = GetPromotedInteger(N->getOperand(1));
4981 EVT OutVT = V0.getValueType();
4982
4983 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4984 }
4985
4986
PromoteIntRes_BUILD_VECTOR(SDNode * N)4987 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4988 EVT OutVT = N->getValueType(0);
4989 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4990 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4991 unsigned NumElems = N->getNumOperands();
4992 EVT NOutVTElem = NOutVT.getVectorElementType();
4993
4994 SDLoc dl(N);
4995
4996 SmallVector<SDValue, 8> Ops;
4997 Ops.reserve(NumElems);
4998 for (unsigned i = 0; i != NumElems; ++i) {
4999 SDValue Op;
5000 // BUILD_VECTOR integer operand types are allowed to be larger than the
5001 // result's element type. This may still be true after the promotion. For
5002 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
5003 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
5004 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
5005 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
5006 else
5007 Op = N->getOperand(i);
5008 Ops.push_back(Op);
5009 }
5010
5011 return DAG.getBuildVector(NOutVT, dl, Ops);
5012 }
5013
PromoteIntRes_SCALAR_TO_VECTOR(SDNode * N)5014 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
5015
5016 SDLoc dl(N);
5017
5018 assert(!N->getOperand(0).getValueType().isVector() &&
5019 "Input must be a scalar");
5020
5021 EVT OutVT = N->getValueType(0);
5022 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5023 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5024 EVT NOutVTElem = NOutVT.getVectorElementType();
5025
5026 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
5027
5028 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
5029 }
5030
PromoteIntRes_SPLAT_VECTOR(SDNode * N)5031 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
5032 SDLoc dl(N);
5033
5034 SDValue SplatVal = N->getOperand(0);
5035
5036 assert(!SplatVal.getValueType().isVector() && "Input must be a scalar");
5037
5038 EVT OutVT = N->getValueType(0);
5039 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5040 assert(NOutVT.isVector() && "Type must be promoted to a vector type");
5041 EVT NOutElemVT = NOutVT.getVectorElementType();
5042
5043 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
5044
5045 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
5046 }
5047
PromoteIntRes_STEP_VECTOR(SDNode * N)5048 SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
5049 SDLoc dl(N);
5050 EVT OutVT = N->getValueType(0);
5051 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5052 assert(NOutVT.isVector() && "Type must be promoted to a vector type");
5053 APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
5054 return DAG.getStepVector(dl, NOutVT,
5055 StepVal.sext(NOutVT.getScalarSizeInBits()));
5056 }
5057
PromoteIntRes_CONCAT_VECTORS(SDNode * N)5058 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
5059 SDLoc dl(N);
5060
5061 EVT OutVT = N->getValueType(0);
5062 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5063 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5064
5065 unsigned NumOperands = N->getNumOperands();
5066 unsigned NumOutElem = NOutVT.getVectorMinNumElements();
5067 EVT OutElemTy = NOutVT.getVectorElementType();
5068 if (OutVT.isScalableVector()) {
5069 // Find the largest promoted element type for each of the operands.
5070 SDUse *MaxSizedValue = std::max_element(
5071 N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) {
5072 EVT AVT = A.getValueType().getVectorElementType();
5073 EVT BVT = B.getValueType().getVectorElementType();
5074 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5075 });
5076 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType();
5077
5078 // Then promote all vectors to the largest element type.
5079 SmallVector<SDValue, 8> Ops;
5080 for (unsigned I = 0; I < NumOperands; ++I) {
5081 SDValue Op = N->getOperand(I);
5082 EVT OpVT = Op.getValueType();
5083 if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger)
5084 Op = GetPromotedInteger(Op);
5085 else
5086 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal &&
5087 "Unhandled legalization type");
5088
5089 if (OpVT.getVectorElementType().getScalarSizeInBits() <
5090 MaxElementVT.getScalarSizeInBits())
5091 Op = DAG.getAnyExtOrTrunc(Op, dl,
5092 OpVT.changeVectorElementType(MaxElementVT));
5093 Ops.push_back(Op);
5094 }
5095
5096 // Do the CONCAT on the promoted type and finally truncate to (the promoted)
5097 // NOutVT.
5098 return DAG.getAnyExtOrTrunc(
5099 DAG.getNode(ISD::CONCAT_VECTORS, dl,
5100 OutVT.changeVectorElementType(MaxElementVT), Ops),
5101 dl, NOutVT);
5102 }
5103
5104 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
5105 assert(NumElem * NumOperands == NumOutElem &&
5106 "Unexpected number of elements");
5107
5108 // Take the elements from the first vector.
5109 SmallVector<SDValue, 8> Ops(NumOutElem);
5110 for (unsigned i = 0; i < NumOperands; ++i) {
5111 SDValue Op = N->getOperand(i);
5112 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
5113 Op = GetPromotedInteger(Op);
5114 EVT SclrTy = Op.getValueType().getVectorElementType();
5115 assert(NumElem == Op.getValueType().getVectorNumElements() &&
5116 "Unexpected number of elements");
5117
5118 for (unsigned j = 0; j < NumElem; ++j) {
5119 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
5120 DAG.getVectorIdxConstant(j, dl));
5121 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
5122 }
5123 }
5124
5125 return DAG.getBuildVector(NOutVT, dl, Ops);
5126 }
5127
PromoteIntRes_EXTEND_VECTOR_INREG(SDNode * N)5128 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
5129 EVT VT = N->getValueType(0);
5130 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5131 assert(NVT.isVector() && "This type must be promoted to a vector type");
5132
5133 SDLoc dl(N);
5134
5135 // For operands whose TypeAction is to promote, extend the promoted node
5136 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
5137 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
5138 // type..
5139 if (getTypeAction(N->getOperand(0).getValueType())
5140 == TargetLowering::TypePromoteInteger) {
5141 SDValue Promoted;
5142
5143 switch(N->getOpcode()) {
5144 case ISD::SIGN_EXTEND_VECTOR_INREG:
5145 Promoted = SExtPromotedInteger(N->getOperand(0));
5146 break;
5147 case ISD::ZERO_EXTEND_VECTOR_INREG:
5148 Promoted = ZExtPromotedInteger(N->getOperand(0));
5149 break;
5150 case ISD::ANY_EXTEND_VECTOR_INREG:
5151 Promoted = GetPromotedInteger(N->getOperand(0));
5152 break;
5153 default:
5154 llvm_unreachable("Node has unexpected Opcode");
5155 }
5156 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
5157 }
5158
5159 // Directly extend to the appropriate transform-to type.
5160 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
5161 }
5162
PromoteIntRes_INSERT_VECTOR_ELT(SDNode * N)5163 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
5164 EVT OutVT = N->getValueType(0);
5165 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5166 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5167
5168 EVT NOutVTElem = NOutVT.getVectorElementType();
5169
5170 SDLoc dl(N);
5171 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5172
5173 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
5174 NOutVTElem, N->getOperand(1));
5175 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
5176 V0, ConvElem, N->getOperand(2));
5177 }
5178
PromoteIntRes_VECREDUCE(SDNode * N)5179 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
5180 // The VECREDUCE result size may be larger than the element size, so
5181 // we can simply change the result type.
5182 SDLoc dl(N);
5183 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5184 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops());
5185 }
5186
PromoteIntRes_VP_REDUCE(SDNode * N)5187 SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) {
5188 // The VP_REDUCE result size may be larger than the element size, so we can
5189 // simply change the result type. However the start value and result must be
5190 // the same.
5191 SDLoc DL(N);
5192 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0));
5193 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start,
5194 N->getOperand(1), N->getOperand(2), N->getOperand(3));
5195 }
5196
PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode * N)5197 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
5198 SDLoc dl(N);
5199 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5200 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
5201 TLI.getVectorIdxTy(DAG.getDataLayout()));
5202 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
5203 V0->getValueType(0).getScalarType(), V0, V1);
5204
5205 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
5206 // element types. If this is the case then we need to expand the outgoing
5207 // value and not truncate it.
5208 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5209 }
5210
PromoteIntOp_INSERT_SUBVECTOR(SDNode * N)5211 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) {
5212 SDLoc dl(N);
5213 // The result type is equal to the first input operand's type, so the
5214 // type that needs promoting must be the second source vector.
5215 SDValue V0 = N->getOperand(0);
5216 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5217 SDValue Idx = N->getOperand(2);
5218 EVT PromVT = EVT::getVectorVT(*DAG.getContext(),
5219 V1.getValueType().getVectorElementType(),
5220 V0.getValueType().getVectorElementCount());
5221 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT);
5222 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx);
5223 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5224 }
5225
PromoteIntOp_EXTRACT_SUBVECTOR(SDNode * N)5226 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
5227 SDLoc dl(N);
5228 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5229 MVT InVT = V0.getValueType().getSimpleVT();
5230 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
5231 N->getValueType(0).getVectorNumElements());
5232 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
5233 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
5234 }
5235
PromoteIntOp_CONCAT_VECTORS(SDNode * N)5236 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
5237 SDLoc dl(N);
5238
5239 EVT ResVT = N->getValueType(0);
5240 unsigned NumElems = N->getNumOperands();
5241
5242 if (ResVT.isScalableVector()) {
5243 SDValue ResVec = DAG.getUNDEF(ResVT);
5244
5245 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
5246 SDValue Op = N->getOperand(OpIdx);
5247 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
5248 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
5249 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
5250 }
5251
5252 return ResVec;
5253 }
5254
5255 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
5256
5257 SmallVector<SDValue, 8> NewOps;
5258 NewOps.reserve(NumElems);
5259
5260 // For each incoming vector
5261 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
5262 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
5263 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
5264 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
5265
5266 for (unsigned i=0; i<NumElem; ++i) {
5267 // Extract element from incoming vector
5268 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
5269 DAG.getVectorIdxConstant(i, dl));
5270 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
5271 NewOps.push_back(Tr);
5272 }
5273 }
5274
5275 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
5276 }
5277