1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19
20 #include "LegalizeTypes.h"
21 #include "llvm/Analysis/TargetLibraryInfo.h"
22 #include "llvm/CodeGen/StackMaps.h"
23 #include "llvm/CodeGen/TargetLowering.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/Support/ErrorHandling.h"
26 #include "llvm/Support/KnownBits.h"
27 #include "llvm/Support/raw_ostream.h"
28 #include <algorithm>
29 using namespace llvm;
30
31 #define DEBUG_TYPE "legalize-types"
32
33 //===----------------------------------------------------------------------===//
34 // Integer Result Promotion
35 //===----------------------------------------------------------------------===//
36
37 /// PromoteIntegerResult - This method is called when a result of a node is
38 /// found to be in need of promotion to a larger type. At this point, the node
39 /// may also have invalid operands or may have other results that need
40 /// expansion, we just know that (at least) one result needs promotion.
PromoteIntegerResult(SDNode * N,unsigned ResNo)41 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG));
43 SDValue Res = SDValue();
44
45 // See if the target wants to custom expand this node.
46 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
47 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
48 return;
49 }
50
51 switch (N->getOpcode()) {
52 default:
53 #ifndef NDEBUG
54 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
55 N->dump(&DAG); dbgs() << "\n";
56 #endif
57 report_fatal_error("Do not know how to promote this operator!");
58 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
59 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
60 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
61 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
62 case ISD::VP_BITREVERSE:
63 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
64 case ISD::VP_BSWAP:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
66 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
67 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
69 case ISD::VP_CTLZ:
70 case ISD::CTLZ_ZERO_UNDEF:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
72 case ISD::PARITY:
73 case ISD::VP_CTPOP:
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
76 case ISD::VP_CTTZ:
77 case ISD::CTTZ_ZERO_UNDEF:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
79 case ISD::EXTRACT_VECTOR_ELT:
80 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
81 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
82 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
83 break;
84 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
85 break;
86 case ISD::SELECT:
87 case ISD::VSELECT:
88 case ISD::VP_SELECT:
89 case ISD::VP_MERGE:
90 Res = PromoteIntRes_Select(N);
91 break;
92 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
93 case ISD::STRICT_FSETCC:
94 case ISD::STRICT_FSETCCS:
95 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
96 case ISD::SMIN:
97 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
98 case ISD::UMIN:
99 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
100
101 case ISD::SHL:
102 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break;
103 case ISD::SIGN_EXTEND_INREG:
104 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
105 case ISD::SRA:
106 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break;
107 case ISD::SRL:
108 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break;
109 case ISD::VP_TRUNCATE:
110 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
111 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
112 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
113 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
114
115 case ISD::EXTRACT_SUBVECTOR:
116 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
117 case ISD::INSERT_SUBVECTOR:
118 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
119 case ISD::VECTOR_REVERSE:
120 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
121 case ISD::VECTOR_SHUFFLE:
122 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
123 case ISD::VECTOR_SPLICE:
124 Res = PromoteIntRes_VECTOR_SPLICE(N); break;
125 case ISD::VECTOR_INTERLEAVE:
126 case ISD::VECTOR_DEINTERLEAVE:
127 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
128 return;
129 case ISD::INSERT_VECTOR_ELT:
130 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
131 case ISD::BUILD_VECTOR:
132 Res = PromoteIntRes_BUILD_VECTOR(N);
133 break;
134 case ISD::SPLAT_VECTOR:
135 case ISD::SCALAR_TO_VECTOR:
136 Res = PromoteIntRes_ScalarOp(N);
137 break;
138 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
139 case ISD::CONCAT_VECTORS:
140 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
141
142 case ISD::ANY_EXTEND_VECTOR_INREG:
143 case ISD::SIGN_EXTEND_VECTOR_INREG:
144 case ISD::ZERO_EXTEND_VECTOR_INREG:
145 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
146
147 case ISD::SIGN_EXTEND:
148 case ISD::VP_SIGN_EXTEND:
149 case ISD::ZERO_EXTEND:
150 case ISD::VP_ZERO_EXTEND:
151 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
152
153 case ISD::VP_FP_TO_SINT:
154 case ISD::VP_FP_TO_UINT:
155 case ISD::STRICT_FP_TO_SINT:
156 case ISD::STRICT_FP_TO_UINT:
157 case ISD::FP_TO_SINT:
158 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
159
160 case ISD::FP_TO_SINT_SAT:
161 case ISD::FP_TO_UINT_SAT:
162 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
163
164 case ISD::FP_TO_BF16:
165 case ISD::FP_TO_FP16:
166 Res = PromoteIntRes_FP_TO_FP16_BF16(N);
167 break;
168 case ISD::STRICT_FP_TO_FP16:
169 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N);
170 break;
171 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break;
172
173 case ISD::AND:
174 case ISD::OR:
175 case ISD::XOR:
176 case ISD::ADD:
177 case ISD::SUB:
178 case ISD::MUL:
179 case ISD::VP_AND:
180 case ISD::VP_OR:
181 case ISD::VP_XOR:
182 case ISD::VP_ADD:
183 case ISD::VP_SUB:
184 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
185
186 case ISD::VP_SMIN:
187 case ISD::VP_SMAX:
188 case ISD::SDIV:
189 case ISD::SREM:
190 case ISD::VP_SDIV:
191 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
192
193 case ISD::VP_UMIN:
194 case ISD::VP_UMAX:
195 case ISD::UDIV:
196 case ISD::UREM:
197 case ISD::VP_UDIV:
198 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
199
200 case ISD::SADDO:
201 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
202 case ISD::UADDO:
203 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
204 case ISD::SMULO:
205 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
206
207 case ISD::ADDE:
208 case ISD::SUBE:
209 case ISD::UADDO_CARRY:
210 case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break;
211
212 case ISD::SADDO_CARRY:
213 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
214
215 case ISD::SADDSAT:
216 case ISD::UADDSAT:
217 case ISD::SSUBSAT:
218 case ISD::USUBSAT:
219 case ISD::SSHLSAT:
220 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break;
221
222 case ISD::SMULFIX:
223 case ISD::SMULFIXSAT:
224 case ISD::UMULFIX:
225 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
226
227 case ISD::SDIVFIX:
228 case ISD::SDIVFIXSAT:
229 case ISD::UDIVFIX:
230 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
231
232 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
233
234 case ISD::ATOMIC_LOAD:
235 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
236
237 case ISD::ATOMIC_LOAD_ADD:
238 case ISD::ATOMIC_LOAD_SUB:
239 case ISD::ATOMIC_LOAD_AND:
240 case ISD::ATOMIC_LOAD_CLR:
241 case ISD::ATOMIC_LOAD_OR:
242 case ISD::ATOMIC_LOAD_XOR:
243 case ISD::ATOMIC_LOAD_NAND:
244 case ISD::ATOMIC_LOAD_MIN:
245 case ISD::ATOMIC_LOAD_MAX:
246 case ISD::ATOMIC_LOAD_UMIN:
247 case ISD::ATOMIC_LOAD_UMAX:
248 case ISD::ATOMIC_SWAP:
249 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
250
251 case ISD::ATOMIC_CMP_SWAP:
252 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
253 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
254 break;
255
256 case ISD::VECREDUCE_ADD:
257 case ISD::VECREDUCE_MUL:
258 case ISD::VECREDUCE_AND:
259 case ISD::VECREDUCE_OR:
260 case ISD::VECREDUCE_XOR:
261 case ISD::VECREDUCE_SMAX:
262 case ISD::VECREDUCE_SMIN:
263 case ISD::VECREDUCE_UMAX:
264 case ISD::VECREDUCE_UMIN:
265 Res = PromoteIntRes_VECREDUCE(N);
266 break;
267
268 case ISD::VP_REDUCE_ADD:
269 case ISD::VP_REDUCE_MUL:
270 case ISD::VP_REDUCE_AND:
271 case ISD::VP_REDUCE_OR:
272 case ISD::VP_REDUCE_XOR:
273 case ISD::VP_REDUCE_SMAX:
274 case ISD::VP_REDUCE_SMIN:
275 case ISD::VP_REDUCE_UMAX:
276 case ISD::VP_REDUCE_UMIN:
277 Res = PromoteIntRes_VP_REDUCE(N);
278 break;
279
280 case ISD::FREEZE:
281 Res = PromoteIntRes_FREEZE(N);
282 break;
283
284 case ISD::ROTL:
285 case ISD::ROTR:
286 Res = PromoteIntRes_Rotate(N);
287 break;
288
289 case ISD::FSHL:
290 case ISD::FSHR:
291 Res = PromoteIntRes_FunnelShift(N);
292 break;
293
294 case ISD::VP_FSHL:
295 case ISD::VP_FSHR:
296 Res = PromoteIntRes_VPFunnelShift(N);
297 break;
298
299 case ISD::IS_FPCLASS:
300 Res = PromoteIntRes_IS_FPCLASS(N);
301 break;
302 case ISD::FFREXP:
303 Res = PromoteIntRes_FFREXP(N);
304 break;
305
306 case ISD::LRINT:
307 case ISD::LLRINT:
308 Res = PromoteIntRes_XRINT(N);
309 break;
310 }
311
312 // If the result is null then the sub-method took care of registering it.
313 if (Res.getNode())
314 SetPromotedInteger(SDValue(N, ResNo), Res);
315 }
316
PromoteIntRes_MERGE_VALUES(SDNode * N,unsigned ResNo)317 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
318 unsigned ResNo) {
319 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
320 return GetPromotedInteger(Op);
321 }
322
PromoteIntRes_AssertSext(SDNode * N)323 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
324 // Sign-extend the new bits, and continue the assertion.
325 SDValue Op = SExtPromotedInteger(N->getOperand(0));
326 return DAG.getNode(ISD::AssertSext, SDLoc(N),
327 Op.getValueType(), Op, N->getOperand(1));
328 }
329
PromoteIntRes_AssertZext(SDNode * N)330 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
331 // Zero the new bits, and continue the assertion.
332 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
333 return DAG.getNode(ISD::AssertZext, SDLoc(N),
334 Op.getValueType(), Op, N->getOperand(1));
335 }
336
PromoteIntRes_Atomic0(AtomicSDNode * N)337 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
338 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
339 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
340 N->getMemoryVT(), ResVT,
341 N->getChain(), N->getBasePtr(),
342 N->getMemOperand());
343 // Legalize the chain result - switch anything that used the old chain to
344 // use the new one.
345 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
346 return Res;
347 }
348
PromoteIntRes_Atomic1(AtomicSDNode * N)349 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
350 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
351 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
352 N->getMemoryVT(),
353 N->getChain(), N->getBasePtr(),
354 Op2, N->getMemOperand());
355 // Legalize the chain result - switch anything that used the old chain to
356 // use the new one.
357 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
358 return Res;
359 }
360
PromoteIntRes_AtomicCmpSwap(AtomicSDNode * N,unsigned ResNo)361 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
362 unsigned ResNo) {
363 if (ResNo == 1) {
364 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
365 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
366 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
367
368 // Only use the result of getSetCCResultType if it is legal,
369 // otherwise just use the promoted result type (NVT).
370 if (!TLI.isTypeLegal(SVT))
371 SVT = NVT;
372
373 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
374 SDValue Res = DAG.getAtomicCmpSwap(
375 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
376 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
377 N->getMemOperand());
378 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
379 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
380 return DAG.getSExtOrTrunc(Res.getValue(1), SDLoc(N), NVT);
381 }
382
383 // Op2 is used for the comparison and thus must be extended according to the
384 // target's atomic operations. Op3 is merely stored and so can be left alone.
385 SDValue Op2 = N->getOperand(2);
386 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
387 switch (TLI.getExtendForAtomicCmpSwapArg()) {
388 case ISD::SIGN_EXTEND:
389 Op2 = SExtPromotedInteger(Op2);
390 break;
391 case ISD::ZERO_EXTEND:
392 Op2 = ZExtPromotedInteger(Op2);
393 break;
394 case ISD::ANY_EXTEND:
395 Op2 = GetPromotedInteger(Op2);
396 break;
397 default:
398 llvm_unreachable("Invalid atomic op extension");
399 }
400
401 SDVTList VTs =
402 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
403 SDValue Res = DAG.getAtomicCmpSwap(
404 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
405 N->getBasePtr(), Op2, Op3, N->getMemOperand());
406 // Update the use to N with the newly created Res.
407 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
408 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
409 return Res;
410 }
411
PromoteIntRes_BITCAST(SDNode * N)412 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
413 SDValue InOp = N->getOperand(0);
414 EVT InVT = InOp.getValueType();
415 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
416 EVT OutVT = N->getValueType(0);
417 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
418 SDLoc dl(N);
419
420 switch (getTypeAction(InVT)) {
421 case TargetLowering::TypeLegal:
422 break;
423 case TargetLowering::TypePromoteInteger:
424 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
425 // The input promotes to the same size. Convert the promoted value.
426 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
427 break;
428 case TargetLowering::TypeSoftenFloat:
429 // Promote the integer operand by hand.
430 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
431 case TargetLowering::TypeSoftPromoteHalf:
432 // Promote the integer operand by hand.
433 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
434 case TargetLowering::TypePromoteFloat: {
435 // Convert the promoted float by hand.
436 if (!NOutVT.isVector())
437 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
438 break;
439 }
440 case TargetLowering::TypeExpandInteger:
441 case TargetLowering::TypeExpandFloat:
442 break;
443 case TargetLowering::TypeScalarizeVector:
444 // Convert the element to an integer and promote it by hand.
445 if (!NOutVT.isVector())
446 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
447 BitConvertToInteger(GetScalarizedVector(InOp)));
448 break;
449 case TargetLowering::TypeScalarizeScalableVector:
450 report_fatal_error("Scalarization of scalable vectors is not supported.");
451 case TargetLowering::TypeSplitVector: {
452 if (!NOutVT.isVector()) {
453 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
454 // pieces of the input into integers and reassemble in the final type.
455 SDValue Lo, Hi;
456 GetSplitVector(N->getOperand(0), Lo, Hi);
457 Lo = BitConvertToInteger(Lo);
458 Hi = BitConvertToInteger(Hi);
459
460 if (DAG.getDataLayout().isBigEndian())
461 std::swap(Lo, Hi);
462
463 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
464 EVT::getIntegerVT(*DAG.getContext(),
465 NOutVT.getSizeInBits()),
466 JoinIntegers(Lo, Hi));
467 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
468 }
469 break;
470 }
471 case TargetLowering::TypeWidenVector:
472 // The input is widened to the same size. Convert to the widened value.
473 // Make sure that the outgoing value is not a vector, because this would
474 // make us bitcast between two vectors which are legalized in different ways.
475 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
476 SDValue Res =
477 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
478
479 // For big endian targets we need to shift the casted value or the
480 // interesting bits will end up at the wrong place.
481 if (DAG.getDataLayout().isBigEndian()) {
482 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
483 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
484 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
485 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl));
486 }
487 return Res;
488 }
489 // If the output type is also a vector and widening it to the same size
490 // as the widened input type would be a legal type, we can widen the bitcast
491 // and handle the promotion after.
492 if (NOutVT.isVector()) {
493 TypeSize WidenInSize = NInVT.getSizeInBits();
494 TypeSize OutSize = OutVT.getSizeInBits();
495 if (WidenInSize.hasKnownScalarFactor(OutSize)) {
496 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize);
497 EVT WideOutVT =
498 EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(),
499 OutVT.getVectorElementCount() * Scale);
500 if (isTypeLegal(WideOutVT)) {
501 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
502 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
503 DAG.getVectorIdxConstant(0, dl));
504 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
505 }
506 }
507 }
508 }
509
510 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
511 CreateStackStoreLoad(InOp, OutVT));
512 }
513
PromoteIntRes_FREEZE(SDNode * N)514 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
515 SDValue V = GetPromotedInteger(N->getOperand(0));
516 return DAG.getNode(ISD::FREEZE, SDLoc(N),
517 V.getValueType(), V);
518 }
519
PromoteIntRes_BSWAP(SDNode * N)520 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
521 SDValue Op = GetPromotedInteger(N->getOperand(0));
522 EVT OVT = N->getValueType(0);
523 EVT NVT = Op.getValueType();
524 SDLoc dl(N);
525
526 // If the larger BSWAP isn't supported by the target, try to expand now.
527 // If we expand later we'll end up with more operations since we lost the
528 // original type. We only do this for scalars since we have a shuffle
529 // based lowering for vectors in LegalizeVectorOps.
530 if (!OVT.isVector() &&
531 !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) {
532 if (SDValue Res = TLI.expandBSWAP(N, DAG))
533 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
534 }
535
536 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
537 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
538 if (N->getOpcode() == ISD::BSWAP)
539 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
540 ShAmt);
541 SDValue Mask = N->getOperand(1);
542 SDValue EVL = N->getOperand(2);
543 return DAG.getNode(ISD::VP_LSHR, dl, NVT,
544 DAG.getNode(ISD::VP_BSWAP, dl, NVT, Op, Mask, EVL), ShAmt,
545 Mask, EVL);
546 }
547
PromoteIntRes_BITREVERSE(SDNode * N)548 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
549 SDValue Op = GetPromotedInteger(N->getOperand(0));
550 EVT OVT = N->getValueType(0);
551 EVT NVT = Op.getValueType();
552 SDLoc dl(N);
553
554 // If the larger BITREVERSE isn't supported by the target, try to expand now.
555 // If we expand later we'll end up with more operations since we lost the
556 // original type. We only do this for scalars since we have a shuffle
557 // based lowering for vectors in LegalizeVectorOps.
558 if (!OVT.isVector() && OVT.isSimple() &&
559 !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) {
560 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
561 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
562 }
563
564 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
565 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
566 if (N->getOpcode() == ISD::BITREVERSE)
567 return DAG.getNode(ISD::SRL, dl, NVT,
568 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), ShAmt);
569 SDValue Mask = N->getOperand(1);
570 SDValue EVL = N->getOperand(2);
571 return DAG.getNode(ISD::VP_LSHR, dl, NVT,
572 DAG.getNode(ISD::VP_BITREVERSE, dl, NVT, Op, Mask, EVL),
573 ShAmt, Mask, EVL);
574 }
575
PromoteIntRes_BUILD_PAIR(SDNode * N)576 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
577 // The pair element type may be legal, or may not promote to the same type as
578 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
579 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
580 TLI.getTypeToTransformTo(*DAG.getContext(),
581 N->getValueType(0)), JoinIntegers(N->getOperand(0),
582 N->getOperand(1)));
583 }
584
PromoteIntRes_Constant(SDNode * N)585 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
586 EVT VT = N->getValueType(0);
587 // FIXME there is no actual debug info here
588 SDLoc dl(N);
589 // Zero extend things like i1, sign extend everything else. It shouldn't
590 // matter in theory which one we pick, but this tends to give better code?
591 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
592 SDValue Result = DAG.getNode(Opc, dl,
593 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
594 SDValue(N, 0));
595 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
596 return Result;
597 }
598
PromoteIntRes_CTLZ(SDNode * N)599 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
600 EVT OVT = N->getValueType(0);
601 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
602 SDLoc dl(N);
603
604 // If the larger CTLZ isn't supported by the target, try to expand now.
605 // If we expand later we'll end up with more operations since we lost the
606 // original type.
607 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
608 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) &&
609 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) {
610 if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
611 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
612 return Result;
613 }
614 }
615
616 // Zero extend to the promoted type and do the count there.
617 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
618
619 // Subtract off the extra leading bits in the bigger type.
620 SDValue ExtractLeadingBits = DAG.getConstant(
621 NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, NVT);
622 if (!N->isVPOpcode())
623 return DAG.getNode(ISD::SUB, dl, NVT,
624 DAG.getNode(N->getOpcode(), dl, NVT, Op),
625 ExtractLeadingBits);
626 SDValue Mask = N->getOperand(1);
627 SDValue EVL = N->getOperand(2);
628 return DAG.getNode(ISD::VP_SUB, dl, NVT,
629 DAG.getNode(N->getOpcode(), dl, NVT, Op, Mask, EVL),
630 ExtractLeadingBits, Mask, EVL);
631 }
632
PromoteIntRes_CTPOP_PARITY(SDNode * N)633 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
634 EVT OVT = N->getValueType(0);
635 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
636
637 // If the larger CTPOP isn't supported by the target, try to expand now.
638 // If we expand later we'll end up with more operations since we lost the
639 // original type.
640 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
641 // TargetLowering.
642 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) &&
643 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) {
644 if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
645 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result);
646 return Result;
647 }
648 }
649
650 // Zero extend to the promoted type and do the count or parity there.
651 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
652 if (!N->isVPOpcode())
653 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
654 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op,
655 N->getOperand(1), N->getOperand(2));
656 }
657
PromoteIntRes_CTTZ(SDNode * N)658 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
659 SDValue Op = GetPromotedInteger(N->getOperand(0));
660 EVT OVT = N->getValueType(0);
661 EVT NVT = Op.getValueType();
662 SDLoc dl(N);
663
664 // If the larger CTTZ isn't supported by the target, try to expand now.
665 // If we expand later we'll end up with more operations since we lost the
666 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the
667 // larger type.
668 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
669 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) &&
670 !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) &&
671 !TLI.isOperationLegal(ISD::CTPOP, NVT) &&
672 !TLI.isOperationLegal(ISD::CTLZ, NVT)) {
673 if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
674 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
675 return Result;
676 }
677 }
678
679 if (N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::VP_CTTZ) {
680 // The count is the same in the promoted type except if the original
681 // value was zero. This can be handled by setting the bit just off
682 // the top of the original type.
683 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
684 OVT.getScalarSizeInBits());
685 if (N->getOpcode() == ISD::CTTZ)
686 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
687 else
688 Op =
689 DAG.getNode(ISD::VP_OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT),
690 N->getOperand(1), N->getOperand(2));
691 }
692 if (!N->isVPOpcode())
693 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
694 return DAG.getNode(N->getOpcode(), dl, NVT, Op, N->getOperand(1),
695 N->getOperand(2));
696 }
697
PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode * N)698 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
699 SDLoc dl(N);
700 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
701
702 SDValue Op0 = N->getOperand(0);
703 SDValue Op1 = N->getOperand(1);
704
705 // If the input also needs to be promoted, do that first so we can get a
706 // get a good idea for the output type.
707 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
708 == TargetLowering::TypePromoteInteger) {
709 SDValue In = GetPromotedInteger(Op0);
710
711 // If the new type is larger than NVT, use it. We probably won't need to
712 // promote it again.
713 EVT SVT = In.getValueType().getScalarType();
714 if (SVT.bitsGE(NVT)) {
715 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
716 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
717 }
718 }
719
720 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
721 }
722
PromoteIntRes_FP_TO_XINT(SDNode * N)723 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
725 unsigned NewOpc = N->getOpcode();
726 SDLoc dl(N);
727
728 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
729 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
730 // and SINT conversions are Custom, there is no way to tell which is
731 // preferable. We choose SINT because that's the right thing on PPC.)
732 if (N->getOpcode() == ISD::FP_TO_UINT &&
733 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
734 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
735 NewOpc = ISD::FP_TO_SINT;
736
737 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
738 !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
739 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
740 NewOpc = ISD::STRICT_FP_TO_SINT;
741
742 if (N->getOpcode() == ISD::VP_FP_TO_UINT &&
743 !TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) &&
744 TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT))
745 NewOpc = ISD::VP_FP_TO_SINT;
746
747 SDValue Res;
748 if (N->isStrictFPOpcode()) {
749 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
750 {N->getOperand(0), N->getOperand(1)});
751 // Legalize the chain result - switch anything that used the old chain to
752 // use the new one.
753 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
754 } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
755 Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1),
756 N->getOperand(2)});
757 } else {
758 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
759 }
760
761 // Assert that the converted value fits in the original type. If it doesn't
762 // (eg: because the value being converted is too big), then the result of the
763 // original operation was undefined anyway, so the assert is still correct.
764 //
765 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
766 // before legalization: fp-to-uint16, 65534. -> 0xfffe
767 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
768 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
769 N->getOpcode() == ISD::STRICT_FP_TO_UINT ||
770 N->getOpcode() == ISD::VP_FP_TO_UINT)
771 ? ISD::AssertZext
772 : ISD::AssertSext,
773 dl, NVT, Res,
774 DAG.getValueType(N->getValueType(0).getScalarType()));
775 }
776
PromoteIntRes_FP_TO_XINT_SAT(SDNode * N)777 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
778 // Promote the result type, while keeping the original width in Op1.
779 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
780 SDLoc dl(N);
781 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
782 N->getOperand(1));
783 }
784
PromoteIntRes_FP_TO_FP16_BF16(SDNode * N)785 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) {
786 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
787 SDLoc dl(N);
788
789 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
790 }
791
PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode * N)792 SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) {
793 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
794 SDLoc dl(N);
795
796 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
797 N->getOperand(0), N->getOperand(1));
798 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
799 return Res;
800 }
801
PromoteIntRes_XRINT(SDNode * N)802 SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) {
803 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
804 SDLoc dl(N);
805 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
806 }
807
PromoteIntRes_GET_ROUNDING(SDNode * N)808 SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) {
809 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
810 SDLoc dl(N);
811
812 SDValue Res =
813 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
814
815 // Legalize the chain result - switch anything that used the old chain to
816 // use the new one.
817 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
818 return Res;
819 }
820
PromoteIntRes_INT_EXTEND(SDNode * N)821 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
822 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
823 SDLoc dl(N);
824
825 if (getTypeAction(N->getOperand(0).getValueType())
826 == TargetLowering::TypePromoteInteger) {
827 SDValue Res = GetPromotedInteger(N->getOperand(0));
828 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
829
830 // If the result and operand types are the same after promotion, simplify
831 // to an in-register extension. Unless this is a VP_*_EXTEND.
832 if (NVT == Res.getValueType() && N->getNumOperands() == 1) {
833 // The high bits are not guaranteed to be anything. Insert an extend.
834 if (N->getOpcode() == ISD::SIGN_EXTEND)
835 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
836 DAG.getValueType(N->getOperand(0).getValueType()));
837 if (N->getOpcode() == ISD::ZERO_EXTEND)
838 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
839 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
840 return Res;
841 }
842 }
843
844 // Otherwise, just extend the original operand all the way to the larger type.
845 if (N->getNumOperands() != 1) {
846 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
847 assert(N->isVPOpcode() && "Expected VP opcode");
848 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
849 N->getOperand(1), N->getOperand(2));
850 }
851 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
852 }
853
PromoteIntRes_LOAD(LoadSDNode * N)854 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
855 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
856 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
857 ISD::LoadExtType ExtType =
858 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
859 SDLoc dl(N);
860 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
861 N->getMemoryVT(), N->getMemOperand());
862
863 // Legalize the chain result - switch anything that used the old chain to
864 // use the new one.
865 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
866 return Res;
867 }
868
PromoteIntRes_MLOAD(MaskedLoadSDNode * N)869 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
870 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
871 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
872
873 ISD::LoadExtType ExtType = N->getExtensionType();
874 if (ExtType == ISD::NON_EXTLOAD)
875 ExtType = ISD::EXTLOAD;
876
877 SDLoc dl(N);
878 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
879 N->getOffset(), N->getMask(), ExtPassThru,
880 N->getMemoryVT(), N->getMemOperand(),
881 N->getAddressingMode(), ExtType,
882 N->isExpandingLoad());
883 // Legalize the chain result - switch anything that used the old chain to
884 // use the new one.
885 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
886 return Res;
887 }
888
PromoteIntRes_MGATHER(MaskedGatherSDNode * N)889 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
890 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
891 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
892 assert(NVT == ExtPassThru.getValueType() &&
893 "Gather result type and the passThru argument type should be the same");
894
895 ISD::LoadExtType ExtType = N->getExtensionType();
896 if (ExtType == ISD::NON_EXTLOAD)
897 ExtType = ISD::EXTLOAD;
898
899 SDLoc dl(N);
900 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
901 N->getIndex(), N->getScale() };
902 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
903 N->getMemoryVT(), dl, Ops,
904 N->getMemOperand(), N->getIndexType(),
905 ExtType);
906 // Legalize the chain result - switch anything that used the old chain to
907 // use the new one.
908 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
909 return Res;
910 }
911
912 /// Promote the overflow flag of an overflowing arithmetic node.
PromoteIntRes_Overflow(SDNode * N)913 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
914 // Change the return type of the boolean result while obeying
915 // getSetCCResultType.
916 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
917 EVT VT = N->getValueType(0);
918 EVT SVT = getSetCCResultType(VT);
919 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
920 unsigned NumOps = N->getNumOperands();
921 assert(NumOps <= 3 && "Too many operands");
922 if (NumOps == 3)
923 Ops[2] = N->getOperand(2);
924
925 SDLoc dl(N);
926 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
927 ArrayRef(Ops, NumOps));
928
929 // Modified the sum result - switch anything that used the old sum to use
930 // the new one.
931 ReplaceValueWith(SDValue(N, 0), Res);
932
933 // Convert to the expected type.
934 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
935 }
936
PromoteIntRes_ADDSUBSHLSAT(SDNode * N)937 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
938 // If the promoted type is legal, we can convert this to:
939 // 1. ANY_EXTEND iN to iM
940 // 2. SHL by M-N
941 // 3. [US][ADD|SUB|SHL]SAT
942 // 4. L/ASHR by M-N
943 // Else it is more efficient to convert this to a min and a max
944 // operation in the higher precision arithmetic.
945 SDLoc dl(N);
946 SDValue Op1 = N->getOperand(0);
947 SDValue Op2 = N->getOperand(1);
948 unsigned OldBits = Op1.getScalarValueSizeInBits();
949
950 unsigned Opcode = N->getOpcode();
951 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
952
953 SDValue Op1Promoted, Op2Promoted;
954 if (IsShift) {
955 Op1Promoted = GetPromotedInteger(Op1);
956 Op2Promoted = ZExtPromotedInteger(Op2);
957 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
958 Op1Promoted = ZExtPromotedInteger(Op1);
959 Op2Promoted = ZExtPromotedInteger(Op2);
960 } else {
961 Op1Promoted = SExtPromotedInteger(Op1);
962 Op2Promoted = SExtPromotedInteger(Op2);
963 }
964 EVT PromotedType = Op1Promoted.getValueType();
965 unsigned NewBits = PromotedType.getScalarSizeInBits();
966
967 if (Opcode == ISD::UADDSAT) {
968 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
969 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
970 SDValue Add =
971 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
972 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
973 }
974
975 // USUBSAT can always be promoted as long as we have zero-extended the args.
976 if (Opcode == ISD::USUBSAT)
977 return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
978 Op2Promoted);
979
980 // Shift cannot use a min/max expansion, we can't detect overflow if all of
981 // the bits have been shifted out.
982 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) {
983 unsigned ShiftOp;
984 switch (Opcode) {
985 case ISD::SADDSAT:
986 case ISD::SSUBSAT:
987 case ISD::SSHLSAT:
988 ShiftOp = ISD::SRA;
989 break;
990 case ISD::USHLSAT:
991 ShiftOp = ISD::SRL;
992 break;
993 default:
994 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
995 "addition, subtraction or left shift");
996 }
997
998 unsigned SHLAmount = NewBits - OldBits;
999 SDValue ShiftAmount =
1000 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
1001 Op1Promoted =
1002 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
1003 if (!IsShift)
1004 Op2Promoted =
1005 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1006
1007 SDValue Result =
1008 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1009 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1010 }
1011
1012 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
1013 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
1014 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
1015 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
1016 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
1017 SDValue Result =
1018 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1019 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
1020 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
1021 return Result;
1022 }
1023
PromoteIntRes_MULFIX(SDNode * N)1024 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
1025 // Can just promote the operands then continue with operation.
1026 SDLoc dl(N);
1027 SDValue Op1Promoted, Op2Promoted;
1028 bool Signed =
1029 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
1030 bool Saturating =
1031 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
1032 if (Signed) {
1033 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1034 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1035 } else {
1036 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1037 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1038 }
1039 EVT OldType = N->getOperand(0).getValueType();
1040 EVT PromotedType = Op1Promoted.getValueType();
1041 unsigned DiffSize =
1042 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
1043
1044 if (Saturating) {
1045 // Promoting the operand and result values changes the saturation width,
1046 // which is extends the values that we clamp to on saturation. This could be
1047 // resolved by shifting one of the operands the same amount, which would
1048 // also shift the result we compare against, then shifting back.
1049 Op1Promoted =
1050 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1051 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1052 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1053 Op2Promoted, N->getOperand(2));
1054 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
1055 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
1056 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1057 }
1058 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1059 N->getOperand(2));
1060 }
1061
SaturateWidenedDIVFIX(SDValue V,SDLoc & dl,unsigned SatW,bool Signed,const TargetLowering & TLI,SelectionDAG & DAG)1062 static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
1063 unsigned SatW, bool Signed,
1064 const TargetLowering &TLI,
1065 SelectionDAG &DAG) {
1066 EVT VT = V.getValueType();
1067 unsigned VTW = VT.getScalarSizeInBits();
1068
1069 if (!Signed) {
1070 // Saturate to the unsigned maximum by getting the minimum of V and the
1071 // maximum.
1072 return DAG.getNode(ISD::UMIN, dl, VT, V,
1073 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
1074 dl, VT));
1075 }
1076
1077 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
1078 // signed minimum of it and V.
1079 V = DAG.getNode(ISD::SMIN, dl, VT, V,
1080 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
1081 dl, VT));
1082 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
1083 // signed maximum of it and V.
1084 V = DAG.getNode(ISD::SMAX, dl, VT, V,
1085 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
1086 dl, VT));
1087 return V;
1088 }
1089
earlyExpandDIVFIX(SDNode * N,SDValue LHS,SDValue RHS,unsigned Scale,const TargetLowering & TLI,SelectionDAG & DAG,unsigned SatW=0)1090 static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
1091 unsigned Scale, const TargetLowering &TLI,
1092 SelectionDAG &DAG, unsigned SatW = 0) {
1093 EVT VT = LHS.getValueType();
1094 unsigned VTSize = VT.getScalarSizeInBits();
1095 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1096 N->getOpcode() == ISD::SDIVFIXSAT;
1097 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1098 N->getOpcode() == ISD::UDIVFIXSAT;
1099
1100 SDLoc dl(N);
1101 // Widen the types by a factor of two. This is guaranteed to expand, since it
1102 // will always have enough high bits in the LHS to shift into.
1103 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
1104 if (VT.isVector())
1105 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
1106 VT.getVectorElementCount());
1107 LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT);
1108 RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT);
1109 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
1110 DAG);
1111 assert(Res && "Expanding DIVFIX with wide type failed?");
1112 if (Saturating) {
1113 // If the caller has told us to saturate at something less, use that width
1114 // instead of the type before doubling. However, it cannot be more than
1115 // what we just widened!
1116 assert(SatW <= VTSize &&
1117 "Tried to saturate to more than the original type?");
1118 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1119 TLI, DAG);
1120 }
1121 return DAG.getZExtOrTrunc(Res, dl, VT);
1122 }
1123
PromoteIntRes_DIVFIX(SDNode * N)1124 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1125 SDLoc dl(N);
1126 SDValue Op1Promoted, Op2Promoted;
1127 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1128 N->getOpcode() == ISD::SDIVFIXSAT;
1129 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1130 N->getOpcode() == ISD::UDIVFIXSAT;
1131 if (Signed) {
1132 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1133 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1134 } else {
1135 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1136 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1137 }
1138 EVT PromotedType = Op1Promoted.getValueType();
1139 unsigned Scale = N->getConstantOperandVal(2);
1140
1141 // If the type is already legal and the operation is legal in that type, we
1142 // should not early expand.
1143 if (TLI.isTypeLegal(PromotedType)) {
1144 TargetLowering::LegalizeAction Action =
1145 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1146 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1147 unsigned Diff = PromotedType.getScalarSizeInBits() -
1148 N->getValueType(0).getScalarSizeInBits();
1149 if (Saturating)
1150 Op1Promoted =
1151 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1152 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1153 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1154 Op2Promoted, N->getOperand(2));
1155 if (Saturating)
1156 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1157 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1158 return Res;
1159 }
1160 }
1161
1162 // See if we can perform the division in this type without expanding.
1163 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1164 Op2Promoted, Scale, DAG)) {
1165 if (Saturating)
1166 Res = SaturateWidenedDIVFIX(Res, dl,
1167 N->getValueType(0).getScalarSizeInBits(),
1168 Signed, TLI, DAG);
1169 return Res;
1170 }
1171 // If we cannot, expand it to twice the type width. If we are saturating, give
1172 // it the original width as a saturating width so we don't need to emit
1173 // two saturations.
1174 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1175 N->getValueType(0).getScalarSizeInBits());
1176 }
1177
PromoteIntRes_SADDSUBO(SDNode * N,unsigned ResNo)1178 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1179 if (ResNo == 1)
1180 return PromoteIntRes_Overflow(N);
1181
1182 // The operation overflowed iff the result in the larger type is not the
1183 // sign extension of its truncation to the original type.
1184 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1185 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1186 EVT OVT = N->getOperand(0).getValueType();
1187 EVT NVT = LHS.getValueType();
1188 SDLoc dl(N);
1189
1190 // Do the arithmetic in the larger type.
1191 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1192 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1193
1194 // Calculate the overflow flag: sign extend the arithmetic result from
1195 // the original type.
1196 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1197 DAG.getValueType(OVT));
1198 // Overflowed if and only if this is not equal to Res.
1199 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1200
1201 // Use the calculated overflow everywhere.
1202 ReplaceValueWith(SDValue(N, 1), Ofl);
1203
1204 return Res;
1205 }
1206
PromoteIntRes_Select(SDNode * N)1207 SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
1208 SDValue Mask = N->getOperand(0);
1209
1210 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1211 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1212
1213 unsigned Opcode = N->getOpcode();
1214 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1215 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS,
1216 N->getOperand(3));
1217 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS);
1218 }
1219
PromoteIntRes_SELECT_CC(SDNode * N)1220 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1221 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1222 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1223 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1224 LHS.getValueType(), N->getOperand(0),
1225 N->getOperand(1), LHS, RHS, N->getOperand(4));
1226 }
1227
PromoteIntRes_SETCC(SDNode * N)1228 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1229 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1230 EVT InVT = N->getOperand(OpNo).getValueType();
1231 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1232
1233 EVT SVT = getSetCCResultType(InVT);
1234
1235 // If we got back a type that needs to be promoted, this likely means the
1236 // the input type also needs to be promoted. So get the promoted type for
1237 // the input and try the query again.
1238 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1239 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1240 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1241 SVT = getSetCCResultType(InVT);
1242 } else {
1243 // Input type isn't promoted, just use the default promoted type.
1244 SVT = NVT;
1245 }
1246 }
1247
1248 SDLoc dl(N);
1249 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1250 "Vector compare must return a vector result!");
1251
1252 // Get the SETCC result using the canonical SETCC type.
1253 SDValue SetCC;
1254 if (N->isStrictFPOpcode()) {
1255 SDVTList VTs = DAG.getVTList({SVT, MVT::Other});
1256 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1257 N->getOperand(2), N->getOperand(3)};
1258 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags());
1259 // Legalize the chain result - switch anything that used the old chain to
1260 // use the new one.
1261 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1262 } else
1263 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1264 N->getOperand(1), N->getOperand(2), N->getFlags());
1265
1266 // Convert to the expected type.
1267 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1268 }
1269
PromoteIntRes_IS_FPCLASS(SDNode * N)1270 SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
1271 SDLoc DL(N);
1272 SDValue Arg = N->getOperand(0);
1273 SDValue Test = N->getOperand(1);
1274 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1275 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test);
1276 }
1277
PromoteIntRes_FFREXP(SDNode * N)1278 SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) {
1279 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
1280 EVT VT = N->getValueType(0);
1281
1282 SDLoc dl(N);
1283 SDValue Res =
1284 DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0));
1285
1286 ReplaceValueWith(SDValue(N, 0), Res);
1287 return Res.getValue(1);
1288 }
1289
PromoteIntRes_SHL(SDNode * N)1290 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1291 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1292 SDValue RHS = N->getOperand(1);
1293 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1294 RHS = ZExtPromotedInteger(RHS);
1295 if (N->getOpcode() != ISD::VP_SHL)
1296 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1297 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1298 N->getOperand(2), N->getOperand(3));
1299 }
1300
PromoteIntRes_SIGN_EXTEND_INREG(SDNode * N)1301 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1302 SDValue Op = GetPromotedInteger(N->getOperand(0));
1303 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1304 Op.getValueType(), Op, N->getOperand(1));
1305 }
1306
PromoteIntRes_SimpleIntBinOp(SDNode * N)1307 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1308 // The input may have strange things in the top bits of the registers, but
1309 // these operations don't care. They may have weird bits going out, but
1310 // that too is okay if they are integer operations.
1311 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1312 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1313 if (N->getNumOperands() == 2)
1314 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1315 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1316 assert(N->isVPOpcode() && "Expected VP opcode");
1317 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1318 N->getOperand(2), N->getOperand(3));
1319 }
1320
PromoteIntRes_SExtIntBinOp(SDNode * N)1321 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1322 // Sign extend the input.
1323 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1324 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1325 if (N->getNumOperands() == 2)
1326 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1327 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1328 assert(N->isVPOpcode() && "Expected VP opcode");
1329 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1330 N->getOperand(2), N->getOperand(3));
1331 }
1332
PromoteIntRes_ZExtIntBinOp(SDNode * N)1333 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1334 // Zero extend the input.
1335 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1336 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1337 if (N->getNumOperands() == 2)
1338 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1339 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1340 assert(N->isVPOpcode() && "Expected VP opcode");
1341 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1342 N->getOperand(2), N->getOperand(3));
1343 }
1344
PromoteIntRes_UMINUMAX(SDNode * N)1345 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1346 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1347 // whatever is best for the target.
1348 SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0));
1349 SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1));
1350 return DAG.getNode(N->getOpcode(), SDLoc(N),
1351 LHS.getValueType(), LHS, RHS);
1352 }
1353
PromoteIntRes_SRA(SDNode * N)1354 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1355 // The input value must be properly sign extended.
1356 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1357 SDValue RHS = N->getOperand(1);
1358 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1359 RHS = ZExtPromotedInteger(RHS);
1360 if (N->getOpcode() != ISD::VP_ASHR)
1361 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1362 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1363 N->getOperand(2), N->getOperand(3));
1364 }
1365
PromoteIntRes_SRL(SDNode * N)1366 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1367 // The input value must be properly zero extended.
1368 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1369 SDValue RHS = N->getOperand(1);
1370 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1371 RHS = ZExtPromotedInteger(RHS);
1372 if (N->getOpcode() != ISD::VP_LSHR)
1373 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1374 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1375 N->getOperand(2), N->getOperand(3));
1376 }
1377
PromoteIntRes_Rotate(SDNode * N)1378 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1379 // Lower the rotate to shifts and ORs which can be promoted.
1380 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
1381 ReplaceValueWith(SDValue(N, 0), Res);
1382 return SDValue();
1383 }
1384
PromoteIntRes_FunnelShift(SDNode * N)1385 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1386 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1387 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1388 SDValue Amt = N->getOperand(2);
1389 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1390 Amt = ZExtPromotedInteger(Amt);
1391 EVT AmtVT = Amt.getValueType();
1392
1393 SDLoc DL(N);
1394 EVT OldVT = N->getOperand(0).getValueType();
1395 EVT VT = Lo.getValueType();
1396 unsigned Opcode = N->getOpcode();
1397 bool IsFSHR = Opcode == ISD::FSHR;
1398 unsigned OldBits = OldVT.getScalarSizeInBits();
1399 unsigned NewBits = VT.getScalarSizeInBits();
1400
1401 // Amount has to be interpreted modulo the old bit width.
1402 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt,
1403 DAG.getConstant(OldBits, DL, AmtVT));
1404
1405 // If the promoted type is twice the size (or more), then we use the
1406 // traditional funnel 'double' shift codegen. This isn't necessary if the
1407 // shift amount is constant.
1408 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1409 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1410 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1411 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1412 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1413 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1414 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1415 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1416 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
1417 if (!IsFSHR)
1418 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1419 return Res;
1420 }
1421
1422 // Shift Lo up to occupy the upper bits of the promoted type.
1423 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1424 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1425
1426 // Increase Amount to shift the result into the lower bits of the promoted
1427 // type.
1428 if (IsFSHR)
1429 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset);
1430
1431 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
1432 }
1433
1434 // A vp version of PromoteIntRes_FunnelShift.
PromoteIntRes_VPFunnelShift(SDNode * N)1435 SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) {
1436 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1437 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1438 SDValue Amt = N->getOperand(2);
1439 SDValue Mask = N->getOperand(3);
1440 SDValue EVL = N->getOperand(4);
1441 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1442 Amt = ZExtPromotedInteger(Amt);
1443 EVT AmtVT = Amt.getValueType();
1444
1445 SDLoc DL(N);
1446 EVT OldVT = N->getOperand(0).getValueType();
1447 EVT VT = Lo.getValueType();
1448 unsigned Opcode = N->getOpcode();
1449 bool IsFSHR = Opcode == ISD::VP_FSHR;
1450 unsigned OldBits = OldVT.getScalarSizeInBits();
1451 unsigned NewBits = VT.getScalarSizeInBits();
1452
1453 // Amount has to be interpreted modulo the old bit width.
1454 Amt = DAG.getNode(ISD::VP_UREM, DL, AmtVT, Amt,
1455 DAG.getConstant(OldBits, DL, AmtVT), Mask, EVL);
1456
1457 // If the promoted type is twice the size (or more), then we use the
1458 // traditional funnel 'double' shift codegen. This isn't necessary if the
1459 // shift amount is constant.
1460 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1461 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1462 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1463 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1464 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1465 Hi = DAG.getNode(ISD::VP_SHL, DL, VT, Hi, HiShift, Mask, EVL);
1466 // FIXME: Replace it by vp operations.
1467 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1468 SDValue Res = DAG.getNode(ISD::VP_OR, DL, VT, Hi, Lo, Mask, EVL);
1469 Res = DAG.getNode(IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL, DL, VT, Res, Amt,
1470 Mask, EVL);
1471 if (!IsFSHR)
1472 Res = DAG.getNode(ISD::VP_LSHR, DL, VT, Res, HiShift, Mask, EVL);
1473 return Res;
1474 }
1475
1476 // Shift Lo up to occupy the upper bits of the promoted type.
1477 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1478 Lo = DAG.getNode(ISD::VP_SHL, DL, VT, Lo, ShiftOffset, Mask, EVL);
1479
1480 // Increase Amount to shift the result into the lower bits of the promoted
1481 // type.
1482 if (IsFSHR)
1483 Amt = DAG.getNode(ISD::VP_ADD, DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1484
1485 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt, Mask, EVL);
1486 }
1487
PromoteIntRes_TRUNCATE(SDNode * N)1488 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1490 SDValue Res;
1491 SDValue InOp = N->getOperand(0);
1492 SDLoc dl(N);
1493
1494 switch (getTypeAction(InOp.getValueType())) {
1495 default: llvm_unreachable("Unknown type action!");
1496 case TargetLowering::TypeLegal:
1497 case TargetLowering::TypeExpandInteger:
1498 Res = InOp;
1499 break;
1500 case TargetLowering::TypePromoteInteger:
1501 Res = GetPromotedInteger(InOp);
1502 break;
1503 case TargetLowering::TypeSplitVector: {
1504 EVT InVT = InOp.getValueType();
1505 assert(InVT.isVector() && "Cannot split scalar types");
1506 ElementCount NumElts = InVT.getVectorElementCount();
1507 assert(NumElts == NVT.getVectorElementCount() &&
1508 "Dst and Src must have the same number of elements");
1509 assert(isPowerOf2_32(NumElts.getKnownMinValue()) &&
1510 "Promoted vector type must be a power of two");
1511
1512 SDValue EOp1, EOp2;
1513 GetSplitVector(InOp, EOp1, EOp2);
1514
1515 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1516 NumElts.divideCoefficientBy(2));
1517 if (N->getOpcode() == ISD::TRUNCATE) {
1518 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1519 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1520 } else {
1521 assert(N->getOpcode() == ISD::VP_TRUNCATE &&
1522 "Expected VP_TRUNCATE opcode");
1523 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1524 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
1525 std::tie(EVLLo, EVLHi) =
1526 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
1527 EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1528 EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1529 }
1530 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1531 }
1532 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some
1533 // targets.
1534 case TargetLowering::TypeWidenVector: {
1535 SDValue WideInOp = GetWidenedVector(InOp);
1536
1537 // Truncate widened InOp.
1538 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1539 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1540 N->getValueType(0).getScalarType(), NumElem);
1541 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1542
1543 // Zero extend so that the elements are of same type as those of NVT
1544 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1545 NumElem);
1546 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1547
1548 // Extract the low NVT subvector.
1549 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1550 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1551 }
1552 }
1553
1554 // Truncate to NVT instead of VT
1555 if (N->getOpcode() == ISD::VP_TRUNCATE)
1556 return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1),
1557 N->getOperand(2));
1558 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1559 }
1560
PromoteIntRes_UADDSUBO(SDNode * N,unsigned ResNo)1561 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1562 if (ResNo == 1)
1563 return PromoteIntRes_Overflow(N);
1564
1565 // The operation overflowed iff the result in the larger type is not the
1566 // zero extension of its truncation to the original type.
1567 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1568 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1569 EVT OVT = N->getOperand(0).getValueType();
1570 EVT NVT = LHS.getValueType();
1571 SDLoc dl(N);
1572
1573 // Do the arithmetic in the larger type.
1574 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1575 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1576
1577 // Calculate the overflow flag: zero extend the arithmetic result from
1578 // the original type.
1579 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1580 // Overflowed if and only if this is not equal to Res.
1581 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1582
1583 // Use the calculated overflow everywhere.
1584 ReplaceValueWith(SDValue(N, 1), Ofl);
1585
1586 return Res;
1587 }
1588
1589 // Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that
1590 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
1591 // the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean.
PromoteIntRes_UADDSUBO_CARRY(SDNode * N,unsigned ResNo)1592 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N,
1593 unsigned ResNo) {
1594 if (ResNo == 1)
1595 return PromoteIntRes_Overflow(N);
1596
1597 // We need to sign-extend the operands so the carry value computed by the
1598 // wide operation will be equivalent to the carry value computed by the
1599 // narrow operation.
1600 // An UADDO_CARRY can generate carry only if any of the operands has its
1601 // most significant bit set. Sign extension propagates the most significant
1602 // bit into the higher bits which means the extra bit that the narrow
1603 // addition would need (i.e. the carry) will be propagated through the higher
1604 // bits of the wide addition.
1605 // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will
1606 // be preserved by sign extension.
1607 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1608 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1609
1610 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1611
1612 // Do the arithmetic in the wide type.
1613 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1614 LHS, RHS, N->getOperand(2));
1615
1616 // Update the users of the original carry/borrow value.
1617 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1618
1619 return SDValue(Res.getNode(), 0);
1620 }
1621
PromoteIntRes_SADDSUBO_CARRY(SDNode * N,unsigned ResNo)1622 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1623 unsigned ResNo) {
1624 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1625 return PromoteIntRes_Overflow(N);
1626 }
1627
PromoteIntRes_ABS(SDNode * N)1628 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1629 EVT OVT = N->getValueType(0);
1630 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1631
1632 // If a larger ABS or SMAX isn't supported by the target, try to expand now.
1633 // If we expand later we'll end up sign extending more than just the sra input
1634 // in sra+xor+sub expansion.
1635 if (!OVT.isVector() &&
1636 !TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) &&
1637 !TLI.isOperationLegal(ISD::SMAX, NVT)) {
1638 if (SDValue Res = TLI.expandABS(N, DAG))
1639 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res);
1640 }
1641
1642 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1643 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1644 }
1645
PromoteIntRes_XMULO(SDNode * N,unsigned ResNo)1646 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1647 // Promote the overflow bit trivially.
1648 if (ResNo == 1)
1649 return PromoteIntRes_Overflow(N);
1650
1651 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1652 SDLoc DL(N);
1653 EVT SmallVT = LHS.getValueType();
1654
1655 // To determine if the result overflowed in a larger type, we extend the
1656 // input to the larger type, do the multiply (checking if it overflows),
1657 // then also check the high bits of the result to see if overflow happened
1658 // there.
1659 if (N->getOpcode() == ISD::SMULO) {
1660 LHS = SExtPromotedInteger(LHS);
1661 RHS = SExtPromotedInteger(RHS);
1662 } else {
1663 LHS = ZExtPromotedInteger(LHS);
1664 RHS = ZExtPromotedInteger(RHS);
1665 }
1666 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1667 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1668
1669 // Overflow occurred if it occurred in the larger type, or if the high part
1670 // of the result does not zero/sign-extend the low part. Check this second
1671 // possibility first.
1672 SDValue Overflow;
1673 if (N->getOpcode() == ISD::UMULO) {
1674 // Unsigned overflow occurred if the high part is non-zero.
1675 unsigned Shift = SmallVT.getScalarSizeInBits();
1676 SDValue Hi =
1677 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1678 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL));
1679 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1680 DAG.getConstant(0, DL, Hi.getValueType()),
1681 ISD::SETNE);
1682 } else {
1683 // Signed overflow occurred if the high part does not sign extend the low.
1684 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1685 Mul, DAG.getValueType(SmallVT));
1686 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1687 }
1688
1689 // The only other way for overflow to occur is if the multiplication in the
1690 // larger type itself overflowed.
1691 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1692 SDValue(Mul.getNode(), 1));
1693
1694 // Use the calculated overflow everywhere.
1695 ReplaceValueWith(SDValue(N, 1), Overflow);
1696 return Mul;
1697 }
1698
PromoteIntRes_UNDEF(SDNode * N)1699 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1700 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1701 N->getValueType(0)));
1702 }
1703
PromoteIntRes_VSCALE(SDNode * N)1704 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1705 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1706
1707 const APInt &MulImm = N->getConstantOperandAPInt(0);
1708 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits()));
1709 }
1710
PromoteIntRes_VAARG(SDNode * N)1711 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1712 SDValue Chain = N->getOperand(0); // Get the chain.
1713 SDValue Ptr = N->getOperand(1); // Get the pointer.
1714 EVT VT = N->getValueType(0);
1715 SDLoc dl(N);
1716
1717 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1718 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1719 // The argument is passed as NumRegs registers of type RegVT.
1720
1721 SmallVector<SDValue, 8> Parts(NumRegs);
1722 for (unsigned i = 0; i < NumRegs; ++i) {
1723 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1724 N->getConstantOperandVal(3));
1725 Chain = Parts[i].getValue(1);
1726 }
1727
1728 // Handle endianness of the load.
1729 if (DAG.getDataLayout().isBigEndian())
1730 std::reverse(Parts.begin(), Parts.end());
1731
1732 // Assemble the parts in the promoted type.
1733 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1734 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1735 for (unsigned i = 1; i < NumRegs; ++i) {
1736 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1737 // Shift it to the right position and "or" it in.
1738 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1739 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1740 TLI.getPointerTy(DAG.getDataLayout())));
1741 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1742 }
1743
1744 // Modified the chain result - switch anything that used the old chain to
1745 // use the new one.
1746 ReplaceValueWith(SDValue(N, 1), Chain);
1747
1748 return Res;
1749 }
1750
1751 //===----------------------------------------------------------------------===//
1752 // Integer Operand Promotion
1753 //===----------------------------------------------------------------------===//
1754
1755 /// PromoteIntegerOperand - This method is called when the specified operand of
1756 /// the specified node is found to need promotion. At this point, all of the
1757 /// result types of the node are known to be legal, but other operands of the
1758 /// node may need promotion or expansion as well as the specified one.
PromoteIntegerOperand(SDNode * N,unsigned OpNo)1759 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1760 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG));
1761 SDValue Res = SDValue();
1762 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1763 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1764 return false;
1765 }
1766
1767 switch (N->getOpcode()) {
1768 default:
1769 #ifndef NDEBUG
1770 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1771 N->dump(&DAG); dbgs() << "\n";
1772 #endif
1773 report_fatal_error("Do not know how to promote this operator's operand!");
1774
1775 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1776 case ISD::ATOMIC_STORE:
1777 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1778 break;
1779 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1780 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1781 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1782 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1783 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1784 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1785 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1786 case ISD::INSERT_VECTOR_ELT:
1787 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
1788 break;
1789 case ISD::SPLAT_VECTOR:
1790 case ISD::SCALAR_TO_VECTOR:
1791 Res = PromoteIntOp_ScalarOp(N);
1792 break;
1793 case ISD::VSELECT:
1794 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1795 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1796 case ISD::VP_SETCC:
1797 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1798 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1799 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break;
1800 case ISD::VP_SINT_TO_FP:
1801 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1802 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1803 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1804 OpNo); break;
1805 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1806 OpNo); break;
1807 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1808 OpNo); break;
1809 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1810 OpNo); break;
1811 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1812 OpNo); break;
1813 case ISD::VP_TRUNCATE:
1814 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1815 case ISD::BF16_TO_FP:
1816 case ISD::FP16_TO_FP:
1817 case ISD::VP_UINT_TO_FP:
1818 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1819 case ISD::STRICT_FP16_TO_FP:
1820 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1821 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1822 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break;
1823 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1824 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
1825
1826 case ISD::SHL:
1827 case ISD::SRA:
1828 case ISD::SRL:
1829 case ISD::ROTL:
1830 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1831
1832 case ISD::FSHL:
1833 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break;
1834
1835 case ISD::SADDO_CARRY:
1836 case ISD::SSUBO_CARRY:
1837 case ISD::UADDO_CARRY:
1838 case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break;
1839
1840 case ISD::FRAMEADDR:
1841 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1842
1843 case ISD::SMULFIX:
1844 case ISD::SMULFIXSAT:
1845 case ISD::UMULFIX:
1846 case ISD::UMULFIXSAT:
1847 case ISD::SDIVFIX:
1848 case ISD::SDIVFIXSAT:
1849 case ISD::UDIVFIX:
1850 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1851 case ISD::FPOWI:
1852 case ISD::STRICT_FPOWI:
1853 case ISD::FLDEXP:
1854 case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break;
1855 case ISD::VECREDUCE_ADD:
1856 case ISD::VECREDUCE_MUL:
1857 case ISD::VECREDUCE_AND:
1858 case ISD::VECREDUCE_OR:
1859 case ISD::VECREDUCE_XOR:
1860 case ISD::VECREDUCE_SMAX:
1861 case ISD::VECREDUCE_SMIN:
1862 case ISD::VECREDUCE_UMAX:
1863 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1864 case ISD::VP_REDUCE_ADD:
1865 case ISD::VP_REDUCE_MUL:
1866 case ISD::VP_REDUCE_AND:
1867 case ISD::VP_REDUCE_OR:
1868 case ISD::VP_REDUCE_XOR:
1869 case ISD::VP_REDUCE_SMAX:
1870 case ISD::VP_REDUCE_SMIN:
1871 case ISD::VP_REDUCE_UMAX:
1872 case ISD::VP_REDUCE_UMIN:
1873 Res = PromoteIntOp_VP_REDUCE(N, OpNo);
1874 break;
1875
1876 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
1877 case ISD::STACKMAP:
1878 Res = PromoteIntOp_STACKMAP(N, OpNo);
1879 break;
1880 case ISD::PATCHPOINT:
1881 Res = PromoteIntOp_PATCHPOINT(N, OpNo);
1882 break;
1883 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1884 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1885 Res = PromoteIntOp_VP_STRIDED(N, OpNo);
1886 break;
1887 case ISD::EXPERIMENTAL_VP_SPLICE:
1888 Res = PromoteIntOp_VP_SPLICE(N, OpNo);
1889 break;
1890 }
1891
1892 // If the result is null, the sub-method took care of registering results etc.
1893 if (!Res.getNode()) return false;
1894
1895 // If the result is N, the sub-method updated N in place. Tell the legalizer
1896 // core about this.
1897 if (Res.getNode() == N)
1898 return true;
1899
1900 const bool IsStrictFp = N->isStrictFPOpcode();
1901 assert(Res.getValueType() == N->getValueType(0) &&
1902 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1903 "Invalid operand expansion");
1904 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1905 Res.dump());
1906
1907 ReplaceValueWith(SDValue(N, 0), Res);
1908 if (IsStrictFp)
1909 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1910
1911 return false;
1912 }
1913
1914 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1915 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
PromoteSetCCOperands(SDValue & LHS,SDValue & RHS,ISD::CondCode CCCode)1916 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
1917 ISD::CondCode CCCode) {
1918 // We have to insert explicit sign or zero extends. Note that we could
1919 // insert sign extends for ALL conditions. For those operations where either
1920 // zero or sign extension would be valid, we ask the target which extension
1921 // it would prefer.
1922
1923 // Signed comparisons always require sign extension.
1924 if (ISD::isSignedIntSetCC(CCCode)) {
1925 LHS = SExtPromotedInteger(LHS);
1926 RHS = SExtPromotedInteger(RHS);
1927 return;
1928 }
1929
1930 assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) &&
1931 "Unknown integer comparison!");
1932
1933 SDValue OpL = GetPromotedInteger(LHS);
1934 SDValue OpR = GetPromotedInteger(RHS);
1935
1936 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) {
1937 // The target would prefer to promote the comparison operand with sign
1938 // extension. Honor that unless the promoted values are already zero
1939 // extended.
1940 unsigned OpLEffectiveBits =
1941 DAG.computeKnownBits(OpL).countMaxActiveBits();
1942 unsigned OpREffectiveBits =
1943 DAG.computeKnownBits(OpR).countMaxActiveBits();
1944 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
1945 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
1946 LHS = OpL;
1947 RHS = OpR;
1948 return;
1949 }
1950
1951 // The promoted values aren't zero extended, use a sext_inreg.
1952 LHS = SExtPromotedInteger(LHS);
1953 RHS = SExtPromotedInteger(RHS);
1954 return;
1955 }
1956
1957 // Prefer to promote the comparison operand with zero extension.
1958
1959 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1960 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation
1961 // that we might not be able to remove.
1962 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
1963 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
1964 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
1965 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
1966 LHS = OpL;
1967 RHS = OpR;
1968 return;
1969 }
1970
1971 // Otherwise, use zext_inreg.
1972 LHS = ZExtPromotedInteger(LHS);
1973 RHS = ZExtPromotedInteger(RHS);
1974 }
1975
PromoteIntOp_ANY_EXTEND(SDNode * N)1976 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1977 SDValue Op = GetPromotedInteger(N->getOperand(0));
1978 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1979 }
1980
PromoteIntOp_ATOMIC_STORE(AtomicSDNode * N)1981 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1982 SDValue Op1 = GetPromotedInteger(N->getOperand(1));
1983 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1984 N->getChain(), Op1, N->getBasePtr(), N->getMemOperand());
1985 }
1986
PromoteIntOp_BITCAST(SDNode * N)1987 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1988 // This should only occur in unusual situations like bitcasting to an
1989 // x86_fp80, so just turn it into a store+load
1990 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1991 }
1992
PromoteIntOp_BR_CC(SDNode * N,unsigned OpNo)1993 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1994 assert(OpNo == 2 && "Don't know how to promote this operand!");
1995
1996 SDValue LHS = N->getOperand(2);
1997 SDValue RHS = N->getOperand(3);
1998 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1999
2000 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
2001 // legal types.
2002 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2003 N->getOperand(1), LHS, RHS, N->getOperand(4)),
2004 0);
2005 }
2006
PromoteIntOp_BRCOND(SDNode * N,unsigned OpNo)2007 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
2008 assert(OpNo == 1 && "only know how to promote condition");
2009
2010 // Promote all the way up to the canonical SetCC type.
2011 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
2012
2013 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
2014 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
2015 N->getOperand(2)), 0);
2016 }
2017
PromoteIntOp_BUILD_PAIR(SDNode * N)2018 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
2019 // Since the result type is legal, the operands must promote to it.
2020 EVT OVT = N->getOperand(0).getValueType();
2021 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
2022 SDValue Hi = GetPromotedInteger(N->getOperand(1));
2023 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
2024 SDLoc dl(N);
2025
2026 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
2027 DAG.getConstant(OVT.getSizeInBits(), dl,
2028 TLI.getPointerTy(DAG.getDataLayout())));
2029 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
2030 }
2031
PromoteIntOp_BUILD_VECTOR(SDNode * N)2032 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
2033 // The vector type is legal but the element type is not. This implies
2034 // that the vector is a power-of-two in length and that the element
2035 // type does not have a strange size (eg: it is not i1).
2036 EVT VecVT = N->getValueType(0);
2037 unsigned NumElts = VecVT.getVectorNumElements();
2038 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
2039 "Legal vector of one illegal element?");
2040
2041 // Promote the inserted value. The type does not need to match the
2042 // vector element type. Check that any extra bits introduced will be
2043 // truncated away.
2044 assert(N->getOperand(0).getValueSizeInBits() >=
2045 N->getValueType(0).getScalarSizeInBits() &&
2046 "Type of inserted value narrower than vector element type!");
2047
2048 SmallVector<SDValue, 16> NewOps;
2049 for (unsigned i = 0; i < NumElts; ++i)
2050 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
2051
2052 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2053 }
2054
PromoteIntOp_INSERT_VECTOR_ELT(SDNode * N,unsigned OpNo)2055 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
2056 unsigned OpNo) {
2057 if (OpNo == 1) {
2058 // Promote the inserted value. This is valid because the type does not
2059 // have to match the vector element type.
2060
2061 // Check that any extra bits introduced will be truncated away.
2062 assert(N->getOperand(1).getValueSizeInBits() >=
2063 N->getValueType(0).getScalarSizeInBits() &&
2064 "Type of inserted value narrower than vector element type!");
2065 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2066 GetPromotedInteger(N->getOperand(1)),
2067 N->getOperand(2)),
2068 0);
2069 }
2070
2071 assert(OpNo == 2 && "Different operand and result vector types?");
2072
2073 // Promote the index.
2074 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
2075 TLI.getVectorIdxTy(DAG.getDataLayout()));
2076 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2077 N->getOperand(1), Idx), 0);
2078 }
2079
PromoteIntOp_ScalarOp(SDNode * N)2080 SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
2081 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
2082 // so just promote the operand in place.
2083 return SDValue(DAG.UpdateNodeOperands(N,
2084 GetPromotedInteger(N->getOperand(0))), 0);
2085 }
2086
PromoteIntOp_SELECT(SDNode * N,unsigned OpNo)2087 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
2088 assert(OpNo == 0 && "Only know how to promote the condition!");
2089 SDValue Cond = N->getOperand(0);
2090 EVT OpTy = N->getOperand(1).getValueType();
2091
2092 if (N->getOpcode() == ISD::VSELECT)
2093 if (SDValue Res = WidenVSELECTMask(N))
2094 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2095 Res, N->getOperand(1), N->getOperand(2));
2096
2097 // Promote all the way up to the canonical SetCC type.
2098 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
2099 Cond = PromoteTargetBoolean(Cond, OpVT);
2100
2101 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
2102 N->getOperand(2)), 0);
2103 }
2104
PromoteIntOp_SELECT_CC(SDNode * N,unsigned OpNo)2105 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2106 assert(OpNo == 0 && "Don't know how to promote this operand!");
2107
2108 SDValue LHS = N->getOperand(0);
2109 SDValue RHS = N->getOperand(1);
2110 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
2111
2112 // The CC (#4) and the possible return values (#2 and #3) have legal types.
2113 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2114 N->getOperand(3), N->getOperand(4)), 0);
2115 }
2116
PromoteIntOp_SETCC(SDNode * N,unsigned OpNo)2117 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
2118 assert(OpNo == 0 && "Don't know how to promote this operand!");
2119
2120 SDValue LHS = N->getOperand(0);
2121 SDValue RHS = N->getOperand(1);
2122 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
2123
2124 // The CC (#2) is always legal.
2125 if (N->getOpcode() == ISD::SETCC)
2126 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
2127
2128 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2129
2130 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2131 N->getOperand(3), N->getOperand(4)),
2132 0);
2133 }
2134
PromoteIntOp_Shift(SDNode * N)2135 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
2136 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2137 ZExtPromotedInteger(N->getOperand(1))), 0);
2138 }
2139
PromoteIntOp_FunnelShift(SDNode * N)2140 SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) {
2141 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2142 ZExtPromotedInteger(N->getOperand(2))), 0);
2143 }
2144
PromoteIntOp_SIGN_EXTEND(SDNode * N)2145 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
2146 SDValue Op = GetPromotedInteger(N->getOperand(0));
2147 SDLoc dl(N);
2148 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2149 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
2150 Op, DAG.getValueType(N->getOperand(0).getValueType()));
2151 }
2152
PromoteIntOp_VP_SIGN_EXTEND(SDNode * N)2153 SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) {
2154 SDLoc dl(N);
2155 EVT VT = N->getValueType(0);
2156 SDValue Op = GetPromotedInteger(N->getOperand(0));
2157 // FIXME: There is no VP_ANY_EXTEND yet.
2158 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
2159 N->getOperand(2));
2160 unsigned Diff =
2161 VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits();
2162 SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl);
2163 // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts.
2164 SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1),
2165 N->getOperand(2));
2166 return DAG.getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt, N->getOperand(1),
2167 N->getOperand(2));
2168 }
2169
PromoteIntOp_SINT_TO_FP(SDNode * N)2170 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
2171 if (N->getOpcode() == ISD::VP_SINT_TO_FP)
2172 return SDValue(DAG.UpdateNodeOperands(N,
2173 SExtPromotedInteger(N->getOperand(0)),
2174 N->getOperand(1), N->getOperand(2)),
2175 0);
2176 return SDValue(DAG.UpdateNodeOperands(N,
2177 SExtPromotedInteger(N->getOperand(0))), 0);
2178 }
2179
PromoteIntOp_STRICT_SINT_TO_FP(SDNode * N)2180 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
2181 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2182 SExtPromotedInteger(N->getOperand(1))), 0);
2183 }
2184
PromoteIntOp_STORE(StoreSDNode * N,unsigned OpNo)2185 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
2186 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2187 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
2188 SDLoc dl(N);
2189
2190 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
2191
2192 // Truncate the value and store the result.
2193 return DAG.getTruncStore(Ch, dl, Val, Ptr,
2194 N->getMemoryVT(), N->getMemOperand());
2195 }
2196
PromoteIntOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)2197 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
2198 unsigned OpNo) {
2199 SDValue DataOp = N->getValue();
2200 SDValue Mask = N->getMask();
2201
2202 if (OpNo == 4) {
2203 // The Mask. Update in place.
2204 EVT DataVT = DataOp.getValueType();
2205 Mask = PromoteTargetBoolean(Mask, DataVT);
2206 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2207 NewOps[4] = Mask;
2208 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2209 }
2210
2211 assert(OpNo == 1 && "Unexpected operand for promotion");
2212 DataOp = GetPromotedInteger(DataOp);
2213
2214 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2215 N->getOffset(), Mask, N->getMemoryVT(),
2216 N->getMemOperand(), N->getAddressingMode(),
2217 /*IsTruncating*/ true, N->isCompressingStore());
2218 }
2219
PromoteIntOp_MLOAD(MaskedLoadSDNode * N,unsigned OpNo)2220 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
2221 unsigned OpNo) {
2222 assert(OpNo == 3 && "Only know how to promote the mask!");
2223 EVT DataVT = N->getValueType(0);
2224 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2225 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2226 NewOps[OpNo] = Mask;
2227 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2228 if (Res == N)
2229 return SDValue(Res, 0);
2230
2231 // Update triggered CSE, do our own replacement since caller can't.
2232 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2233 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2234 return SDValue();
2235 }
2236
PromoteIntOp_MGATHER(MaskedGatherSDNode * N,unsigned OpNo)2237 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
2238 unsigned OpNo) {
2239 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
2240
2241 if (OpNo == 2) {
2242 // The Mask
2243 EVT DataVT = N->getValueType(0);
2244 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2245 } else if (OpNo == 4) {
2246 // The Index
2247 if (N->isIndexSigned())
2248 // Need to sign extend the index since the bits will likely be used.
2249 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2250 else
2251 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2252 } else
2253 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2254
2255 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2256 if (Res == N)
2257 return SDValue(Res, 0);
2258
2259 // Update triggered CSE, do our own replacement since caller can't.
2260 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2261 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2262 return SDValue();
2263 }
2264
PromoteIntOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)2265 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
2266 unsigned OpNo) {
2267 bool TruncateStore = N->isTruncatingStore();
2268 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
2269
2270 if (OpNo == 2) {
2271 // The Mask
2272 EVT DataVT = N->getValue().getValueType();
2273 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2274 } else if (OpNo == 4) {
2275 // The Index
2276 if (N->isIndexSigned())
2277 // Need to sign extend the index since the bits will likely be used.
2278 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2279 else
2280 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2281 } else {
2282 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2283 TruncateStore = true;
2284 }
2285
2286 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2287 SDLoc(N), NewOps, N->getMemOperand(),
2288 N->getIndexType(), TruncateStore);
2289 }
2290
PromoteIntOp_TRUNCATE(SDNode * N)2291 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2292 SDValue Op = GetPromotedInteger(N->getOperand(0));
2293 if (N->getOpcode() == ISD::VP_TRUNCATE)
2294 return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op,
2295 N->getOperand(1), N->getOperand(2));
2296 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2297 }
2298
PromoteIntOp_UINT_TO_FP(SDNode * N)2299 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2300 if (N->getOpcode() == ISD::VP_UINT_TO_FP)
2301 return SDValue(DAG.UpdateNodeOperands(N,
2302 ZExtPromotedInteger(N->getOperand(0)),
2303 N->getOperand(1), N->getOperand(2)),
2304 0);
2305 return SDValue(DAG.UpdateNodeOperands(N,
2306 ZExtPromotedInteger(N->getOperand(0))), 0);
2307 }
2308
PromoteIntOp_STRICT_UINT_TO_FP(SDNode * N)2309 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2310 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2311 ZExtPromotedInteger(N->getOperand(1))), 0);
2312 }
2313
PromoteIntOp_ZERO_EXTEND(SDNode * N)2314 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2315 SDLoc dl(N);
2316 SDValue Op = GetPromotedInteger(N->getOperand(0));
2317 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2318 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
2319 }
2320
PromoteIntOp_VP_ZERO_EXTEND(SDNode * N)2321 SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) {
2322 SDLoc dl(N);
2323 EVT VT = N->getValueType(0);
2324 SDValue Op = GetPromotedInteger(N->getOperand(0));
2325 // FIXME: There is no VP_ANY_EXTEND yet.
2326 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
2327 N->getOperand(2));
2328 APInt Imm = APInt::getLowBitsSet(VT.getScalarSizeInBits(),
2329 N->getOperand(0).getScalarValueSizeInBits());
2330 return DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(Imm, dl, VT),
2331 N->getOperand(1), N->getOperand(2));
2332 }
2333
PromoteIntOp_ADDSUBO_CARRY(SDNode * N,unsigned OpNo)2334 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) {
2335 assert(OpNo == 2 && "Don't know how to promote this operand!");
2336
2337 SDValue LHS = N->getOperand(0);
2338 SDValue RHS = N->getOperand(1);
2339 SDValue Carry = N->getOperand(2);
2340 SDLoc DL(N);
2341
2342 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
2343
2344 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
2345 }
2346
PromoteIntOp_FIX(SDNode * N)2347 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2348 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2349 return SDValue(
2350 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2351 }
2352
PromoteIntOp_FRAMERETURNADDR(SDNode * N)2353 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2354 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2355 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2356 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2357 }
2358
PromoteIntOp_ExpOp(SDNode * N)2359 SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) {
2360 bool IsStrict = N->isStrictFPOpcode();
2361 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2362
2363 bool IsPowI =
2364 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
2365
2366 // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result
2367 // and floating point operand is already type legalized).
2368 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
2369 : RTLIB::getLDEXP(N->getValueType(0));
2370
2371 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2372 SDValue Op = SExtPromotedInteger(N->getOperand(1));
2373 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2374 }
2375
2376 // We can't just promote the exponent type in FPOWI, since we want to lower
2377 // the node to a libcall and we if we promote to a type larger than
2378 // sizeof(int) the libcall might not be according to the targets ABI. Instead
2379 // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2380 // if the target accepts it according to shouldSignExtendTypeInLibCall.
2381
2382 unsigned OpOffset = IsStrict ? 1 : 0;
2383 // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2384 assert(DAG.getLibInfo().getIntSize() ==
2385 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2386 "POWI exponent should match with sizeof(int) when doing the libcall.");
2387 TargetLowering::MakeLibCallOptions CallOptions;
2388 CallOptions.setSExt(true);
2389 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
2390 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2391 DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
2392 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2393 if (IsStrict)
2394 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2395 return SDValue();
2396 }
2397
getExtendForIntVecReduction(SDNode * N)2398 static unsigned getExtendForIntVecReduction(SDNode *N) {
2399 switch (N->getOpcode()) {
2400 default:
2401 llvm_unreachable("Expected integer vector reduction");
2402 case ISD::VECREDUCE_ADD:
2403 case ISD::VECREDUCE_MUL:
2404 case ISD::VECREDUCE_AND:
2405 case ISD::VECREDUCE_OR:
2406 case ISD::VECREDUCE_XOR:
2407 case ISD::VP_REDUCE_ADD:
2408 case ISD::VP_REDUCE_MUL:
2409 case ISD::VP_REDUCE_AND:
2410 case ISD::VP_REDUCE_OR:
2411 case ISD::VP_REDUCE_XOR:
2412 return ISD::ANY_EXTEND;
2413 case ISD::VECREDUCE_SMAX:
2414 case ISD::VECREDUCE_SMIN:
2415 case ISD::VP_REDUCE_SMAX:
2416 case ISD::VP_REDUCE_SMIN:
2417 return ISD::SIGN_EXTEND;
2418 case ISD::VECREDUCE_UMAX:
2419 case ISD::VECREDUCE_UMIN:
2420 case ISD::VP_REDUCE_UMAX:
2421 case ISD::VP_REDUCE_UMIN:
2422 return ISD::ZERO_EXTEND;
2423 }
2424 }
2425
PromoteIntOpVectorReduction(SDNode * N,SDValue V)2426 SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2427 switch (getExtendForIntVecReduction(N)) {
2428 default:
2429 llvm_unreachable("Impossible extension kind for integer reduction");
2430 case ISD::ANY_EXTEND:
2431 return GetPromotedInteger(V);
2432 case ISD::SIGN_EXTEND:
2433 return SExtPromotedInteger(V);
2434 case ISD::ZERO_EXTEND:
2435 return ZExtPromotedInteger(V);
2436 }
2437 }
2438
PromoteIntOp_VECREDUCE(SDNode * N)2439 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2440 SDLoc dl(N);
2441 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2442
2443 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType();
2444 EVT InVT = Op.getValueType();
2445 EVT EltVT = InVT.getVectorElementType();
2446 EVT ResVT = N->getValueType(0);
2447 unsigned Opcode = N->getOpcode();
2448
2449 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if
2450 // vecreduce_xor is not legal
2451 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 &&
2452 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) &&
2453 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT))
2454 Opcode = ISD::VECREDUCE_ADD;
2455
2456 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if
2457 // vecreduce_or is not legal
2458 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 &&
2459 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) &&
2460 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT)) {
2461 Opcode = ISD::VECREDUCE_UMAX;
2462 // Can't use promoteTargetBoolean here because we still need
2463 // to either sign_ext or zero_ext in the undefined case.
2464 switch (TLI.getBooleanContents(InVT)) {
2465 case TargetLoweringBase::UndefinedBooleanContent:
2466 case TargetLoweringBase::ZeroOrOneBooleanContent:
2467 Op = ZExtPromotedInteger(N->getOperand(0));
2468 break;
2469 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2470 Op = SExtPromotedInteger(N->getOperand(0));
2471 break;
2472 }
2473 }
2474
2475 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if
2476 // vecreduce_and is not legal
2477 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 &&
2478 !TLI.isOperationLegalOrCustom(ISD::VECREDUCE_AND, InVT) &&
2479 TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMIN, InVT)) {
2480 Opcode = ISD::VECREDUCE_UMIN;
2481 // Can't use promoteTargetBoolean here because we still need
2482 // to either sign_ext or zero_ext in the undefined case.
2483 switch (TLI.getBooleanContents(InVT)) {
2484 case TargetLoweringBase::UndefinedBooleanContent:
2485 case TargetLoweringBase::ZeroOrOneBooleanContent:
2486 Op = ZExtPromotedInteger(N->getOperand(0));
2487 break;
2488 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2489 Op = SExtPromotedInteger(N->getOperand(0));
2490 break;
2491 }
2492 }
2493
2494 if (ResVT.bitsGE(EltVT))
2495 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op);
2496
2497 // Result size must be >= element size. If this is not the case after
2498 // promotion, also promote the result type and then truncate.
2499 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op);
2500 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce);
2501 }
2502
PromoteIntOp_VP_REDUCE(SDNode * N,unsigned OpNo)2503 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2504 SDLoc DL(N);
2505 SDValue Op = N->getOperand(OpNo);
2506 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2507
2508 if (OpNo == 2) { // Mask
2509 // Update in place.
2510 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2511 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2512 }
2513
2514 assert(OpNo == 1 && "Unexpected operand for promotion");
2515
2516 Op = PromoteIntOpVectorReduction(N, Op);
2517
2518 NewOps[OpNo] = Op;
2519
2520 EVT VT = N->getValueType(0);
2521 EVT EltVT = Op.getValueType().getScalarType();
2522
2523 if (VT.bitsGE(EltVT))
2524 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
2525
2526 // Result size must be >= element/start-value size. If this is not the case
2527 // after promotion, also promote both the start value and result type and
2528 // then truncate.
2529 NewOps[0] =
2530 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
2531 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
2532 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2533 }
2534
PromoteIntOp_SET_ROUNDING(SDNode * N)2535 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2536 SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2537 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2538 }
2539
PromoteIntOp_STACKMAP(SDNode * N,unsigned OpNo)2540 SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
2541 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
2542 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
2543 SDValue Operand = N->getOperand(OpNo);
2544 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2545 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2546 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2547 }
2548
PromoteIntOp_PATCHPOINT(SDNode * N,unsigned OpNo)2549 SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
2550 assert(OpNo >= 7);
2551 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
2552 SDValue Operand = N->getOperand(OpNo);
2553 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2554 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2555 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2556 }
2557
PromoteIntOp_VP_STRIDED(SDNode * N,unsigned OpNo)2558 SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
2559 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2560 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2561
2562 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end());
2563 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2564
2565 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2566 }
2567
PromoteIntOp_VP_SPLICE(SDNode * N,unsigned OpNo)2568 SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) {
2569 SmallVector<SDValue, 6> NewOps(N->op_begin(), N->op_end());
2570
2571 if (OpNo == 2) { // Offset operand
2572 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2573 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2574 }
2575
2576 assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion");
2577
2578 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2579 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2580 }
2581
2582 //===----------------------------------------------------------------------===//
2583 // Integer Result Expansion
2584 //===----------------------------------------------------------------------===//
2585
2586 /// ExpandIntegerResult - This method is called when the specified result of the
2587 /// specified node is found to need expansion. At this point, the node may also
2588 /// have invalid operands or may have other results that need promotion, we just
2589 /// know that (at least) one result needs expansion.
ExpandIntegerResult(SDNode * N,unsigned ResNo)2590 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2591 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG));
2592 SDValue Lo, Hi;
2593 Lo = Hi = SDValue();
2594
2595 // See if the target wants to custom expand this node.
2596 if (CustomLowerNode(N, N->getValueType(ResNo), true))
2597 return;
2598
2599 switch (N->getOpcode()) {
2600 default:
2601 #ifndef NDEBUG
2602 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2603 N->dump(&DAG); dbgs() << "\n";
2604 #endif
2605 report_fatal_error("Do not know how to expand the result of this "
2606 "operator!");
2607
2608 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
2609 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2610 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
2611 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2612 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2613 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2614
2615 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2616 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2617 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2618 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2619 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2620
2621 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2622 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2623 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2624 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2625 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2626 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2627 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2628 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2629 case ISD::CTLZ_ZERO_UNDEF:
2630 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2631 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2632 case ISD::CTTZ_ZERO_UNDEF:
2633 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2634 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break;
2635 case ISD::STRICT_FP_TO_SINT:
2636 case ISD::FP_TO_SINT:
2637 case ISD::STRICT_FP_TO_UINT:
2638 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break;
2639 case ISD::FP_TO_SINT_SAT:
2640 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2641 case ISD::STRICT_LROUND:
2642 case ISD::STRICT_LRINT:
2643 case ISD::LROUND:
2644 case ISD::LRINT:
2645 case ISD::STRICT_LLROUND:
2646 case ISD::STRICT_LLRINT:
2647 case ISD::LLROUND:
2648 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break;
2649 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2650 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2651 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
2652 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2653 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2654 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2655 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2656 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2657 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2658 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2659 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2660 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2661
2662 case ISD::ATOMIC_LOAD_ADD:
2663 case ISD::ATOMIC_LOAD_SUB:
2664 case ISD::ATOMIC_LOAD_AND:
2665 case ISD::ATOMIC_LOAD_CLR:
2666 case ISD::ATOMIC_LOAD_OR:
2667 case ISD::ATOMIC_LOAD_XOR:
2668 case ISD::ATOMIC_LOAD_NAND:
2669 case ISD::ATOMIC_LOAD_MIN:
2670 case ISD::ATOMIC_LOAD_MAX:
2671 case ISD::ATOMIC_LOAD_UMIN:
2672 case ISD::ATOMIC_LOAD_UMAX:
2673 case ISD::ATOMIC_SWAP:
2674 case ISD::ATOMIC_CMP_SWAP: {
2675 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2676 SplitInteger(Tmp.first, Lo, Hi);
2677 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2678 break;
2679 }
2680 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2681 AtomicSDNode *AN = cast<AtomicSDNode>(N);
2682 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2683 SDValue Tmp = DAG.getAtomicCmpSwap(
2684 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
2685 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2686 AN->getMemOperand());
2687
2688 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2689 // success simply by comparing the loaded value against the ingoing
2690 // comparison.
2691 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2692 N->getOperand(2), ISD::SETEQ);
2693
2694 SplitInteger(Tmp, Lo, Hi);
2695 ReplaceValueWith(SDValue(N, 1), Success);
2696 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2697 break;
2698 }
2699
2700 case ISD::AND:
2701 case ISD::OR:
2702 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2703
2704 case ISD::UMAX:
2705 case ISD::SMAX:
2706 case ISD::UMIN:
2707 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2708
2709 case ISD::ADD:
2710 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2711
2712 case ISD::ADDC:
2713 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2714
2715 case ISD::ADDE:
2716 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2717
2718 case ISD::UADDO_CARRY:
2719 case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break;
2720
2721 case ISD::SADDO_CARRY:
2722 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2723
2724 case ISD::SHL:
2725 case ISD::SRA:
2726 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2727
2728 case ISD::SADDO:
2729 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2730 case ISD::UADDO:
2731 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2732 case ISD::UMULO:
2733 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2734
2735 case ISD::SADDSAT:
2736 case ISD::UADDSAT:
2737 case ISD::SSUBSAT:
2738 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2739
2740 case ISD::SSHLSAT:
2741 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2742
2743 case ISD::SMULFIX:
2744 case ISD::SMULFIXSAT:
2745 case ISD::UMULFIX:
2746 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2747
2748 case ISD::SDIVFIX:
2749 case ISD::SDIVFIXSAT:
2750 case ISD::UDIVFIX:
2751 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2752
2753 case ISD::VECREDUCE_ADD:
2754 case ISD::VECREDUCE_MUL:
2755 case ISD::VECREDUCE_AND:
2756 case ISD::VECREDUCE_OR:
2757 case ISD::VECREDUCE_XOR:
2758 case ISD::VECREDUCE_SMAX:
2759 case ISD::VECREDUCE_SMIN:
2760 case ISD::VECREDUCE_UMAX:
2761 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2762
2763 case ISD::ROTL:
2764 case ISD::ROTR:
2765 ExpandIntRes_Rotate(N, Lo, Hi);
2766 break;
2767
2768 case ISD::FSHL:
2769 case ISD::FSHR:
2770 ExpandIntRes_FunnelShift(N, Lo, Hi);
2771 break;
2772
2773 case ISD::VSCALE:
2774 ExpandIntRes_VSCALE(N, Lo, Hi);
2775 break;
2776 }
2777
2778 // If Lo/Hi is null, the sub-method took care of registering results etc.
2779 if (Lo.getNode())
2780 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2781 }
2782
2783 /// Lower an atomic node to the appropriate builtin call.
ExpandAtomic(SDNode * Node)2784 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2785 unsigned Opc = Node->getOpcode();
2786 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2787 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
2788 // Lower to outline atomic libcall if outline atomics enabled,
2789 // or to sync libcall otherwise
2790 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2791 EVT RetVT = Node->getValueType(0);
2792 TargetLowering::MakeLibCallOptions CallOptions;
2793 SmallVector<SDValue, 4> Ops;
2794 if (TLI.getLibcallName(LC)) {
2795 Ops.append(Node->op_begin() + 2, Node->op_end());
2796 Ops.push_back(Node->getOperand(1));
2797 } else {
2798 LC = RTLIB::getSYNC(Opc, VT);
2799 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2800 "Unexpected atomic op or value type!");
2801 Ops.append(Node->op_begin() + 1, Node->op_end());
2802 }
2803 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2804 Node->getOperand(0));
2805 }
2806
2807 /// N is a shift by a value that needs to be expanded,
2808 /// and the shift amount is a constant 'Amt'. Expand the operation.
ExpandShiftByConstant(SDNode * N,const APInt & Amt,SDValue & Lo,SDValue & Hi)2809 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2810 SDValue &Lo, SDValue &Hi) {
2811 SDLoc DL(N);
2812 // Expand the incoming operand to be shifted, so that we have its parts
2813 SDValue InL, InH;
2814 GetExpandedInteger(N->getOperand(0), InL, InH);
2815
2816 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2817 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2818 if (!Amt) {
2819 Lo = InL;
2820 Hi = InH;
2821 return;
2822 }
2823
2824 EVT NVT = InL.getValueType();
2825 unsigned VTBits = N->getValueType(0).getSizeInBits();
2826 unsigned NVTBits = NVT.getSizeInBits();
2827 EVT ShTy = N->getOperand(1).getValueType();
2828
2829 if (N->getOpcode() == ISD::SHL) {
2830 if (Amt.uge(VTBits)) {
2831 Lo = Hi = DAG.getConstant(0, DL, NVT);
2832 } else if (Amt.ugt(NVTBits)) {
2833 Lo = DAG.getConstant(0, DL, NVT);
2834 Hi = DAG.getNode(ISD::SHL, DL,
2835 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2836 } else if (Amt == NVTBits) {
2837 Lo = DAG.getConstant(0, DL, NVT);
2838 Hi = InL;
2839 } else {
2840 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
2841 Hi = DAG.getNode(ISD::OR, DL, NVT,
2842 DAG.getNode(ISD::SHL, DL, NVT, InH,
2843 DAG.getConstant(Amt, DL, ShTy)),
2844 DAG.getNode(ISD::SRL, DL, NVT, InL,
2845 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2846 }
2847 return;
2848 }
2849
2850 if (N->getOpcode() == ISD::SRL) {
2851 if (Amt.uge(VTBits)) {
2852 Lo = Hi = DAG.getConstant(0, DL, NVT);
2853 } else if (Amt.ugt(NVTBits)) {
2854 Lo = DAG.getNode(ISD::SRL, DL,
2855 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2856 Hi = DAG.getConstant(0, DL, NVT);
2857 } else if (Amt == NVTBits) {
2858 Lo = InH;
2859 Hi = DAG.getConstant(0, DL, NVT);
2860 } else {
2861 Lo = DAG.getNode(ISD::OR, DL, NVT,
2862 DAG.getNode(ISD::SRL, DL, NVT, InL,
2863 DAG.getConstant(Amt, DL, ShTy)),
2864 DAG.getNode(ISD::SHL, DL, NVT, InH,
2865 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2866 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2867 }
2868 return;
2869 }
2870
2871 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2872 if (Amt.uge(VTBits)) {
2873 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2874 DAG.getConstant(NVTBits - 1, DL, ShTy));
2875 } else if (Amt.ugt(NVTBits)) {
2876 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2877 DAG.getConstant(Amt - NVTBits, DL, ShTy));
2878 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2879 DAG.getConstant(NVTBits - 1, DL, ShTy));
2880 } else if (Amt == NVTBits) {
2881 Lo = InH;
2882 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2883 DAG.getConstant(NVTBits - 1, DL, ShTy));
2884 } else {
2885 Lo = DAG.getNode(ISD::OR, DL, NVT,
2886 DAG.getNode(ISD::SRL, DL, NVT, InL,
2887 DAG.getConstant(Amt, DL, ShTy)),
2888 DAG.getNode(ISD::SHL, DL, NVT, InH,
2889 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2890 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2891 }
2892 }
2893
2894 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2895 /// this shift based on knowledge of the high bit of the shift amount. If we
2896 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2897 /// shift amount.
2898 bool DAGTypeLegalizer::
ExpandShiftWithKnownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2899 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2900 SDValue Amt = N->getOperand(1);
2901 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2902 EVT ShTy = Amt.getValueType();
2903 unsigned ShBits = ShTy.getScalarSizeInBits();
2904 unsigned NVTBits = NVT.getScalarSizeInBits();
2905 assert(isPowerOf2_32(NVTBits) &&
2906 "Expanded integer type size not a power of two!");
2907 SDLoc dl(N);
2908
2909 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2910 KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2911
2912 // If we don't know anything about the high bits, exit.
2913 if (((Known.Zero|Known.One) & HighBitMask) == 0)
2914 return false;
2915
2916 // Get the incoming operand to be shifted.
2917 SDValue InL, InH;
2918 GetExpandedInteger(N->getOperand(0), InL, InH);
2919
2920 // If we know that any of the high bits of the shift amount are one, then we
2921 // can do this as a couple of simple shifts.
2922 if (Known.One.intersects(HighBitMask)) {
2923 // Mask out the high bit, which we know is set.
2924 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2925 DAG.getConstant(~HighBitMask, dl, ShTy));
2926
2927 switch (N->getOpcode()) {
2928 default: llvm_unreachable("Unknown shift");
2929 case ISD::SHL:
2930 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2931 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2932 return true;
2933 case ISD::SRL:
2934 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2935 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2936 return true;
2937 case ISD::SRA:
2938 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2939 DAG.getConstant(NVTBits - 1, dl, ShTy));
2940 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2941 return true;
2942 }
2943 }
2944
2945 // If we know that all of the high bits of the shift amount are zero, then we
2946 // can do this as a couple of simple shifts.
2947 if (HighBitMask.isSubsetOf(Known.Zero)) {
2948 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2949 // shift if x is zero. We can use XOR here because x is known to be smaller
2950 // than 32.
2951 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2952 DAG.getConstant(NVTBits - 1, dl, ShTy));
2953
2954 unsigned Op1, Op2;
2955 switch (N->getOpcode()) {
2956 default: llvm_unreachable("Unknown shift");
2957 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2958 case ISD::SRL:
2959 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2960 }
2961
2962 // When shifting right the arithmetic for Lo and Hi is swapped.
2963 if (N->getOpcode() != ISD::SHL)
2964 std::swap(InL, InH);
2965
2966 // Use a little trick to get the bits that move from Lo to Hi. First
2967 // shift by one bit.
2968 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2969 // Then compute the remaining shift with amount-1.
2970 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2971
2972 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2973 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2974
2975 if (N->getOpcode() != ISD::SHL)
2976 std::swap(Hi, Lo);
2977 return true;
2978 }
2979
2980 return false;
2981 }
2982
2983 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2984 /// of any size.
2985 bool DAGTypeLegalizer::
ExpandShiftWithUnknownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)2986 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2987 SDValue Amt = N->getOperand(1);
2988 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2989 EVT ShTy = Amt.getValueType();
2990 unsigned NVTBits = NVT.getSizeInBits();
2991 assert(isPowerOf2_32(NVTBits) &&
2992 "Expanded integer type size not a power of two!");
2993 SDLoc dl(N);
2994
2995 // Get the incoming operand to be shifted.
2996 SDValue InL, InH;
2997 GetExpandedInteger(N->getOperand(0), InL, InH);
2998
2999 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
3000 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
3001 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
3002 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3003 Amt, NVBitsNode, ISD::SETULT);
3004 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3005 Amt, DAG.getConstant(0, dl, ShTy),
3006 ISD::SETEQ);
3007
3008 SDValue LoS, HiS, LoL, HiL;
3009 switch (N->getOpcode()) {
3010 default: llvm_unreachable("Unknown shift");
3011 case ISD::SHL:
3012 // Short: ShAmt < NVTBits
3013 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
3014 HiS = DAG.getNode(ISD::OR, dl, NVT,
3015 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
3016 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
3017
3018 // Long: ShAmt >= NVTBits
3019 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
3020 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
3021
3022 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
3023 Hi = DAG.getSelect(dl, NVT, isZero, InH,
3024 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
3025 return true;
3026 case ISD::SRL:
3027 // Short: ShAmt < NVTBits
3028 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
3029 LoS = DAG.getNode(ISD::OR, dl, NVT,
3030 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3031 // FIXME: If Amt is zero, the following shift generates an undefined result
3032 // on some architectures.
3033 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3034
3035 // Long: ShAmt >= NVTBits
3036 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
3037 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3038
3039 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3040 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3041 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3042 return true;
3043 case ISD::SRA:
3044 // Short: ShAmt < NVTBits
3045 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
3046 LoS = DAG.getNode(ISD::OR, dl, NVT,
3047 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3048 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3049
3050 // Long: ShAmt >= NVTBits
3051 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
3052 DAG.getConstant(NVTBits - 1, dl, ShTy));
3053 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3054
3055 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3056 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3057 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3058 return true;
3059 }
3060 }
3061
getExpandedMinMaxOps(int Op)3062 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
3063
3064 switch (Op) {
3065 default: llvm_unreachable("invalid min/max opcode");
3066 case ISD::SMAX:
3067 return std::make_pair(ISD::SETGT, ISD::UMAX);
3068 case ISD::UMAX:
3069 return std::make_pair(ISD::SETUGT, ISD::UMAX);
3070 case ISD::SMIN:
3071 return std::make_pair(ISD::SETLT, ISD::UMIN);
3072 case ISD::UMIN:
3073 return std::make_pair(ISD::SETULT, ISD::UMIN);
3074 }
3075 }
3076
ExpandIntRes_MINMAX(SDNode * N,SDValue & Lo,SDValue & Hi)3077 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
3078 SDValue &Lo, SDValue &Hi) {
3079 SDLoc DL(N);
3080
3081 SDValue LHS = N->getOperand(0);
3082 SDValue RHS = N->getOperand(1);
3083
3084 // If the upper halves are all sign bits, then we can perform the MINMAX on
3085 // the lower half and sign-extend the result to the upper half.
3086 unsigned NumBits = N->getValueType(0).getScalarSizeInBits();
3087 unsigned NumHalfBits = NumBits / 2;
3088 if (DAG.ComputeNumSignBits(LHS) > NumHalfBits &&
3089 DAG.ComputeNumSignBits(RHS) > NumHalfBits) {
3090 SDValue LHSL, LHSH, RHSL, RHSH;
3091 GetExpandedInteger(LHS, LHSL, LHSH);
3092 GetExpandedInteger(RHS, RHSL, RHSH);
3093 EVT NVT = LHSL.getValueType();
3094
3095 Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL);
3096 Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo,
3097 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL));
3098 return;
3099 }
3100
3101 // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1.
3102 // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL.
3103 if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) ||
3104 (N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) {
3105 SDValue LHSL, LHSH, RHSL, RHSH;
3106 GetExpandedInteger(LHS, LHSL, LHSH);
3107 GetExpandedInteger(RHS, RHSL, RHSH);
3108 EVT NVT = LHSL.getValueType();
3109 EVT CCT = getSetCCResultType(NVT);
3110
3111 SDValue HiNeg =
3112 DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT);
3113 if (N->getOpcode() == ISD::SMIN) {
3114 Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT));
3115 } else {
3116 Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL);
3117 }
3118 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3119 return;
3120 }
3121
3122 const APInt *RHSVal = nullptr;
3123 if (auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3124 RHSVal = &RHSConst->getAPIntValue();
3125
3126 // The high half of MIN/MAX is always just the the MIN/MAX of the
3127 // high halves of the operands. Expand this way if it appears profitable.
3128 if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) &&
3129 (RHSVal->countLeadingOnes() >= NumHalfBits ||
3130 RHSVal->countLeadingZeros() >= NumHalfBits)) {
3131 SDValue LHSL, LHSH, RHSL, RHSH;
3132 GetExpandedInteger(LHS, LHSL, LHSH);
3133 GetExpandedInteger(RHS, RHSL, RHSH);
3134 EVT NVT = LHSL.getValueType();
3135 EVT CCT = getSetCCResultType(NVT);
3136
3137 ISD::NodeType LoOpc;
3138 ISD::CondCode CondC;
3139 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
3140
3141 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3142 // We need to know whether to select Lo part that corresponds to 'winning'
3143 // Hi part or if Hi parts are equal.
3144 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
3145 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
3146
3147 // Lo part corresponding to the 'winning' Hi part
3148 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
3149
3150 // Recursed Lo part if Hi parts are equal, this uses unsigned version
3151 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
3152
3153 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
3154 return;
3155 }
3156
3157 // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies
3158 // the compare.
3159 ISD::CondCode Pred;
3160 switch (N->getOpcode()) {
3161 default: llvm_unreachable("How did we get here?");
3162 case ISD::SMAX:
3163 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3164 Pred = ISD::SETGE;
3165 else
3166 Pred = ISD::SETGT;
3167 break;
3168 case ISD::SMIN:
3169 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3170 Pred = ISD::SETLE;
3171 else
3172 Pred = ISD::SETLT;
3173 break;
3174 case ISD::UMAX:
3175 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3176 Pred = ISD::SETUGE;
3177 else
3178 Pred = ISD::SETUGT;
3179 break;
3180 case ISD::UMIN:
3181 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3182 Pred = ISD::SETULE;
3183 else
3184 Pred = ISD::SETULT;
3185 break;
3186 }
3187 EVT VT = N->getValueType(0);
3188 EVT CCT = getSetCCResultType(VT);
3189 SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred);
3190 SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS);
3191 SplitInteger(Result, Lo, Hi);
3192 }
3193
ExpandIntRes_ADDSUB(SDNode * N,SDValue & Lo,SDValue & Hi)3194 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
3195 SDValue &Lo, SDValue &Hi) {
3196 SDLoc dl(N);
3197 // Expand the subcomponents.
3198 SDValue LHSL, LHSH, RHSL, RHSH;
3199 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3200 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3201
3202 EVT NVT = LHSL.getValueType();
3203 SDValue LoOps[2] = { LHSL, RHSL };
3204 SDValue HiOps[3] = { LHSH, RHSH };
3205
3206 bool HasOpCarry = TLI.isOperationLegalOrCustom(
3207 N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
3208 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3209 if (HasOpCarry) {
3210 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3211 if (N->getOpcode() == ISD::ADD) {
3212 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3213 HiOps[2] = Lo.getValue(1);
3214 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3215 ? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2))
3216 : DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps);
3217 } else {
3218 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3219 HiOps[2] = Lo.getValue(1);
3220 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3221 ? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2))
3222 : DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps);
3223 }
3224 return;
3225 }
3226
3227 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
3228 // them. TODO: Teach operation legalization how to expand unsupported
3229 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
3230 // a carry of type MVT::Glue, but there doesn't seem to be any way to
3231 // generate a value of this type in the expanded code sequence.
3232 bool hasCarry =
3233 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3234 ISD::ADDC : ISD::SUBC,
3235 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3236
3237 if (hasCarry) {
3238 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
3239 if (N->getOpcode() == ISD::ADD) {
3240 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3241 HiOps[2] = Lo.getValue(1);
3242 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3243 } else {
3244 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3245 HiOps[2] = Lo.getValue(1);
3246 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3247 }
3248 return;
3249 }
3250
3251 bool hasOVF =
3252 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3253 ISD::UADDO : ISD::USUBO,
3254 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3255 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
3256
3257 if (hasOVF) {
3258 EVT OvfVT = getSetCCResultType(NVT);
3259 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
3260 int RevOpc;
3261 if (N->getOpcode() == ISD::ADD) {
3262 RevOpc = ISD::SUB;
3263 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3264 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3265 } else {
3266 RevOpc = ISD::ADD;
3267 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3268 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3269 }
3270 SDValue OVF = Lo.getValue(1);
3271
3272 switch (BoolType) {
3273 case TargetLoweringBase::UndefinedBooleanContent:
3274 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
3275 [[fallthrough]];
3276 case TargetLoweringBase::ZeroOrOneBooleanContent:
3277 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
3278 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
3279 break;
3280 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
3281 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
3282 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
3283 }
3284 return;
3285 }
3286
3287 if (N->getOpcode() == ISD::ADD) {
3288 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
3289 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3290 SDValue Cmp;
3291 // Special case: X+1 has a carry out if X+1==0. This may reduce the live
3292 // range of X. We assume comparing with 0 is cheap.
3293 if (isOneConstant(LoOps[1]))
3294 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3295 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3296 else if (isAllOnesConstant(LoOps[1])) {
3297 if (isAllOnesConstant(HiOps[1]))
3298 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3299 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3300 else
3301 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3302 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3303 } else
3304 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
3305 ISD::SETULT);
3306
3307 SDValue Carry;
3308 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
3309 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3310 else
3311 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3312 DAG.getConstant(0, dl, NVT));
3313
3314 if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1]))
3315 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry);
3316 else
3317 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
3318 } else {
3319 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
3320 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3321 SDValue Cmp =
3322 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3323 LoOps[0], LoOps[1], ISD::SETULT);
3324
3325 SDValue Borrow;
3326 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
3327 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3328 else
3329 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3330 DAG.getConstant(0, dl, NVT));
3331
3332 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
3333 }
3334 }
3335
ExpandIntRes_ADDSUBC(SDNode * N,SDValue & Lo,SDValue & Hi)3336 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
3337 SDValue &Lo, SDValue &Hi) {
3338 // Expand the subcomponents.
3339 SDValue LHSL, LHSH, RHSL, RHSH;
3340 SDLoc dl(N);
3341 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3342 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3343 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3344 SDValue LoOps[2] = { LHSL, RHSL };
3345 SDValue HiOps[3] = { LHSH, RHSH };
3346
3347 if (N->getOpcode() == ISD::ADDC) {
3348 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3349 HiOps[2] = Lo.getValue(1);
3350 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3351 } else {
3352 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3353 HiOps[2] = Lo.getValue(1);
3354 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3355 }
3356
3357 // Legalized the flag result - switch anything that used the old flag to
3358 // use the new one.
3359 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3360 }
3361
ExpandIntRes_ADDSUBE(SDNode * N,SDValue & Lo,SDValue & Hi)3362 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
3363 SDValue &Lo, SDValue &Hi) {
3364 // Expand the subcomponents.
3365 SDValue LHSL, LHSH, RHSL, RHSH;
3366 SDLoc dl(N);
3367 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3368 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3369 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3370 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3371 SDValue HiOps[3] = { LHSH, RHSH };
3372
3373 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3374 HiOps[2] = Lo.getValue(1);
3375 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3376
3377 // Legalized the flag result - switch anything that used the old flag to
3378 // use the new one.
3379 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3380 }
3381
ExpandIntRes_UADDSUBO(SDNode * N,SDValue & Lo,SDValue & Hi)3382 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
3383 SDValue &Lo, SDValue &Hi) {
3384 SDValue LHS = N->getOperand(0);
3385 SDValue RHS = N->getOperand(1);
3386 SDLoc dl(N);
3387
3388 SDValue Ovf;
3389
3390 unsigned CarryOp, NoCarryOp;
3391 ISD::CondCode Cond;
3392 switch(N->getOpcode()) {
3393 case ISD::UADDO:
3394 CarryOp = ISD::UADDO_CARRY;
3395 NoCarryOp = ISD::ADD;
3396 Cond = ISD::SETULT;
3397 break;
3398 case ISD::USUBO:
3399 CarryOp = ISD::USUBO_CARRY;
3400 NoCarryOp = ISD::SUB;
3401 Cond = ISD::SETUGT;
3402 break;
3403 default:
3404 llvm_unreachable("Node has unexpected Opcode");
3405 }
3406
3407 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3408 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3409
3410 if (HasCarryOp) {
3411 // Expand the subcomponents.
3412 SDValue LHSL, LHSH, RHSL, RHSH;
3413 GetExpandedInteger(LHS, LHSL, LHSH);
3414 GetExpandedInteger(RHS, RHSL, RHSH);
3415 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3416 SDValue LoOps[2] = { LHSL, RHSL };
3417 SDValue HiOps[3] = { LHSH, RHSH };
3418
3419 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3420 HiOps[2] = Lo.getValue(1);
3421 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
3422
3423 Ovf = Hi.getValue(1);
3424 } else {
3425 // Expand the result by simply replacing it with the equivalent
3426 // non-overflow-checking operation.
3427 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
3428 SplitInteger(Sum, Lo, Hi);
3429
3430 if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) {
3431 // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this
3432 // with (Lo | Hi) == 0.
3433 SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi);
3434 Ovf = DAG.getSetCC(dl, N->getValueType(1), Or,
3435 DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ);
3436 } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) {
3437 // Special case: uaddo X, -1 overflows if X == 0.
3438 Ovf =
3439 DAG.getSetCC(dl, N->getValueType(1), LHS,
3440 DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE);
3441 } else {
3442 // Calculate the overflow: addition overflows iff a + b < a, and
3443 // subtraction overflows iff a - b > a.
3444 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
3445 }
3446 }
3447
3448 // Legalized the flag result - switch anything that used the old flag to
3449 // use the new one.
3450 ReplaceValueWith(SDValue(N, 1), Ovf);
3451 }
3452
ExpandIntRes_UADDSUBO_CARRY(SDNode * N,SDValue & Lo,SDValue & Hi)3453 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo,
3454 SDValue &Hi) {
3455 // Expand the subcomponents.
3456 SDValue LHSL, LHSH, RHSL, RHSH;
3457 SDLoc dl(N);
3458 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3459 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3460 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3461 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3462 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
3463
3464 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3465 HiOps[2] = Lo.getValue(1);
3466 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3467
3468 // Legalized the flag result - switch anything that used the old flag to
3469 // use the new one.
3470 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3471 }
3472
ExpandIntRes_SADDSUBO_CARRY(SDNode * N,SDValue & Lo,SDValue & Hi)3473 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
3474 SDValue &Lo, SDValue &Hi) {
3475 // Expand the subcomponents.
3476 SDValue LHSL, LHSH, RHSL, RHSH;
3477 SDLoc dl(N);
3478 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3479 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3480 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3481
3482 // We need to use an unsigned carry op for the lo part.
3483 unsigned CarryOp =
3484 N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
3485 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
3486 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3487
3488 // Legalized the flag result - switch anything that used the old flag to
3489 // use the new one.
3490 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3491 }
3492
ExpandIntRes_ANY_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)3493 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
3494 SDValue &Lo, SDValue &Hi) {
3495 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3496 SDLoc dl(N);
3497 SDValue Op = N->getOperand(0);
3498 if (Op.getValueType().bitsLE(NVT)) {
3499 // The low part is any extension of the input (which degenerates to a copy).
3500 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
3501 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
3502 } else {
3503 // For example, extension of an i48 to an i64. The operand type necessarily
3504 // promotes to the result type, so will end up being expanded too.
3505 assert(getTypeAction(Op.getValueType()) ==
3506 TargetLowering::TypePromoteInteger &&
3507 "Only know how to promote this result!");
3508 SDValue Res = GetPromotedInteger(Op);
3509 assert(Res.getValueType() == N->getValueType(0) &&
3510 "Operand over promoted?");
3511 // Split the promoted operand. This will simplify when it is expanded.
3512 SplitInteger(Res, Lo, Hi);
3513 }
3514 }
3515
ExpandIntRes_AssertSext(SDNode * N,SDValue & Lo,SDValue & Hi)3516 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3517 SDValue &Lo, SDValue &Hi) {
3518 SDLoc dl(N);
3519 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3520 EVT NVT = Lo.getValueType();
3521 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3522 unsigned NVTBits = NVT.getSizeInBits();
3523 unsigned EVTBits = EVT.getSizeInBits();
3524
3525 if (NVTBits < EVTBits) {
3526 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
3527 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3528 EVTBits - NVTBits)));
3529 } else {
3530 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
3531 // The high part replicates the sign bit of Lo, make it explicit.
3532 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3533 DAG.getConstant(NVTBits - 1, dl,
3534 TLI.getPointerTy(DAG.getDataLayout())));
3535 }
3536 }
3537
ExpandIntRes_AssertZext(SDNode * N,SDValue & Lo,SDValue & Hi)3538 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
3539 SDValue &Lo, SDValue &Hi) {
3540 SDLoc dl(N);
3541 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3542 EVT NVT = Lo.getValueType();
3543 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3544 unsigned NVTBits = NVT.getSizeInBits();
3545 unsigned EVTBits = EVT.getSizeInBits();
3546
3547 if (NVTBits < EVTBits) {
3548 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
3549 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3550 EVTBits - NVTBits)));
3551 } else {
3552 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
3553 // The high part must be zero, make it explicit.
3554 Hi = DAG.getConstant(0, dl, NVT);
3555 }
3556 }
3557
ExpandIntRes_BITREVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)3558 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
3559 SDValue &Lo, SDValue &Hi) {
3560 SDLoc dl(N);
3561 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3562 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
3563 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3564 }
3565
ExpandIntRes_BSWAP(SDNode * N,SDValue & Lo,SDValue & Hi)3566 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
3567 SDValue &Lo, SDValue &Hi) {
3568 SDLoc dl(N);
3569 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3570 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
3571 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3572 }
3573
ExpandIntRes_PARITY(SDNode * N,SDValue & Lo,SDValue & Hi)3574 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
3575 SDValue &Hi) {
3576 SDLoc dl(N);
3577 // parity(HiLo) -> parity(Lo^Hi)
3578 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3579 EVT NVT = Lo.getValueType();
3580 Lo =
3581 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
3582 Hi = DAG.getConstant(0, dl, NVT);
3583 }
3584
ExpandIntRes_Constant(SDNode * N,SDValue & Lo,SDValue & Hi)3585 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
3586 SDValue &Lo, SDValue &Hi) {
3587 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3588 unsigned NBitWidth = NVT.getSizeInBits();
3589 auto Constant = cast<ConstantSDNode>(N);
3590 const APInt &Cst = Constant->getAPIntValue();
3591 bool IsTarget = Constant->isTargetOpcode();
3592 bool IsOpaque = Constant->isOpaque();
3593 SDLoc dl(N);
3594 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
3595 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
3596 IsOpaque);
3597 }
3598
ExpandIntRes_ABS(SDNode * N,SDValue & Lo,SDValue & Hi)3599 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
3600 SDLoc dl(N);
3601
3602 SDValue N0 = N->getOperand(0);
3603 GetExpandedInteger(N0, Lo, Hi);
3604 EVT NVT = Lo.getValueType();
3605
3606 // If the upper half is all sign bits, then we can perform the ABS on the
3607 // lower half and zero-extend.
3608 if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) {
3609 Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo);
3610 Hi = DAG.getConstant(0, dl, NVT);
3611 return;
3612 }
3613
3614 // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence
3615 // we use in LegalizeDAG. The SUB part of the expansion is based on
3616 // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that
3617 // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further
3618 // expanded if needed. Shift expansion has a special case for filling with
3619 // sign bits so that we will only end up with one SRA.
3620 bool HasSubCarry = TLI.isOperationLegalOrCustom(
3621 ISD::USUBO_CARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3622 if (HasSubCarry) {
3623 SDValue Sign = DAG.getNode(
3624 ISD::SRA, dl, NVT, Hi,
3625 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl));
3626 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3627 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
3628 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
3629 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign);
3630 Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
3631 return;
3632 }
3633
3634 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
3635 EVT VT = N->getValueType(0);
3636 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
3637 DAG.getConstant(0, dl, VT), N0);
3638 SDValue NegLo, NegHi;
3639 SplitInteger(Neg, NegLo, NegHi);
3640
3641 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3642 DAG.getConstant(0, dl, NVT), ISD::SETLT);
3643 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
3644 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
3645 }
3646
ExpandIntRes_CTLZ(SDNode * N,SDValue & Lo,SDValue & Hi)3647 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
3648 SDValue &Lo, SDValue &Hi) {
3649 SDLoc dl(N);
3650 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
3651 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3652 EVT NVT = Lo.getValueType();
3653
3654 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3655 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3656
3657 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
3658 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
3659
3660 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
3661 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
3662 DAG.getConstant(NVT.getSizeInBits(), dl,
3663 NVT)));
3664 Hi = DAG.getConstant(0, dl, NVT);
3665 }
3666
ExpandIntRes_CTPOP(SDNode * N,SDValue & Lo,SDValue & Hi)3667 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
3668 SDValue &Lo, SDValue &Hi) {
3669 SDLoc dl(N);
3670 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
3671 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3672 EVT NVT = Lo.getValueType();
3673 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
3674 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
3675 Hi = DAG.getConstant(0, dl, NVT);
3676 }
3677
ExpandIntRes_CTTZ(SDNode * N,SDValue & Lo,SDValue & Hi)3678 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
3679 SDValue &Lo, SDValue &Hi) {
3680 SDLoc dl(N);
3681 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
3682 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3683 EVT NVT = Lo.getValueType();
3684
3685 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3686 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3687
3688 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
3689 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
3690
3691 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
3692 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
3693 DAG.getConstant(NVT.getSizeInBits(), dl,
3694 NVT)));
3695 Hi = DAG.getConstant(0, dl, NVT);
3696 }
3697
ExpandIntRes_GET_ROUNDING(SDNode * N,SDValue & Lo,SDValue & Hi)3698 void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo,
3699 SDValue &Hi) {
3700 SDLoc dl(N);
3701 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3702 unsigned NBitWidth = NVT.getSizeInBits();
3703
3704 Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0));
3705 SDValue Chain = Lo.getValue(1);
3706 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING
3707 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3708 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl));
3709
3710 // Legalize the chain result - switch anything that used the old chain to
3711 // use the new one.
3712 ReplaceValueWith(SDValue(N, 1), Chain);
3713 }
3714
3715 // Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op.
fpExtendHelper(SDValue Op,SDValue & Chain,bool IsStrict,EVT VT,SDLoc DL,SelectionDAG & DAG)3716 static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT,
3717 SDLoc DL, SelectionDAG &DAG) {
3718 if (IsStrict) {
3719 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op});
3720 Chain = Op.getValue(1);
3721 return Op;
3722 }
3723 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Op);
3724 }
3725
ExpandIntRes_FP_TO_XINT(SDNode * N,SDValue & Lo,SDValue & Hi)3726 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo,
3727 SDValue &Hi) {
3728 SDLoc dl(N);
3729 EVT VT = N->getValueType(0);
3730
3731 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3732 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3733 bool IsStrict = N->isStrictFPOpcode();
3734 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3735 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3736 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3737 Op = GetPromotedFloat(Op);
3738
3739 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3740 EVT OFPVT = Op.getValueType();
3741 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), OFPVT);
3742 Op = GetSoftPromotedHalf(Op);
3743 Op = DAG.getNode(OFPVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, dl,
3744 NFPVT, Op);
3745 Op = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, VT, Op);
3746 SplitInteger(Op, Lo, Hi);
3747 return;
3748 }
3749
3750 if (Op.getValueType() == MVT::bf16) {
3751 // Extend to f32 as there is no bf16 libcall.
3752 Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG);
3753 }
3754
3755 RTLIB::Libcall LC = IsSigned ? RTLIB::getFPTOSINT(Op.getValueType(), VT)
3756 : RTLIB::getFPTOUINT(Op.getValueType(), VT);
3757 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!");
3758 TargetLowering::MakeLibCallOptions CallOptions;
3759 CallOptions.setSExt(true);
3760 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3761 CallOptions, dl, Chain);
3762 SplitInteger(Tmp.first, Lo, Hi);
3763
3764 if (IsStrict)
3765 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3766 }
3767
ExpandIntRes_FP_TO_XINT_SAT(SDNode * N,SDValue & Lo,SDValue & Hi)3768 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3769 SDValue &Hi) {
3770 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3771 SplitInteger(Res, Lo, Hi);
3772 }
3773
ExpandIntRes_XROUND_XRINT(SDNode * N,SDValue & Lo,SDValue & Hi)3774 void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo,
3775 SDValue &Hi) {
3776 SDLoc dl(N);
3777 bool IsStrict = N->isStrictFPOpcode();
3778 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3779 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3780
3781 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
3782 "Input type needs to be promoted!");
3783
3784 EVT VT = Op.getValueType();
3785
3786 if (VT == MVT::f16) {
3787 // Extend to f32.
3788 VT = MVT::f32;
3789 Op = fpExtendHelper(Op, Chain, IsStrict, VT, dl, DAG);
3790 }
3791
3792 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3793 if (N->getOpcode() == ISD::LROUND ||
3794 N->getOpcode() == ISD::STRICT_LROUND) {
3795 if (VT == MVT::f32)
3796 LC = RTLIB::LROUND_F32;
3797 else if (VT == MVT::f64)
3798 LC = RTLIB::LROUND_F64;
3799 else if (VT == MVT::f80)
3800 LC = RTLIB::LROUND_F80;
3801 else if (VT == MVT::f128)
3802 LC = RTLIB::LROUND_F128;
3803 else if (VT == MVT::ppcf128)
3804 LC = RTLIB::LROUND_PPCF128;
3805 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!");
3806 } else if (N->getOpcode() == ISD::LRINT ||
3807 N->getOpcode() == ISD::STRICT_LRINT) {
3808 if (VT == MVT::f32)
3809 LC = RTLIB::LRINT_F32;
3810 else if (VT == MVT::f64)
3811 LC = RTLIB::LRINT_F64;
3812 else if (VT == MVT::f80)
3813 LC = RTLIB::LRINT_F80;
3814 else if (VT == MVT::f128)
3815 LC = RTLIB::LRINT_F128;
3816 else if (VT == MVT::ppcf128)
3817 LC = RTLIB::LRINT_PPCF128;
3818 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!");
3819 } else if (N->getOpcode() == ISD::LLROUND ||
3820 N->getOpcode() == ISD::STRICT_LLROUND) {
3821 if (VT == MVT::f32)
3822 LC = RTLIB::LLROUND_F32;
3823 else if (VT == MVT::f64)
3824 LC = RTLIB::LLROUND_F64;
3825 else if (VT == MVT::f80)
3826 LC = RTLIB::LLROUND_F80;
3827 else if (VT == MVT::f128)
3828 LC = RTLIB::LLROUND_F128;
3829 else if (VT == MVT::ppcf128)
3830 LC = RTLIB::LLROUND_PPCF128;
3831 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
3832 } else if (N->getOpcode() == ISD::LLRINT ||
3833 N->getOpcode() == ISD::STRICT_LLRINT) {
3834 if (VT == MVT::f32)
3835 LC = RTLIB::LLRINT_F32;
3836 else if (VT == MVT::f64)
3837 LC = RTLIB::LLRINT_F64;
3838 else if (VT == MVT::f80)
3839 LC = RTLIB::LLRINT_F80;
3840 else if (VT == MVT::f128)
3841 LC = RTLIB::LLRINT_F128;
3842 else if (VT == MVT::ppcf128)
3843 LC = RTLIB::LLRINT_PPCF128;
3844 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
3845 } else
3846 llvm_unreachable("Unexpected opcode!");
3847
3848 EVT RetVT = N->getValueType(0);
3849
3850 TargetLowering::MakeLibCallOptions CallOptions;
3851 CallOptions.setSExt(true);
3852 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3853 Op, CallOptions, dl,
3854 Chain);
3855 SplitInteger(Tmp.first, Lo, Hi);
3856
3857 if (N->isStrictFPOpcode())
3858 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3859 }
3860
ExpandIntRes_LOAD(LoadSDNode * N,SDValue & Lo,SDValue & Hi)3861 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
3862 SDValue &Lo, SDValue &Hi) {
3863 assert(!N->isAtomic() && "Should have been a ATOMIC_LOAD?");
3864
3865 if (ISD::isNormalLoad(N)) {
3866 ExpandRes_NormalLoad(N, Lo, Hi);
3867 return;
3868 }
3869
3870 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
3871
3872 EVT VT = N->getValueType(0);
3873 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3874 SDValue Ch = N->getChain();
3875 SDValue Ptr = N->getBasePtr();
3876 ISD::LoadExtType ExtType = N->getExtensionType();
3877 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3878 AAMDNodes AAInfo = N->getAAInfo();
3879 SDLoc dl(N);
3880
3881 assert(NVT.isByteSized() && "Expanded type not byte sized!");
3882
3883 if (N->getMemoryVT().bitsLE(NVT)) {
3884 EVT MemVT = N->getMemoryVT();
3885
3886 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
3887 N->getOriginalAlign(), MMOFlags, AAInfo);
3888
3889 // Remember the chain.
3890 Ch = Lo.getValue(1);
3891
3892 if (ExtType == ISD::SEXTLOAD) {
3893 // The high part is obtained by SRA'ing all but one of the bits of the
3894 // lo part.
3895 unsigned LoSize = Lo.getValueSizeInBits();
3896 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3897 DAG.getConstant(LoSize - 1, dl,
3898 TLI.getPointerTy(DAG.getDataLayout())));
3899 } else if (ExtType == ISD::ZEXTLOAD) {
3900 // The high part is just a zero.
3901 Hi = DAG.getConstant(0, dl, NVT);
3902 } else {
3903 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
3904 // The high part is undefined.
3905 Hi = DAG.getUNDEF(NVT);
3906 }
3907 } else if (DAG.getDataLayout().isLittleEndian()) {
3908 // Little-endian - low bits are at low addresses.
3909 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
3910 N->getOriginalAlign(), MMOFlags, AAInfo);
3911
3912 unsigned ExcessBits =
3913 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3914 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3915
3916 // Increment the pointer to the other half.
3917 unsigned IncrementSize = NVT.getSizeInBits()/8;
3918 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
3919 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3920 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3921 N->getOriginalAlign(), MMOFlags, AAInfo);
3922
3923 // Build a factor node to remember that this load is independent of the
3924 // other one.
3925 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3926 Hi.getValue(1));
3927 } else {
3928 // Big-endian - high bits are at low addresses. Favor aligned loads at
3929 // the cost of some bit-fiddling.
3930 EVT MemVT = N->getMemoryVT();
3931 unsigned EBytes = MemVT.getStoreSize();
3932 unsigned IncrementSize = NVT.getSizeInBits()/8;
3933 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3934
3935 // Load both the high bits and maybe some of the low bits.
3936 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3937 EVT::getIntegerVT(*DAG.getContext(),
3938 MemVT.getSizeInBits() - ExcessBits),
3939 N->getOriginalAlign(), MMOFlags, AAInfo);
3940
3941 // Increment the pointer to the other half.
3942 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
3943 // Load the rest of the low bits.
3944 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3945 N->getPointerInfo().getWithOffset(IncrementSize),
3946 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3947 N->getOriginalAlign(), MMOFlags, AAInfo);
3948
3949 // Build a factor node to remember that this load is independent of the
3950 // other one.
3951 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3952 Hi.getValue(1));
3953
3954 if (ExcessBits < NVT.getSizeInBits()) {
3955 // Transfer low bits from the bottom of Hi to the top of Lo.
3956 Lo = DAG.getNode(
3957 ISD::OR, dl, NVT, Lo,
3958 DAG.getNode(ISD::SHL, dl, NVT, Hi,
3959 DAG.getConstant(ExcessBits, dl,
3960 TLI.getPointerTy(DAG.getDataLayout()))));
3961 // Move high bits to the right position in Hi.
3962 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
3963 Hi,
3964 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3965 TLI.getPointerTy(DAG.getDataLayout())));
3966 }
3967 }
3968
3969 // Legalize the chain result - switch anything that used the old chain to
3970 // use the new one.
3971 ReplaceValueWith(SDValue(N, 1), Ch);
3972 }
3973
ExpandIntRes_Logical(SDNode * N,SDValue & Lo,SDValue & Hi)3974 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
3975 SDValue &Lo, SDValue &Hi) {
3976 SDLoc dl(N);
3977 SDValue LL, LH, RL, RH;
3978 GetExpandedInteger(N->getOperand(0), LL, LH);
3979 GetExpandedInteger(N->getOperand(1), RL, RH);
3980 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
3981 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3982 }
3983
ExpandIntRes_MUL(SDNode * N,SDValue & Lo,SDValue & Hi)3984 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
3985 SDValue &Lo, SDValue &Hi) {
3986 EVT VT = N->getValueType(0);
3987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3988 SDLoc dl(N);
3989
3990 SDValue LL, LH, RL, RH;
3991 GetExpandedInteger(N->getOperand(0), LL, LH);
3992 GetExpandedInteger(N->getOperand(1), RL, RH);
3993
3994 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
3995 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
3996 LL, LH, RL, RH))
3997 return;
3998
3999 // If nothing else, we can make a libcall.
4000 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4001 if (VT == MVT::i16)
4002 LC = RTLIB::MUL_I16;
4003 else if (VT == MVT::i32)
4004 LC = RTLIB::MUL_I32;
4005 else if (VT == MVT::i64)
4006 LC = RTLIB::MUL_I64;
4007 else if (VT == MVT::i128)
4008 LC = RTLIB::MUL_I128;
4009
4010 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
4011 // We'll expand the multiplication by brute force because we have no other
4012 // options. This is a trivially-generalized version of the code from
4013 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
4014 // 4.3.1).
4015 unsigned Bits = NVT.getSizeInBits();
4016 unsigned HalfBits = Bits >> 1;
4017 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
4018 NVT);
4019 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
4020 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
4021
4022 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
4023 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
4024
4025 SDValue Shift = DAG.getShiftAmountConstant(HalfBits, NVT, dl);
4026 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
4027 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
4028 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
4029
4030 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
4031 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
4032 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
4033 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
4034
4035 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
4036 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
4037 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
4038
4039 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
4040 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
4041 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
4042 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
4043 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
4044
4045 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
4046 DAG.getNode(ISD::ADD, dl, NVT,
4047 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
4048 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
4049 return;
4050 }
4051
4052 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4053 TargetLowering::MakeLibCallOptions CallOptions;
4054 CallOptions.setSExt(true);
4055 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4056 Lo, Hi);
4057 }
4058
ExpandIntRes_READCYCLECOUNTER(SDNode * N,SDValue & Lo,SDValue & Hi)4059 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
4060 SDValue &Hi) {
4061 SDLoc DL(N);
4062 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4063 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
4064 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
4065 Lo = R.getValue(0);
4066 Hi = R.getValue(1);
4067 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
4068 }
4069
ExpandIntRes_ADDSUBSAT(SDNode * N,SDValue & Lo,SDValue & Hi)4070 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
4071 SDValue &Hi) {
4072 SDValue Result = TLI.expandAddSubSat(N, DAG);
4073 SplitInteger(Result, Lo, Hi);
4074 }
4075
ExpandIntRes_SHLSAT(SDNode * N,SDValue & Lo,SDValue & Hi)4076 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
4077 SDValue &Hi) {
4078 SDValue Result = TLI.expandShlSat(N, DAG);
4079 SplitInteger(Result, Lo, Hi);
4080 }
4081
4082 /// This performs an expansion of the integer result for a fixed point
4083 /// multiplication. The default expansion performs rounding down towards
4084 /// negative infinity, though targets that do care about rounding should specify
4085 /// a target hook for rounding and provide their own expansion or lowering of
4086 /// fixed point multiplication to be consistent with rounding.
ExpandIntRes_MULFIX(SDNode * N,SDValue & Lo,SDValue & Hi)4087 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
4088 SDValue &Hi) {
4089 SDLoc dl(N);
4090 EVT VT = N->getValueType(0);
4091 unsigned VTSize = VT.getScalarSizeInBits();
4092 SDValue LHS = N->getOperand(0);
4093 SDValue RHS = N->getOperand(1);
4094 uint64_t Scale = N->getConstantOperandVal(2);
4095 bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
4096 N->getOpcode() == ISD::UMULFIXSAT);
4097 bool Signed = (N->getOpcode() == ISD::SMULFIX ||
4098 N->getOpcode() == ISD::SMULFIXSAT);
4099
4100 // Handle special case when scale is equal to zero.
4101 if (!Scale) {
4102 SDValue Result;
4103 if (!Saturating) {
4104 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
4105 } else {
4106 EVT BoolVT = getSetCCResultType(VT);
4107 unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
4108 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
4109 SDValue Product = Result.getValue(0);
4110 SDValue Overflow = Result.getValue(1);
4111 if (Signed) {
4112 APInt MinVal = APInt::getSignedMinValue(VTSize);
4113 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
4114 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
4115 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4116 SDValue Zero = DAG.getConstant(0, dl, VT);
4117 // Xor the inputs, if resulting sign bit is 0 the product will be
4118 // positive, else negative.
4119 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
4120 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
4121 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
4122 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
4123 } else {
4124 // For unsigned multiplication, we only need to check the max since we
4125 // can't really overflow towards zero.
4126 APInt MaxVal = APInt::getMaxValue(VTSize);
4127 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4128 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
4129 }
4130 }
4131 SplitInteger(Result, Lo, Hi);
4132 return;
4133 }
4134
4135 // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
4136 // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
4137 assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
4138
4139 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4140 SDValue LL, LH, RL, RH;
4141 GetExpandedInteger(LHS, LL, LH);
4142 GetExpandedInteger(RHS, RL, RH);
4143 SmallVector<SDValue, 4> Result;
4144
4145 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
4146 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4147 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
4148 LL, LH, RL, RH)) {
4149 report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
4150 return;
4151 }
4152
4153 unsigned NVTSize = NVT.getScalarSizeInBits();
4154 assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
4155 "the size of the current value type");
4156
4157 // After getting the multiplication result in 4 parts, we need to perform a
4158 // shift right by the amount of the scale to get the result in that scale.
4159 //
4160 // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
4161 // 128 bits that are cut into 4 32-bit parts:
4162 //
4163 // HH HL LH LL
4164 // |---32---|---32---|---32---|---32---|
4165 // 128 96 64 32 0
4166 //
4167 // |------VTSize-----|
4168 //
4169 // |NVTSize-|
4170 //
4171 // The resulting Lo and Hi would normally be in LL and LH after the shift. But
4172 // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
4173 // amount and get Lo and Hi using two funnel shifts. Or for the special case
4174 // when Scale is a multiple of NVTSize we can just pick the result without
4175 // shifting.
4176 uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
4177 if (Scale % NVTSize) {
4178 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl);
4179 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
4180 ShiftAmount);
4181 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
4182 ShiftAmount);
4183 } else {
4184 Lo = Result[Part0];
4185 Hi = Result[Part0 + 1];
4186 }
4187
4188 // Unless saturation is requested we are done. The result is in <Hi,Lo>.
4189 if (!Saturating)
4190 return;
4191
4192 // Can not overflow when there is no integer part.
4193 if (Scale == VTSize)
4194 return;
4195
4196 // To handle saturation we must check for overflow in the multiplication.
4197 //
4198 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
4199 // aren't all zeroes.
4200 //
4201 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
4202 // aren't all ones or all zeroes.
4203 //
4204 // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
4205 // highest bit of HH determines saturation direction in the event of signed
4206 // saturation.
4207
4208 SDValue ResultHL = Result[2];
4209 SDValue ResultHH = Result[3];
4210
4211 SDValue SatMax, SatMin;
4212 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
4213 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
4214 EVT BoolNVT = getSetCCResultType(NVT);
4215
4216 if (!Signed) {
4217 if (Scale < NVTSize) {
4218 // Overflow happened if ((HH | (HL >> Scale)) != 0).
4219 SDValue HLAdjusted =
4220 DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
4221 DAG.getShiftAmountConstant(Scale, NVT, dl));
4222 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
4223 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
4224 } else if (Scale == NVTSize) {
4225 // Overflow happened if (HH != 0).
4226 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
4227 } else if (Scale < VTSize) {
4228 // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
4229 SDValue HLAdjusted =
4230 DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
4231 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl));
4232 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
4233 } else
4234 llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
4235 "(and saturation can't happen with Scale==VTSize).");
4236
4237 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
4238 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
4239 return;
4240 }
4241
4242 if (Scale < NVTSize) {
4243 // The number of overflow bits we can check are VTSize - Scale + 1 (we
4244 // include the sign bit). If these top bits are > 0, then we overflowed past
4245 // the max value. If these top bits are < -1, then we overflowed past the
4246 // min value. Otherwise, we did not overflow.
4247 unsigned OverflowBits = VTSize - Scale + 1;
4248 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4249 "Extent of overflow bits must start within HL");
4250 SDValue HLHiMask = DAG.getConstant(
4251 APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
4252 SDValue HLLoMask = DAG.getConstant(
4253 APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
4254 // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
4255 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
4256 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
4257 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
4258 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
4259 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
4260 // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
4261 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
4262 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
4263 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
4264 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
4265 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
4266 } else if (Scale == NVTSize) {
4267 // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
4268 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
4269 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
4270 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
4271 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
4272 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
4273 // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
4274 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
4275 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
4276 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
4277 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
4278 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
4279 } else if (Scale < VTSize) {
4280 // This is similar to the case when we saturate if Scale < NVTSize, but we
4281 // only need to check HH.
4282 unsigned OverflowBits = VTSize - Scale + 1;
4283 SDValue HHHiMask = DAG.getConstant(
4284 APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
4285 SDValue HHLoMask = DAG.getConstant(
4286 APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
4287 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
4288 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
4289 } else
4290 llvm_unreachable("Illegal scale for signed fixed point mul.");
4291
4292 // Saturate to signed maximum.
4293 APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
4294 APInt MaxLo = APInt::getAllOnes(NVTSize);
4295 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
4296 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
4297 // Saturate to signed minimum.
4298 APInt MinHi = APInt::getSignedMinValue(NVTSize);
4299 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
4300 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
4301 }
4302
ExpandIntRes_DIVFIX(SDNode * N,SDValue & Lo,SDValue & Hi)4303 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
4304 SDValue &Hi) {
4305 SDLoc dl(N);
4306 // Try expanding in the existing type first.
4307 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
4308 N->getOperand(1),
4309 N->getConstantOperandVal(2), DAG);
4310
4311 if (!Res)
4312 Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
4313 N->getConstantOperandVal(2), TLI, DAG);
4314 SplitInteger(Res, Lo, Hi);
4315 }
4316
ExpandIntRes_SADDSUBO(SDNode * Node,SDValue & Lo,SDValue & Hi)4317 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
4318 SDValue &Lo, SDValue &Hi) {
4319 assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
4320 "Node has unexpected Opcode");
4321 SDValue LHS = Node->getOperand(0);
4322 SDValue RHS = Node->getOperand(1);
4323 SDLoc dl(Node);
4324
4325 SDValue Ovf;
4326
4327 bool IsAdd = Node->getOpcode() == ISD::SADDO;
4328 unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
4329
4330 bool HasCarryOp = TLI.isOperationLegalOrCustom(
4331 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
4332
4333 if (HasCarryOp) {
4334 // Expand the subcomponents.
4335 SDValue LHSL, LHSH, RHSL, RHSH;
4336 GetExpandedInteger(LHS, LHSL, LHSH);
4337 GetExpandedInteger(RHS, RHSL, RHSH);
4338 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
4339
4340 Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
4341 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
4342
4343 Ovf = Hi.getValue(1);
4344 } else {
4345 // Expand the result by simply replacing it with the equivalent
4346 // non-overflow-checking operation.
4347 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
4348 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
4349 LHS, RHS);
4350 SplitInteger(Sum, Lo, Hi);
4351
4352 // Compute the overflow.
4353 //
4354 // LHSSign -> LHS < 0
4355 // RHSSign -> RHS < 0
4356 // SumSign -> Sum < 0
4357 //
4358 // Add:
4359 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
4360 // Sub:
4361 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
4362 //
4363 // To get better codegen we can rewrite this by doing bitwise math on
4364 // the integers and extract the final sign bit at the end. So the
4365 // above becomes:
4366 //
4367 // Add:
4368 // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
4369 // Sub:
4370 // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
4371 //
4372 // NOTE: This is different than the expansion we do in expandSADDSUBO
4373 // because it is more costly to determine the RHS is > 0 for SSUBO with the
4374 // integers split.
4375 EVT VT = LHS.getValueType();
4376 SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
4377 if (IsAdd)
4378 SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
4379
4380 SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
4381 Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
4382 EVT OType = Node->getValueType(1);
4383 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
4384 }
4385
4386 // Use the calculated overflow everywhere.
4387 ReplaceValueWith(SDValue(Node, 1), Ovf);
4388 }
4389
ExpandIntRes_SDIV(SDNode * N,SDValue & Lo,SDValue & Hi)4390 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
4391 SDValue &Lo, SDValue &Hi) {
4392 EVT VT = N->getValueType(0);
4393 SDLoc dl(N);
4394 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4395
4396 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
4397 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4398 SplitInteger(Res.getValue(0), Lo, Hi);
4399 return;
4400 }
4401
4402 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4403 if (VT == MVT::i16)
4404 LC = RTLIB::SDIV_I16;
4405 else if (VT == MVT::i32)
4406 LC = RTLIB::SDIV_I32;
4407 else if (VT == MVT::i64)
4408 LC = RTLIB::SDIV_I64;
4409 else if (VT == MVT::i128)
4410 LC = RTLIB::SDIV_I128;
4411 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
4412
4413 TargetLowering::MakeLibCallOptions CallOptions;
4414 CallOptions.setSExt(true);
4415 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4416 }
4417
ExpandIntRes_ShiftThroughStack(SDNode * N,SDValue & Lo,SDValue & Hi)4418 void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo,
4419 SDValue &Hi) {
4420 SDLoc dl(N);
4421 SDValue Shiftee = N->getOperand(0);
4422 EVT VT = Shiftee.getValueType();
4423 SDValue ShAmt = N->getOperand(1);
4424 EVT ShAmtVT = ShAmt.getValueType();
4425
4426 // This legalization is optimal when the shift is by a multiple of byte width,
4427 // %x * 8 <-> %x << 3 so 3 low bits should be be known zero.
4428 bool ShiftByByteMultiple =
4429 DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >= 3;
4430
4431 // If we can't do it as one step, we'll have two uses of shift amount,
4432 // and thus must freeze it.
4433 if (!ShiftByByteMultiple)
4434 ShAmt = DAG.getFreeze(ShAmt);
4435
4436 unsigned VTBitWidth = VT.getScalarSizeInBits();
4437 assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?");
4438 unsigned VTByteWidth = VTBitWidth / 8;
4439 assert(isPowerOf2_32(VTByteWidth) &&
4440 "Shiftee type size is not a power of two!");
4441 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4442 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4443 EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth);
4444
4445 // Get a temporary stack slot 2x the width of our VT.
4446 // FIXME: reuse stack slots?
4447 // FIXME: should we be more picky about alignment?
4448 Align StackSlotAlignment(1);
4449 SDValue StackPtr = DAG.CreateStackTemporary(
4450 TypeSize::getFixed(StackSlotByteWidth), StackSlotAlignment);
4451 EVT PtrTy = StackPtr.getValueType();
4452 SDValue Ch = DAG.getEntryNode();
4453
4454 MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack(
4455 DAG.getMachineFunction(),
4456 cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex());
4457
4458 // Extend the value, that is being shifted, to the entire stack slot's width.
4459 SDValue Init;
4460 if (N->getOpcode() != ISD::SHL) {
4461 unsigned WideningOpc =
4462 N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4463 Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4464 } else {
4465 // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width.
4466 SDValue AllZeros = DAG.getConstant(0, dl, VT);
4467 Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee);
4468 }
4469 // And spill it into the stack slot.
4470 Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4471
4472 // Now, compute the full-byte offset into stack slot from where we can load.
4473 // We have shift amount, which is in bits, but in multiples of byte.
4474 // So just divide by CHAR_BIT.
4475 SDNodeFlags Flags;
4476 if (ShiftByByteMultiple)
4477 Flags.setExact(true);
4478 SDValue ByteOffset = DAG.getNode(ISD::SRL, dl, ShAmtVT, ShAmt,
4479 DAG.getConstant(3, dl, ShAmtVT), Flags);
4480 // And clamp it, because OOB load is an immediate UB,
4481 // while shift overflow would have *just* been poison.
4482 ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset,
4483 DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT));
4484 // We have exactly two strategies on indexing into stack slot here:
4485 // 1. upwards starting from the beginning of the slot
4486 // 2. downwards starting from the middle of the slot
4487 // On little-endian machine, we pick 1. for right shifts and 2. for left-shift
4488 // and vice versa on big-endian machine.
4489 bool WillIndexUpwards = N->getOpcode() != ISD::SHL;
4490 if (DAG.getDataLayout().isBigEndian())
4491 WillIndexUpwards = !WillIndexUpwards;
4492
4493 SDValue AdjStackPtr;
4494 if (WillIndexUpwards) {
4495 AdjStackPtr = StackPtr;
4496 } else {
4497 AdjStackPtr = DAG.getMemBasePlusOffset(
4498 StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl);
4499 ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT);
4500 }
4501
4502 // Get the pointer somewhere into the stack slot from which we need to load.
4503 ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy);
4504 AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl);
4505
4506 // And load it! While the load is not legal, legalizing it is obvious.
4507 SDValue Res = DAG.getLoad(
4508 VT, dl, Ch, AdjStackPtr,
4509 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), Align(1));
4510 // We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_]
4511
4512 // If we may still have a less-than-CHAR_BIT to shift by, do so now.
4513 if (!ShiftByByteMultiple) {
4514 SDValue ShAmtRem = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
4515 DAG.getConstant(7, dl, ShAmtVT));
4516 Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem);
4517 }
4518
4519 // Finally, split the computed value.
4520 SplitInteger(Res, Lo, Hi);
4521 }
4522
ExpandIntRes_Shift(SDNode * N,SDValue & Lo,SDValue & Hi)4523 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
4524 SDValue &Lo, SDValue &Hi) {
4525 EVT VT = N->getValueType(0);
4526 SDLoc dl(N);
4527
4528 // If we can emit an efficient shift operation, do so now. Check to see if
4529 // the RHS is a constant.
4530 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
4531 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
4532
4533 // If we can determine that the high bit of the shift is zero or one, even if
4534 // the low bits are variable, emit this shift in an optimized form.
4535 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
4536 return;
4537
4538 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
4539 unsigned PartsOpc;
4540 if (N->getOpcode() == ISD::SHL) {
4541 PartsOpc = ISD::SHL_PARTS;
4542 } else if (N->getOpcode() == ISD::SRL) {
4543 PartsOpc = ISD::SRL_PARTS;
4544 } else {
4545 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
4546 PartsOpc = ISD::SRA_PARTS;
4547 }
4548
4549 // Next check to see if the target supports this SHL_PARTS operation or if it
4550 // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
4551 // size, but create a libcall instead.
4552 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4553 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
4554 const bool LegalOrCustom =
4555 (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
4556 Action == TargetLowering::Custom;
4557
4558 unsigned ExpansionFactor = 1;
4559 // That VT->NVT expansion is one step. But will we re-expand NVT?
4560 for (EVT TmpVT = NVT;;) {
4561 EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT);
4562 if (NewTMPVT == TmpVT)
4563 break;
4564 TmpVT = NewTMPVT;
4565 ++ExpansionFactor;
4566 }
4567
4568 TargetLowering::ShiftLegalizationStrategy S =
4569 TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor);
4570
4571 if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack)
4572 return ExpandIntRes_ShiftThroughStack(N, Lo, Hi);
4573
4574 if (LegalOrCustom &&
4575 S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) {
4576 // Expand the subcomponents.
4577 SDValue LHSL, LHSH;
4578 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
4579 EVT VT = LHSL.getValueType();
4580
4581 // If the shift amount operand is coming from a vector legalization it may
4582 // have an illegal type. Fix that first by casting the operand, otherwise
4583 // the new SHL_PARTS operation would need further legalization.
4584 SDValue ShiftOp = N->getOperand(1);
4585 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
4586 if (ShiftOp.getValueType() != ShiftTy)
4587 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
4588
4589 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4590 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
4591 Hi = Lo.getValue(1);
4592 return;
4593 }
4594
4595 // Otherwise, emit a libcall.
4596 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4597 bool isSigned;
4598 if (N->getOpcode() == ISD::SHL) {
4599 isSigned = false; /*sign irrelevant*/
4600 if (VT == MVT::i16)
4601 LC = RTLIB::SHL_I16;
4602 else if (VT == MVT::i32)
4603 LC = RTLIB::SHL_I32;
4604 else if (VT == MVT::i64)
4605 LC = RTLIB::SHL_I64;
4606 else if (VT == MVT::i128)
4607 LC = RTLIB::SHL_I128;
4608 } else if (N->getOpcode() == ISD::SRL) {
4609 isSigned = false;
4610 if (VT == MVT::i16)
4611 LC = RTLIB::SRL_I16;
4612 else if (VT == MVT::i32)
4613 LC = RTLIB::SRL_I32;
4614 else if (VT == MVT::i64)
4615 LC = RTLIB::SRL_I64;
4616 else if (VT == MVT::i128)
4617 LC = RTLIB::SRL_I128;
4618 } else {
4619 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
4620 isSigned = true;
4621 if (VT == MVT::i16)
4622 LC = RTLIB::SRA_I16;
4623 else if (VT == MVT::i32)
4624 LC = RTLIB::SRA_I32;
4625 else if (VT == MVT::i64)
4626 LC = RTLIB::SRA_I64;
4627 else if (VT == MVT::i128)
4628 LC = RTLIB::SRA_I128;
4629 }
4630
4631 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
4632 EVT ShAmtTy =
4633 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());
4634 SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy);
4635 SDValue Ops[2] = {N->getOperand(0), ShAmt};
4636 TargetLowering::MakeLibCallOptions CallOptions;
4637 CallOptions.setSExt(isSigned);
4638 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4639 return;
4640 }
4641
4642 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
4643 llvm_unreachable("Unsupported shift!");
4644 }
4645
ExpandIntRes_SIGN_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)4646 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
4647 SDValue &Lo, SDValue &Hi) {
4648 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4649 SDLoc dl(N);
4650 SDValue Op = N->getOperand(0);
4651 if (Op.getValueType().bitsLE(NVT)) {
4652 // The low part is sign extension of the input (degenerates to a copy).
4653 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
4654 // The high part is obtained by SRA'ing all but one of the bits of low part.
4655 unsigned LoSize = NVT.getSizeInBits();
4656 Hi = DAG.getNode(
4657 ISD::SRA, dl, NVT, Lo,
4658 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
4659 } else {
4660 // For example, extension of an i48 to an i64. The operand type necessarily
4661 // promotes to the result type, so will end up being expanded too.
4662 assert(getTypeAction(Op.getValueType()) ==
4663 TargetLowering::TypePromoteInteger &&
4664 "Only know how to promote this result!");
4665 SDValue Res = GetPromotedInteger(Op);
4666 assert(Res.getValueType() == N->getValueType(0) &&
4667 "Operand over promoted?");
4668 // Split the promoted operand. This will simplify when it is expanded.
4669 SplitInteger(Res, Lo, Hi);
4670 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4671 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4672 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
4673 ExcessBits)));
4674 }
4675 }
4676
4677 void DAGTypeLegalizer::
ExpandIntRes_SIGN_EXTEND_INREG(SDNode * N,SDValue & Lo,SDValue & Hi)4678 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4679 SDLoc dl(N);
4680 GetExpandedInteger(N->getOperand(0), Lo, Hi);
4681 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
4682
4683 if (EVT.bitsLE(Lo.getValueType())) {
4684 // sext_inreg the low part if needed.
4685 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
4686 N->getOperand(1));
4687
4688 // The high part gets the sign extension from the lo-part. This handles
4689 // things like sextinreg V:i64 from i8.
4690 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
4691 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
4692 TLI.getPointerTy(DAG.getDataLayout())));
4693 } else {
4694 // For example, extension of an i48 to an i64. Leave the low part alone,
4695 // sext_inreg the high part.
4696 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
4697 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4698 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
4699 ExcessBits)));
4700 }
4701 }
4702
ExpandIntRes_SREM(SDNode * N,SDValue & Lo,SDValue & Hi)4703 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
4704 SDValue &Lo, SDValue &Hi) {
4705 EVT VT = N->getValueType(0);
4706 SDLoc dl(N);
4707 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4708
4709 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
4710 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4711 SplitInteger(Res.getValue(1), Lo, Hi);
4712 return;
4713 }
4714
4715 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4716 if (VT == MVT::i16)
4717 LC = RTLIB::SREM_I16;
4718 else if (VT == MVT::i32)
4719 LC = RTLIB::SREM_I32;
4720 else if (VT == MVT::i64)
4721 LC = RTLIB::SREM_I64;
4722 else if (VT == MVT::i128)
4723 LC = RTLIB::SREM_I128;
4724 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
4725
4726 TargetLowering::MakeLibCallOptions CallOptions;
4727 CallOptions.setSExt(true);
4728 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4729 }
4730
ExpandIntRes_TRUNCATE(SDNode * N,SDValue & Lo,SDValue & Hi)4731 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
4732 SDValue &Lo, SDValue &Hi) {
4733 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4734 SDLoc dl(N);
4735 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
4736 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
4737 N->getOperand(0),
4738 DAG.getConstant(NVT.getSizeInBits(), dl,
4739 TLI.getPointerTy(DAG.getDataLayout())));
4740 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
4741 }
4742
ExpandIntRes_XMULO(SDNode * N,SDValue & Lo,SDValue & Hi)4743 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
4744 SDValue &Lo, SDValue &Hi) {
4745 EVT VT = N->getValueType(0);
4746 SDLoc dl(N);
4747
4748 if (N->getOpcode() == ISD::UMULO) {
4749 // This section expands the operation into the following sequence of
4750 // instructions. `iNh` here refers to a type which has half the bit width of
4751 // the type the original operation operated on.
4752 //
4753 // %0 = %LHS.HI != 0 && %RHS.HI != 0
4754 // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
4755 // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
4756 // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
4757 // %4 = add iNh %1.0, %2.0 as iN
4758 // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
4759 //
4760 // %lo = %3.LO
4761 // %hi = %5.0
4762 // %ovf = %0 || %1.1 || %2.1 || %5.1
4763 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
4764 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4765 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4766 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4767 EVT HalfVT = LHSLow.getValueType();
4768 EVT BitVT = N->getValueType(1);
4769 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
4770
4771 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
4772 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
4773 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
4774 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
4775
4776 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
4777 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
4778
4779 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
4780 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
4781
4782 SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
4783
4784 // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
4785 // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
4786 // operation recursively legalized?).
4787 //
4788 // Many backends understand this pattern and will convert into LOHI
4789 // themselves, if applicable.
4790 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
4791 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
4792 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
4793 SplitInteger(Three, Lo, Hi);
4794
4795 Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
4796 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
4797 ReplaceValueWith(SDValue(N, 1), Overflow);
4798 return;
4799 }
4800
4801 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
4802 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
4803 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
4804
4805 // Replace this with a libcall that will check overflow.
4806 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4807 if (VT == MVT::i32)
4808 LC = RTLIB::MULO_I32;
4809 else if (VT == MVT::i64)
4810 LC = RTLIB::MULO_I64;
4811 else if (VT == MVT::i128)
4812 LC = RTLIB::MULO_I128;
4813
4814 // If we don't have the libcall or if the function we are compiling is the
4815 // implementation of the expected libcall (avoid inf-loop), expand inline.
4816 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC) ||
4817 TLI.getLibcallName(LC) == DAG.getMachineFunction().getName()) {
4818 // FIXME: This is not an optimal expansion, but better than crashing.
4819 EVT WideVT =
4820 EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
4821 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0));
4822 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1));
4823 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
4824 SDValue MulLo, MulHi;
4825 SplitInteger(Mul, MulLo, MulHi);
4826 SDValue SRA =
4827 DAG.getNode(ISD::SRA, dl, VT, MulLo,
4828 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT));
4829 SDValue Overflow =
4830 DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE);
4831 SplitInteger(MulLo, Lo, Hi);
4832 ReplaceValueWith(SDValue(N, 1), Overflow);
4833 return;
4834 }
4835
4836 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
4837 // Temporary for the overflow value, default it to zero.
4838 SDValue Chain =
4839 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
4840 MachinePointerInfo());
4841
4842 TargetLowering::ArgListTy Args;
4843 TargetLowering::ArgListEntry Entry;
4844 for (const SDValue &Op : N->op_values()) {
4845 EVT ArgVT = Op.getValueType();
4846 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
4847 Entry.Node = Op;
4848 Entry.Ty = ArgTy;
4849 Entry.IsSExt = true;
4850 Entry.IsZExt = false;
4851 Args.push_back(Entry);
4852 }
4853
4854 // Also pass the address of the overflow check.
4855 Entry.Node = Temp;
4856 Entry.Ty = PointerType::getUnqual(PtrTy->getContext());
4857 Entry.IsSExt = true;
4858 Entry.IsZExt = false;
4859 Args.push_back(Entry);
4860
4861 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
4862
4863 TargetLowering::CallLoweringInfo CLI(DAG);
4864 CLI.setDebugLoc(dl)
4865 .setChain(Chain)
4866 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
4867 .setSExtResult();
4868
4869 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
4870
4871 SplitInteger(CallInfo.first, Lo, Hi);
4872 SDValue Temp2 =
4873 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
4874 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
4875 DAG.getConstant(0, dl, PtrVT),
4876 ISD::SETNE);
4877 // Use the overflow from the libcall everywhere.
4878 ReplaceValueWith(SDValue(N, 1), Ofl);
4879 }
4880
ExpandIntRes_UDIV(SDNode * N,SDValue & Lo,SDValue & Hi)4881 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
4882 SDValue &Lo, SDValue &Hi) {
4883 EVT VT = N->getValueType(0);
4884 SDLoc dl(N);
4885 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4886
4887 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4888 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4889 SplitInteger(Res.getValue(0), Lo, Hi);
4890 return;
4891 }
4892
4893 // Try to expand UDIV by constant.
4894 if (isa<ConstantSDNode>(N->getOperand(1))) {
4895 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4896 // Only if the new type is legal.
4897 if (isTypeLegal(NVT)) {
4898 SDValue InL, InH;
4899 GetExpandedInteger(N->getOperand(0), InL, InH);
4900 SmallVector<SDValue> Result;
4901 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) {
4902 Lo = Result[0];
4903 Hi = Result[1];
4904 return;
4905 }
4906 }
4907 }
4908
4909 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4910 if (VT == MVT::i16)
4911 LC = RTLIB::UDIV_I16;
4912 else if (VT == MVT::i32)
4913 LC = RTLIB::UDIV_I32;
4914 else if (VT == MVT::i64)
4915 LC = RTLIB::UDIV_I64;
4916 else if (VT == MVT::i128)
4917 LC = RTLIB::UDIV_I128;
4918 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
4919
4920 TargetLowering::MakeLibCallOptions CallOptions;
4921 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4922 }
4923
ExpandIntRes_UREM(SDNode * N,SDValue & Lo,SDValue & Hi)4924 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
4925 SDValue &Lo, SDValue &Hi) {
4926 EVT VT = N->getValueType(0);
4927 SDLoc dl(N);
4928 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4929
4930 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
4931 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4932 SplitInteger(Res.getValue(1), Lo, Hi);
4933 return;
4934 }
4935
4936 // Try to expand UREM by constant.
4937 if (isa<ConstantSDNode>(N->getOperand(1))) {
4938 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4939 // Only if the new type is legal.
4940 if (isTypeLegal(NVT)) {
4941 SDValue InL, InH;
4942 GetExpandedInteger(N->getOperand(0), InL, InH);
4943 SmallVector<SDValue> Result;
4944 if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) {
4945 Lo = Result[0];
4946 Hi = Result[1];
4947 return;
4948 }
4949 }
4950 }
4951
4952 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4953 if (VT == MVT::i16)
4954 LC = RTLIB::UREM_I16;
4955 else if (VT == MVT::i32)
4956 LC = RTLIB::UREM_I32;
4957 else if (VT == MVT::i64)
4958 LC = RTLIB::UREM_I64;
4959 else if (VT == MVT::i128)
4960 LC = RTLIB::UREM_I128;
4961 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
4962
4963 TargetLowering::MakeLibCallOptions CallOptions;
4964 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4965 }
4966
ExpandIntRes_ZERO_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)4967 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
4968 SDValue &Lo, SDValue &Hi) {
4969 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4970 SDLoc dl(N);
4971 SDValue Op = N->getOperand(0);
4972 if (Op.getValueType().bitsLE(NVT)) {
4973 // The low part is zero extension of the input (degenerates to a copy).
4974 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
4975 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
4976 } else {
4977 // For example, extension of an i48 to an i64. The operand type necessarily
4978 // promotes to the result type, so will end up being expanded too.
4979 assert(getTypeAction(Op.getValueType()) ==
4980 TargetLowering::TypePromoteInteger &&
4981 "Only know how to promote this result!");
4982 SDValue Res = GetPromotedInteger(Op);
4983 assert(Res.getValueType() == N->getValueType(0) &&
4984 "Operand over promoted?");
4985 // Split the promoted operand. This will simplify when it is expanded.
4986 SplitInteger(Res, Lo, Hi);
4987 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4988 Hi = DAG.getZeroExtendInReg(Hi, dl,
4989 EVT::getIntegerVT(*DAG.getContext(),
4990 ExcessBits));
4991 }
4992 }
4993
ExpandIntRes_ATOMIC_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)4994 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
4995 SDValue &Lo, SDValue &Hi) {
4996 SDLoc dl(N);
4997 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
4998 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
4999 SDValue Zero = DAG.getConstant(0, dl, VT);
5000 SDValue Swap = DAG.getAtomicCmpSwap(
5001 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
5002 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
5003 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
5004
5005 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
5006 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
5007 }
5008
ExpandIntRes_VECREDUCE(SDNode * N,SDValue & Lo,SDValue & Hi)5009 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
5010 SDValue &Lo, SDValue &Hi) {
5011 // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
5012 // both halves independently.
5013 SDValue Res = TLI.expandVecReduce(N, DAG);
5014 SplitInteger(Res, Lo, Hi);
5015 }
5016
ExpandIntRes_Rotate(SDNode * N,SDValue & Lo,SDValue & Hi)5017 void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
5018 SDValue &Lo, SDValue &Hi) {
5019 // Delegate to funnel-shift expansion.
5020 SDLoc DL(N);
5021 unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
5022 SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0),
5023 N->getOperand(0), N->getOperand(1));
5024 SplitInteger(Res, Lo, Hi);
5025 }
5026
ExpandIntRes_FunnelShift(SDNode * N,SDValue & Lo,SDValue & Hi)5027 void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
5028 SDValue &Hi) {
5029 // Values numbered from least significant to most significant.
5030 SDValue In1, In2, In3, In4;
5031 GetExpandedInteger(N->getOperand(0), In3, In4);
5032 GetExpandedInteger(N->getOperand(1), In1, In2);
5033 EVT HalfVT = In1.getValueType();
5034
5035 SDLoc DL(N);
5036 unsigned Opc = N->getOpcode();
5037 SDValue ShAmt = N->getOperand(2);
5038 EVT ShAmtVT = ShAmt.getValueType();
5039 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5040
5041 // If the shift amount is at least half the bitwidth, swap the inputs.
5042 unsigned HalfVTBits = HalfVT.getScalarSizeInBits();
5043 SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt,
5044 DAG.getConstant(HalfVTBits, DL, ShAmtVT));
5045 SDValue Cond =
5046 DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT),
5047 Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ);
5048
5049 // Expand to a pair of funnel shifts.
5050 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
5051 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT);
5052
5053 SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2);
5054 SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3);
5055 SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4);
5056 Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt);
5057 Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt);
5058 }
5059
ExpandIntRes_VSCALE(SDNode * N,SDValue & Lo,SDValue & Hi)5060 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
5061 SDValue &Hi) {
5062 EVT VT = N->getValueType(0);
5063 EVT HalfVT =
5064 EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2);
5065 SDLoc dl(N);
5066
5067 // We assume VSCALE(1) fits into a legal integer.
5068 APInt One(HalfVT.getSizeInBits(), 1);
5069 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One);
5070 VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase);
5071 SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0));
5072 SplitInteger(Res, Lo, Hi);
5073 }
5074
5075 //===----------------------------------------------------------------------===//
5076 // Integer Operand Expansion
5077 //===----------------------------------------------------------------------===//
5078
5079 /// ExpandIntegerOperand - This method is called when the specified operand of
5080 /// the specified node is found to need expansion. At this point, all of the
5081 /// result types of the node are known to be legal, but other operands of the
5082 /// node may need promotion or expansion as well as the specified one.
ExpandIntegerOperand(SDNode * N,unsigned OpNo)5083 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
5084 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG));
5085 SDValue Res = SDValue();
5086
5087 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
5088 return false;
5089
5090 switch (N->getOpcode()) {
5091 default:
5092 #ifndef NDEBUG
5093 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
5094 N->dump(&DAG); dbgs() << "\n";
5095 #endif
5096 report_fatal_error("Do not know how to expand this operator's operand!");
5097
5098 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
5099 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
5100 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
5101 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
5102 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
5103 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
5104 case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
5105 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
5106 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
5107 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
5108 case ISD::STRICT_SINT_TO_FP:
5109 case ISD::SINT_TO_FP:
5110 case ISD::STRICT_UINT_TO_FP:
5111 case ISD::UINT_TO_FP: Res = ExpandIntOp_XINT_TO_FP(N); break;
5112 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
5113 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
5114
5115 case ISD::SHL:
5116 case ISD::SRA:
5117 case ISD::SRL:
5118 case ISD::ROTL:
5119 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
5120 case ISD::RETURNADDR:
5121 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
5122
5123 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
5124 case ISD::STACKMAP:
5125 Res = ExpandIntOp_STACKMAP(N, OpNo);
5126 break;
5127 case ISD::PATCHPOINT:
5128 Res = ExpandIntOp_PATCHPOINT(N, OpNo);
5129 break;
5130 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5131 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5132 Res = ExpandIntOp_VP_STRIDED(N, OpNo);
5133 break;
5134 }
5135
5136 // If the result is null, the sub-method took care of registering results etc.
5137 if (!Res.getNode()) return false;
5138
5139 // If the result is N, the sub-method updated N in place. Tell the legalizer
5140 // core about this.
5141 if (Res.getNode() == N)
5142 return true;
5143
5144 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
5145 "Invalid operand expansion");
5146
5147 ReplaceValueWith(SDValue(N, 0), Res);
5148 return false;
5149 }
5150
5151 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
5152 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
IntegerExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl)5153 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
5154 SDValue &NewRHS,
5155 ISD::CondCode &CCCode,
5156 const SDLoc &dl) {
5157 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5158 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5159 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5160
5161 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
5162 if (RHSLo == RHSHi && isAllOnesConstant(RHSLo)) {
5163 // Equality comparison to -1.
5164 NewLHS = DAG.getNode(ISD::AND, dl, LHSLo.getValueType(), LHSLo, LHSHi);
5165 NewRHS = RHSLo;
5166 return;
5167 }
5168
5169 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
5170 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
5171 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
5172 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
5173 return;
5174 }
5175
5176 // If this is a comparison of the sign bit, just look at the top part.
5177 // X > -1, x < 0
5178 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
5179 if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0
5180 (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1
5181 NewLHS = LHSHi;
5182 NewRHS = RHSHi;
5183 return;
5184 }
5185
5186 // FIXME: This generated code sucks.
5187 ISD::CondCode LowCC;
5188 switch (CCCode) {
5189 default: llvm_unreachable("Unknown integer setcc!");
5190 case ISD::SETLT:
5191 case ISD::SETULT: LowCC = ISD::SETULT; break;
5192 case ISD::SETGT:
5193 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
5194 case ISD::SETLE:
5195 case ISD::SETULE: LowCC = ISD::SETULE; break;
5196 case ISD::SETGE:
5197 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
5198 }
5199
5200 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
5201 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
5202 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
5203
5204 // NOTE: on targets without efficient SELECT of bools, we can always use
5205 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
5206 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
5207 nullptr);
5208 SDValue LoCmp, HiCmp;
5209 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
5210 TLI.isTypeLegal(RHSLo.getValueType()))
5211 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
5212 RHSLo, LowCC, false, DagCombineInfo, dl);
5213 if (!LoCmp.getNode())
5214 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
5215 RHSLo, LowCC);
5216 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
5217 TLI.isTypeLegal(RHSHi.getValueType()))
5218 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
5219 RHSHi, CCCode, false, DagCombineInfo, dl);
5220 if (!HiCmp.getNode())
5221 HiCmp =
5222 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
5223 LHSHi, RHSHi, DAG.getCondCode(CCCode));
5224
5225 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
5226 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
5227
5228 bool EqAllowed = ISD::isTrueWhenEqual(CCCode);
5229
5230 // FIXME: Is the HiCmpC->isOne() here correct for
5231 // ZeroOrNegativeOneBooleanContent.
5232 if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) ||
5233 (!EqAllowed &&
5234 ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) {
5235 // For LE / GE, if high part is known false, ignore the low part.
5236 // For LT / GT: if low part is known false, return the high part.
5237 // if high part is known true, ignore the low part.
5238 NewLHS = HiCmp;
5239 NewRHS = SDValue();
5240 return;
5241 }
5242
5243 if (LHSHi == RHSHi) {
5244 // Comparing the low bits is enough.
5245 NewLHS = LoCmp;
5246 NewRHS = SDValue();
5247 return;
5248 }
5249
5250 // Lower with SETCCCARRY if the target supports it.
5251 EVT HiVT = LHSHi.getValueType();
5252 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
5253 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
5254
5255 // FIXME: Make all targets support this, then remove the other lowering.
5256 if (HasSETCCCARRY) {
5257 // SETCCCARRY can detect < and >= directly. For > and <=, flip
5258 // operands and condition code.
5259 bool FlipOperands = false;
5260 switch (CCCode) {
5261 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
5262 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
5263 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
5264 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
5265 default: break;
5266 }
5267 if (FlipOperands) {
5268 std::swap(LHSLo, RHSLo);
5269 std::swap(LHSHi, RHSHi);
5270 }
5271 // Perform a wide subtraction, feeding the carry from the low part into
5272 // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
5273 // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
5274 // zero or positive iff LHS >= RHS.
5275 EVT LoVT = LHSLo.getValueType();
5276 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
5277 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
5278 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
5279 LHSHi, RHSHi, LowCmp.getValue(1),
5280 DAG.getCondCode(CCCode));
5281 NewLHS = Res;
5282 NewRHS = SDValue();
5283 return;
5284 }
5285
5286 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
5287 false, DagCombineInfo, dl);
5288 if (!NewLHS.getNode())
5289 NewLHS =
5290 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
5291 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
5292 NewRHS = SDValue();
5293 }
5294
ExpandIntOp_BR_CC(SDNode * N)5295 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
5296 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
5297 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
5298 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5299
5300 // If ExpandSetCCOperands returned a scalar, we need to compare the result
5301 // against zero to select between true and false values.
5302 if (!NewRHS.getNode()) {
5303 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
5304 CCCode = ISD::SETNE;
5305 }
5306
5307 // Update N to have the operands specified.
5308 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
5309 DAG.getCondCode(CCCode), NewLHS, NewRHS,
5310 N->getOperand(4)), 0);
5311 }
5312
ExpandIntOp_SELECT_CC(SDNode * N)5313 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
5314 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
5315 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
5316 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5317
5318 // If ExpandSetCCOperands returned a scalar, we need to compare the result
5319 // against zero to select between true and false values.
5320 if (!NewRHS.getNode()) {
5321 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
5322 CCCode = ISD::SETNE;
5323 }
5324
5325 // Update N to have the operands specified.
5326 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
5327 N->getOperand(2), N->getOperand(3),
5328 DAG.getCondCode(CCCode)), 0);
5329 }
5330
ExpandIntOp_SETCC(SDNode * N)5331 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
5332 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
5333 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
5334 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
5335
5336 // If ExpandSetCCOperands returned a scalar, use it.
5337 if (!NewRHS.getNode()) {
5338 assert(NewLHS.getValueType() == N->getValueType(0) &&
5339 "Unexpected setcc expansion!");
5340 return NewLHS;
5341 }
5342
5343 // Otherwise, update N to have the operands specified.
5344 return SDValue(
5345 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
5346 }
5347
ExpandIntOp_SETCCCARRY(SDNode * N)5348 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
5349 SDValue LHS = N->getOperand(0);
5350 SDValue RHS = N->getOperand(1);
5351 SDValue Carry = N->getOperand(2);
5352 SDValue Cond = N->getOperand(3);
5353 SDLoc dl = SDLoc(N);
5354
5355 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5356 GetExpandedInteger(LHS, LHSLo, LHSHi);
5357 GetExpandedInteger(RHS, RHSLo, RHSHi);
5358
5359 // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high.
5360 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
5361 SDValue LowCmp =
5362 DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry);
5363 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
5364 LowCmp.getValue(1), Cond);
5365 }
5366
ExpandIntOp_SPLAT_VECTOR(SDNode * N)5367 SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
5368 // Split the operand and replace with SPLAT_VECTOR_PARTS.
5369 SDValue Lo, Hi;
5370 GetExpandedInteger(N->getOperand(0), Lo, Hi);
5371 return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo,
5372 Hi);
5373 }
5374
ExpandIntOp_Shift(SDNode * N)5375 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
5376 // The value being shifted is legal, but the shift amount is too big.
5377 // It follows that either the result of the shift is undefined, or the
5378 // upper half of the shift amount is zero. Just use the lower half.
5379 SDValue Lo, Hi;
5380 GetExpandedInteger(N->getOperand(1), Lo, Hi);
5381 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
5382 }
5383
ExpandIntOp_RETURNADDR(SDNode * N)5384 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
5385 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
5386 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
5387 // constant to valid type.
5388 SDValue Lo, Hi;
5389 GetExpandedInteger(N->getOperand(0), Lo, Hi);
5390 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
5391 }
5392
ExpandIntOp_XINT_TO_FP(SDNode * N)5393 SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) {
5394 bool IsStrict = N->isStrictFPOpcode();
5395 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
5396 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
5397 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
5398 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
5399 EVT DstVT = N->getValueType(0);
5400 RTLIB::Libcall LC = IsSigned ? RTLIB::getSINTTOFP(Op.getValueType(), DstVT)
5401 : RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
5402 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5403 "Don't know how to expand this XINT_TO_FP!");
5404 TargetLowering::MakeLibCallOptions CallOptions;
5405 CallOptions.setSExt(true);
5406 std::pair<SDValue, SDValue> Tmp =
5407 TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);
5408
5409 if (!IsStrict)
5410 return Tmp.first;
5411
5412 ReplaceValueWith(SDValue(N, 1), Tmp.second);
5413 ReplaceValueWith(SDValue(N, 0), Tmp.first);
5414 return SDValue();
5415 }
5416
ExpandIntOp_STORE(StoreSDNode * N,unsigned OpNo)5417 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
5418 assert(!N->isAtomic() && "Should have been a ATOMIC_STORE?");
5419
5420 if (ISD::isNormalStore(N))
5421 return ExpandOp_NormalStore(N, OpNo);
5422
5423 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
5424 assert(OpNo == 1 && "Can only expand the stored value so far");
5425
5426 EVT VT = N->getOperand(1).getValueType();
5427 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5428 SDValue Ch = N->getChain();
5429 SDValue Ptr = N->getBasePtr();
5430 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
5431 AAMDNodes AAInfo = N->getAAInfo();
5432 SDLoc dl(N);
5433 SDValue Lo, Hi;
5434
5435 assert(NVT.isByteSized() && "Expanded type not byte sized!");
5436
5437 if (N->getMemoryVT().bitsLE(NVT)) {
5438 GetExpandedInteger(N->getValue(), Lo, Hi);
5439 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
5440 N->getMemoryVT(), N->getOriginalAlign(), MMOFlags,
5441 AAInfo);
5442 }
5443
5444 if (DAG.getDataLayout().isLittleEndian()) {
5445 // Little-endian - low bits are at low addresses.
5446 GetExpandedInteger(N->getValue(), Lo, Hi);
5447
5448 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
5449 N->getOriginalAlign(), MMOFlags, AAInfo);
5450
5451 unsigned ExcessBits =
5452 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
5453 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
5454
5455 // Increment the pointer to the other half.
5456 unsigned IncrementSize = NVT.getSizeInBits()/8;
5457 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
5458 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
5459 N->getPointerInfo().getWithOffset(IncrementSize),
5460 NEVT, N->getOriginalAlign(), MMOFlags, AAInfo);
5461 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
5462 }
5463
5464 // Big-endian - high bits are at low addresses. Favor aligned stores at
5465 // the cost of some bit-fiddling.
5466 GetExpandedInteger(N->getValue(), Lo, Hi);
5467
5468 EVT ExtVT = N->getMemoryVT();
5469 unsigned EBytes = ExtVT.getStoreSize();
5470 unsigned IncrementSize = NVT.getSizeInBits()/8;
5471 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5472 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
5473 ExtVT.getSizeInBits() - ExcessBits);
5474
5475 if (ExcessBits < NVT.getSizeInBits()) {
5476 // Transfer high bits from the top of Lo to the bottom of Hi.
5477 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
5478 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
5479 TLI.getPointerTy(DAG.getDataLayout())));
5480 Hi = DAG.getNode(
5481 ISD::OR, dl, NVT, Hi,
5482 DAG.getNode(ISD::SRL, dl, NVT, Lo,
5483 DAG.getConstant(ExcessBits, dl,
5484 TLI.getPointerTy(DAG.getDataLayout()))));
5485 }
5486
5487 // Store both the high bits and maybe some of the low bits.
5488 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
5489 N->getOriginalAlign(), MMOFlags, AAInfo);
5490
5491 // Increment the pointer to the other half.
5492 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
5493 // Store the lowest ExcessBits bits in the second half.
5494 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
5495 N->getPointerInfo().getWithOffset(IncrementSize),
5496 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
5497 N->getOriginalAlign(), MMOFlags, AAInfo);
5498 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
5499 }
5500
ExpandIntOp_TRUNCATE(SDNode * N)5501 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
5502 SDValue InL, InH;
5503 GetExpandedInteger(N->getOperand(0), InL, InH);
5504 // Just truncate the low part of the source.
5505 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
5506 }
5507
ExpandIntOp_ATOMIC_STORE(SDNode * N)5508 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
5509 SDLoc dl(N);
5510 SDValue Swap =
5511 DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(N)->getMemoryVT(),
5512 N->getOperand(0), N->getOperand(2), N->getOperand(1),
5513 cast<AtomicSDNode>(N)->getMemOperand());
5514 return Swap.getValue(1);
5515 }
5516
ExpandIntOp_VP_STRIDED(SDNode * N,unsigned OpNo)5517 SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
5518 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5519 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5520
5521 SDValue Hi; // The upper half is dropped out.
5522 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end());
5523 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi);
5524
5525 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
5526 }
5527
PromoteIntRes_VECTOR_SPLICE(SDNode * N)5528 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
5529 SDLoc dl(N);
5530
5531 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5532 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5533 EVT OutVT = V0.getValueType();
5534
5535 return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2));
5536 }
5537
PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode * N)5538 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
5539 SDLoc dl(N);
5540
5541 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5542 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5543 EVT ResVT = V0.getValueType();
5544 SDValue Res = DAG.getNode(N->getOpcode(), dl,
5545 DAG.getVTList(ResVT, ResVT), V0, V1);
5546 SetPromotedInteger(SDValue(N, 0), Res.getValue(0));
5547 SetPromotedInteger(SDValue(N, 1), Res.getValue(1));
5548 return SDValue();
5549 }
5550
PromoteIntRes_EXTRACT_SUBVECTOR(SDNode * N)5551 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
5552
5553 EVT OutVT = N->getValueType(0);
5554 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5555 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5556 EVT NOutVTElem = NOutVT.getVectorElementType();
5557
5558 SDLoc dl(N);
5559 SDValue BaseIdx = N->getOperand(1);
5560
5561 // TODO: We may be able to use this for types other than scalable
5562 // vectors and fix those tests that expect BUILD_VECTOR to be used
5563 if (OutVT.isScalableVector()) {
5564 SDValue InOp0 = N->getOperand(0);
5565 EVT InVT = InOp0.getValueType();
5566
5567 // Try and extract from a smaller type so that it eventually falls
5568 // into the promotion code below.
5569 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector ||
5570 getTypeAction(InVT) == TargetLowering::TypeLegal) {
5571 EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext());
5572 unsigned NElts = NInVT.getVectorMinNumElements();
5573 uint64_t IdxVal = BaseIdx->getAsZExtVal();
5574
5575 SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0,
5576 DAG.getConstant(alignDown(IdxVal, NElts), dl,
5577 BaseIdx.getValueType()));
5578 SDValue Step2 = DAG.getNode(
5579 ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1,
5580 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType()));
5581 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2);
5582 }
5583
5584 // Try and extract from a widened type.
5585 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
5586 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5587 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops);
5588 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
5589 }
5590
5591 // Promote operands and see if this is handled by target lowering,
5592 // Otherwise, use the BUILD_VECTOR approach below
5593 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
5594 // Collect the (promoted) operands
5595 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5596
5597 EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
5598 assert(PromEltVT.bitsLE(NOutVTElem) &&
5599 "Promoted operand has an element type greater than result");
5600
5601 EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
5602 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
5603 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
5604 }
5605 }
5606
5607 if (OutVT.isScalableVector())
5608 report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");
5609
5610 SDValue InOp0 = N->getOperand(0);
5611 if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
5612 InOp0 = GetPromotedInteger(N->getOperand(0));
5613
5614 EVT InVT = InOp0.getValueType();
5615
5616 unsigned OutNumElems = OutVT.getVectorNumElements();
5617 SmallVector<SDValue, 8> Ops;
5618 Ops.reserve(OutNumElems);
5619 for (unsigned i = 0; i != OutNumElems; ++i) {
5620
5621 // Extract the element from the original vector.
5622 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
5623 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
5624 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
5625 InVT.getVectorElementType(), N->getOperand(0), Index);
5626
5627 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
5628 // Insert the converted element to the new vector.
5629 Ops.push_back(Op);
5630 }
5631
5632 return DAG.getBuildVector(NOutVT, dl, Ops);
5633 }
5634
PromoteIntRes_INSERT_SUBVECTOR(SDNode * N)5635 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) {
5636 EVT OutVT = N->getValueType(0);
5637 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5638 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5639
5640 SDLoc dl(N);
5641 SDValue Vec = N->getOperand(0);
5642 SDValue SubVec = N->getOperand(1);
5643 SDValue Idx = N->getOperand(2);
5644
5645 EVT SubVecVT = SubVec.getValueType();
5646 EVT NSubVT =
5647 EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(),
5648 SubVecVT.getVectorElementCount());
5649
5650 Vec = GetPromotedInteger(Vec);
5651 SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec);
5652
5653 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx);
5654 }
5655
PromoteIntRes_VECTOR_REVERSE(SDNode * N)5656 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
5657 SDLoc dl(N);
5658
5659 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5660 EVT OutVT = V0.getValueType();
5661
5662 return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0);
5663 }
5664
PromoteIntRes_VECTOR_SHUFFLE(SDNode * N)5665 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
5666 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
5667 EVT VT = N->getValueType(0);
5668 SDLoc dl(N);
5669
5670 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
5671
5672 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5673 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5674 EVT OutVT = V0.getValueType();
5675
5676 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
5677 }
5678
PromoteIntRes_BUILD_VECTOR(SDNode * N)5679 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
5680 EVT OutVT = N->getValueType(0);
5681 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5682 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5683 unsigned NumElems = N->getNumOperands();
5684 EVT NOutVTElem = NOutVT.getVectorElementType();
5685 TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT);
5686 unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType);
5687 SDLoc dl(N);
5688
5689 SmallVector<SDValue, 8> Ops;
5690 Ops.reserve(NumElems);
5691 for (unsigned i = 0; i != NumElems; ++i) {
5692 SDValue Op = N->getOperand(i);
5693 EVT OpVT = Op.getValueType();
5694 // BUILD_VECTOR integer operand types are allowed to be larger than the
5695 // result's element type. This may still be true after the promotion. For
5696 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
5697 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
5698 if (OpVT.bitsLT(NOutVTElem)) {
5699 unsigned ExtOpc = ISD::ANY_EXTEND;
5700 // Attempt to extend constant bool vectors to match target's BooleanContent.
5701 // While not necessary, this improves chances of the constant correctly
5702 // folding with compare results (e.g. for NOT patterns).
5703 if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant)
5704 ExtOpc = NOutExtOpc;
5705 Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op);
5706 }
5707 Ops.push_back(Op);
5708 }
5709
5710 return DAG.getBuildVector(NOutVT, dl, Ops);
5711 }
5712
PromoteIntRes_ScalarOp(SDNode * N)5713 SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) {
5714
5715 SDLoc dl(N);
5716
5717 assert(!N->getOperand(0).getValueType().isVector() &&
5718 "Input must be a scalar");
5719
5720 EVT OutVT = N->getValueType(0);
5721 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5722 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5723 EVT NOutElemVT = NOutVT.getVectorElementType();
5724
5725 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0));
5726
5727 return DAG.getNode(N->getOpcode(), dl, NOutVT, Op);
5728 }
5729
PromoteIntRes_STEP_VECTOR(SDNode * N)5730 SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
5731 SDLoc dl(N);
5732 EVT OutVT = N->getValueType(0);
5733 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5734 assert(NOutVT.isScalableVector() &&
5735 "Type must be promoted to a scalable vector type");
5736 const APInt &StepVal = N->getConstantOperandAPInt(0);
5737 return DAG.getStepVector(dl, NOutVT,
5738 StepVal.sext(NOutVT.getScalarSizeInBits()));
5739 }
5740
PromoteIntRes_CONCAT_VECTORS(SDNode * N)5741 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
5742 SDLoc dl(N);
5743
5744 EVT OutVT = N->getValueType(0);
5745 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5746 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5747
5748 unsigned NumOperands = N->getNumOperands();
5749 unsigned NumOutElem = NOutVT.getVectorMinNumElements();
5750 EVT OutElemTy = NOutVT.getVectorElementType();
5751 if (OutVT.isScalableVector()) {
5752 // Find the largest promoted element type for each of the operands.
5753 SDUse *MaxSizedValue = std::max_element(
5754 N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) {
5755 EVT AVT = A.getValueType().getVectorElementType();
5756 EVT BVT = B.getValueType().getVectorElementType();
5757 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5758 });
5759 EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType();
5760
5761 // Then promote all vectors to the largest element type.
5762 SmallVector<SDValue, 8> Ops;
5763 for (unsigned I = 0; I < NumOperands; ++I) {
5764 SDValue Op = N->getOperand(I);
5765 EVT OpVT = Op.getValueType();
5766 if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger)
5767 Op = GetPromotedInteger(Op);
5768 else
5769 assert(getTypeAction(OpVT) == TargetLowering::TypeLegal &&
5770 "Unhandled legalization type");
5771
5772 if (OpVT.getVectorElementType().getScalarSizeInBits() <
5773 MaxElementVT.getScalarSizeInBits())
5774 Op = DAG.getAnyExtOrTrunc(Op, dl,
5775 OpVT.changeVectorElementType(MaxElementVT));
5776 Ops.push_back(Op);
5777 }
5778
5779 // Do the CONCAT on the promoted type and finally truncate to (the promoted)
5780 // NOutVT.
5781 return DAG.getAnyExtOrTrunc(
5782 DAG.getNode(ISD::CONCAT_VECTORS, dl,
5783 OutVT.changeVectorElementType(MaxElementVT), Ops),
5784 dl, NOutVT);
5785 }
5786
5787 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
5788 assert(NumElem * NumOperands == NumOutElem &&
5789 "Unexpected number of elements");
5790
5791 // Take the elements from the first vector.
5792 SmallVector<SDValue, 8> Ops(NumOutElem);
5793 for (unsigned i = 0; i < NumOperands; ++i) {
5794 SDValue Op = N->getOperand(i);
5795 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
5796 Op = GetPromotedInteger(Op);
5797 EVT SclrTy = Op.getValueType().getVectorElementType();
5798 assert(NumElem == Op.getValueType().getVectorNumElements() &&
5799 "Unexpected number of elements");
5800
5801 for (unsigned j = 0; j < NumElem; ++j) {
5802 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
5803 DAG.getVectorIdxConstant(j, dl));
5804 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
5805 }
5806 }
5807
5808 return DAG.getBuildVector(NOutVT, dl, Ops);
5809 }
5810
PromoteIntRes_EXTEND_VECTOR_INREG(SDNode * N)5811 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
5812 EVT VT = N->getValueType(0);
5813 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5814 assert(NVT.isVector() && "This type must be promoted to a vector type");
5815
5816 SDLoc dl(N);
5817
5818 // For operands whose TypeAction is to promote, extend the promoted node
5819 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
5820 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
5821 // type..
5822 if (getTypeAction(N->getOperand(0).getValueType())
5823 == TargetLowering::TypePromoteInteger) {
5824 SDValue Promoted;
5825
5826 switch(N->getOpcode()) {
5827 case ISD::SIGN_EXTEND_VECTOR_INREG:
5828 Promoted = SExtPromotedInteger(N->getOperand(0));
5829 break;
5830 case ISD::ZERO_EXTEND_VECTOR_INREG:
5831 Promoted = ZExtPromotedInteger(N->getOperand(0));
5832 break;
5833 case ISD::ANY_EXTEND_VECTOR_INREG:
5834 Promoted = GetPromotedInteger(N->getOperand(0));
5835 break;
5836 default:
5837 llvm_unreachable("Node has unexpected Opcode");
5838 }
5839 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
5840 }
5841
5842 // Directly extend to the appropriate transform-to type.
5843 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
5844 }
5845
PromoteIntRes_INSERT_VECTOR_ELT(SDNode * N)5846 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
5847 EVT OutVT = N->getValueType(0);
5848 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
5849 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
5850
5851 EVT NOutVTElem = NOutVT.getVectorElementType();
5852
5853 SDLoc dl(N);
5854 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5855
5856 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
5857 NOutVTElem, N->getOperand(1));
5858 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
5859 V0, ConvElem, N->getOperand(2));
5860 }
5861
PromoteIntRes_VECREDUCE(SDNode * N)5862 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
5863 // The VECREDUCE result size may be larger than the element size, so
5864 // we can simply change the result type.
5865 SDLoc dl(N);
5866 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5867 return DAG.getNode(N->getOpcode(), dl, NVT, N->ops());
5868 }
5869
PromoteIntRes_VP_REDUCE(SDNode * N)5870 SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) {
5871 // The VP_REDUCE result size may be larger than the element size, so we can
5872 // simply change the result type. However the start value and result must be
5873 // the same.
5874 SDLoc DL(N);
5875 SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0));
5876 return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start,
5877 N->getOperand(1), N->getOperand(2), N->getOperand(3));
5878 }
5879
PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode * N)5880 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
5881 SDLoc dl(N);
5882 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5883 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
5884 TLI.getVectorIdxTy(DAG.getDataLayout()));
5885 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
5886 V0->getValueType(0).getScalarType(), V0, V1);
5887
5888 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
5889 // element types. If this is the case then we need to expand the outgoing
5890 // value and not truncate it.
5891 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5892 }
5893
PromoteIntOp_INSERT_SUBVECTOR(SDNode * N)5894 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) {
5895 SDLoc dl(N);
5896 // The result type is equal to the first input operand's type, so the
5897 // type that needs promoting must be the second source vector.
5898 SDValue V0 = N->getOperand(0);
5899 SDValue V1 = GetPromotedInteger(N->getOperand(1));
5900 SDValue Idx = N->getOperand(2);
5901 EVT PromVT = EVT::getVectorVT(*DAG.getContext(),
5902 V1.getValueType().getVectorElementType(),
5903 V0.getValueType().getVectorElementCount());
5904 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT);
5905 SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx);
5906 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5907 }
5908
PromoteIntOp_EXTRACT_SUBVECTOR(SDNode * N)5909 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
5910 SDLoc dl(N);
5911 SDValue V0 = GetPromotedInteger(N->getOperand(0));
5912 MVT InVT = V0.getValueType().getSimpleVT();
5913 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
5914 N->getValueType(0).getVectorNumElements());
5915 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
5916 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
5917 }
5918
PromoteIntOp_CONCAT_VECTORS(SDNode * N)5919 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
5920 SDLoc dl(N);
5921
5922 EVT ResVT = N->getValueType(0);
5923 unsigned NumElems = N->getNumOperands();
5924
5925 if (ResVT.isScalableVector()) {
5926 SDValue ResVec = DAG.getUNDEF(ResVT);
5927
5928 for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
5929 SDValue Op = N->getOperand(OpIdx);
5930 unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
5931 ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
5932 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
5933 }
5934
5935 return ResVec;
5936 }
5937
5938 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
5939
5940 SmallVector<SDValue, 8> NewOps;
5941 NewOps.reserve(NumElems);
5942
5943 // For each incoming vector
5944 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
5945 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
5946 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
5947 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
5948
5949 for (unsigned i=0; i<NumElem; ++i) {
5950 // Extract element from incoming vector
5951 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
5952 DAG.getVectorIdxConstant(i, dl));
5953 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
5954 NewOps.push_back(Tr);
5955 }
5956 }
5957
5958 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
5959 }
5960
ExpandIntOp_STACKMAP(SDNode * N,unsigned OpNo)5961 SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
5962 assert(OpNo > 1);
5963 SDValue Op = N->getOperand(OpNo);
5964
5965 // FIXME: Non-constant operands are not yet handled:
5966 // - https://github.com/llvm/llvm-project/issues/26431
5967 // - https://github.com/llvm/llvm-project/issues/55957
5968 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
5969 if (!CN)
5970 return SDValue();
5971
5972 // Copy operands before the one being expanded.
5973 SmallVector<SDValue> NewOps;
5974 for (unsigned I = 0; I < OpNo; I++)
5975 NewOps.push_back(N->getOperand(I));
5976
5977 EVT Ty = Op.getValueType();
5978 SDLoc DL = SDLoc(N);
5979 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
5980 NewOps.push_back(
5981 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
5982 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
5983 } else {
5984 // FIXME: https://github.com/llvm/llvm-project/issues/55609
5985 return SDValue();
5986 }
5987
5988 // Copy remaining operands.
5989 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
5990 NewOps.push_back(N->getOperand(I));
5991
5992 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
5993
5994 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
5995 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
5996
5997 return SDValue(); // Signal that we have replaced the node already.
5998 }
5999
ExpandIntOp_PATCHPOINT(SDNode * N,unsigned OpNo)6000 SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
6001 assert(OpNo >= 7);
6002 SDValue Op = N->getOperand(OpNo);
6003
6004 // FIXME: Non-constant operands are not yet handled:
6005 // - https://github.com/llvm/llvm-project/issues/26431
6006 // - https://github.com/llvm/llvm-project/issues/55957
6007 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op);
6008 if (!CN)
6009 return SDValue();
6010
6011 // Copy operands before the one being expanded.
6012 SmallVector<SDValue> NewOps;
6013 for (unsigned I = 0; I < OpNo; I++)
6014 NewOps.push_back(N->getOperand(I));
6015
6016 EVT Ty = Op.getValueType();
6017 SDLoc DL = SDLoc(N);
6018 if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
6019 NewOps.push_back(
6020 DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
6021 NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
6022 } else {
6023 // FIXME: https://github.com/llvm/llvm-project/issues/55609
6024 return SDValue();
6025 }
6026
6027 // Copy remaining operands.
6028 for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
6029 NewOps.push_back(N->getOperand(I));
6030
6031 SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
6032
6033 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
6034 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
6035
6036 return SDValue(); // Signal that we have replaced the node already.
6037 }
6038