1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements float type expansion and softening for LegalizeTypes.
10 // Softening is the act of turning a computation in an illegal floating point
11 // type into a computation in an integer type of the same size; also known as
12 // "soft float". For example, turning f32 arithmetic into operations using i32.
13 // The resulting integer value is the same as what you would get by performing
14 // the floating point operation and bitcasting the result to the integer type.
15 // Expansion is the act of changing a computation in an illegal type to be a
16 // computation in two identical registers of a smaller type. For example,
17 // implementing ppcf128 arithmetic in two f64 registers.
18 //
19 //===----------------------------------------------------------------------===//
20
21 #include "LegalizeTypes.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include "llvm/Support/raw_ostream.h"
24 using namespace llvm;
25
26 #define DEBUG_TYPE "legalize-types"
27
28 /// GetFPLibCall - Return the right libcall for the given floating point type.
GetFPLibCall(EVT VT,RTLIB::Libcall Call_F32,RTLIB::Libcall Call_F64,RTLIB::Libcall Call_F80,RTLIB::Libcall Call_F128,RTLIB::Libcall Call_PPCF128)29 static RTLIB::Libcall GetFPLibCall(EVT VT,
30 RTLIB::Libcall Call_F32,
31 RTLIB::Libcall Call_F64,
32 RTLIB::Libcall Call_F80,
33 RTLIB::Libcall Call_F128,
34 RTLIB::Libcall Call_PPCF128) {
35 return
36 VT == MVT::f32 ? Call_F32 :
37 VT == MVT::f64 ? Call_F64 :
38 VT == MVT::f80 ? Call_F80 :
39 VT == MVT::f128 ? Call_F128 :
40 VT == MVT::ppcf128 ? Call_PPCF128 :
41 RTLIB::UNKNOWN_LIBCALL;
42 }
43
44 //===----------------------------------------------------------------------===//
45 // Convert Float Results to Integer
46 //===----------------------------------------------------------------------===//
47
SoftenFloatResult(SDNode * N,unsigned ResNo)48 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
49 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
50 dbgs() << "\n");
51 SDValue R = SDValue();
52
53 switch (N->getOpcode()) {
54 default:
55 #ifndef NDEBUG
56 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
57 N->dump(&DAG); dbgs() << "\n";
58 #endif
59 llvm_unreachable("Do not know how to soften the result of this operator!");
60
61 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
62 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
63 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
64 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
65 case ISD::EXTRACT_VECTOR_ELT:
66 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
67 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
68 case ISD::STRICT_FMINNUM:
69 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
70 case ISD::STRICT_FMAXNUM:
71 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
72 case ISD::STRICT_FADD:
73 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
74 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
75 case ISD::STRICT_FCEIL:
76 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
77 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
78 case ISD::STRICT_FCOS:
79 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
80 case ISD::STRICT_FDIV:
81 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
82 case ISD::STRICT_FEXP:
83 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
84 case ISD::STRICT_FEXP2:
85 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
86 case ISD::STRICT_FFLOOR:
87 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
88 case ISD::STRICT_FLOG:
89 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
90 case ISD::STRICT_FLOG2:
91 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
92 case ISD::STRICT_FLOG10:
93 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
94 case ISD::STRICT_FMA:
95 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
96 case ISD::STRICT_FMUL:
97 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
98 case ISD::STRICT_FNEARBYINT:
99 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
100 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
101 case ISD::STRICT_FP_EXTEND:
102 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
103 case ISD::STRICT_FP_ROUND:
104 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
105 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
106 case ISD::STRICT_FPOW:
107 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
108 case ISD::STRICT_FPOWI:
109 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break;
110 case ISD::STRICT_FREM:
111 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
112 case ISD::STRICT_FRINT:
113 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
114 case ISD::STRICT_FROUND:
115 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
116 case ISD::STRICT_FROUNDEVEN:
117 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
118 case ISD::STRICT_FSIN:
119 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
120 case ISD::STRICT_FSQRT:
121 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
122 case ISD::STRICT_FSUB:
123 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
124 case ISD::STRICT_FTRUNC:
125 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
126 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
127 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
128 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
129 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
130 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
131 case ISD::STRICT_SINT_TO_FP:
132 case ISD::STRICT_UINT_TO_FP:
133 case ISD::SINT_TO_FP:
134 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
135 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
136 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
137 case ISD::VECREDUCE_FADD:
138 case ISD::VECREDUCE_FMUL:
139 case ISD::VECREDUCE_FMIN:
140 case ISD::VECREDUCE_FMAX:
141 R = SoftenFloatRes_VECREDUCE(N);
142 break;
143 case ISD::VECREDUCE_SEQ_FADD:
144 case ISD::VECREDUCE_SEQ_FMUL:
145 R = SoftenFloatRes_VECREDUCE_SEQ(N);
146 break;
147 }
148
149 // If R is null, the sub-method took care of registering the result.
150 if (R.getNode()) {
151 assert(R.getNode() != N);
152 SetSoftenedFloat(SDValue(N, ResNo), R);
153 }
154 }
155
SoftenFloatRes_Unary(SDNode * N,RTLIB::Libcall LC)156 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
157 bool IsStrict = N->isStrictFPOpcode();
158 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
159 unsigned Offset = IsStrict ? 1 : 0;
160 assert(N->getNumOperands() == (1 + Offset) &&
161 "Unexpected number of operands!");
162 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
163 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
164 TargetLowering::MakeLibCallOptions CallOptions;
165 EVT OpVT = N->getOperand(0 + Offset).getValueType();
166 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
167 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
168 CallOptions, SDLoc(N),
169 Chain);
170 if (IsStrict)
171 ReplaceValueWith(SDValue(N, 1), Tmp.second);
172 return Tmp.first;
173 }
174
SoftenFloatRes_Binary(SDNode * N,RTLIB::Libcall LC)175 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
176 bool IsStrict = N->isStrictFPOpcode();
177 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
178 unsigned Offset = IsStrict ? 1 : 0;
179 assert(N->getNumOperands() == (2 + Offset) &&
180 "Unexpected number of operands!");
181 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
182 GetSoftenedFloat(N->getOperand(1 + Offset)) };
183 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
184 TargetLowering::MakeLibCallOptions CallOptions;
185 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
186 N->getOperand(1 + Offset).getValueType() };
187 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
188 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
189 CallOptions, SDLoc(N),
190 Chain);
191 if (IsStrict)
192 ReplaceValueWith(SDValue(N, 1), Tmp.second);
193 return Tmp.first;
194 }
195
SoftenFloatRes_BITCAST(SDNode * N)196 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
197 return BitConvertToInteger(N->getOperand(0));
198 }
199
SoftenFloatRes_FREEZE(SDNode * N)200 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
201 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
202 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
203 GetSoftenedFloat(N->getOperand(0)));
204 }
205
SoftenFloatRes_MERGE_VALUES(SDNode * N,unsigned ResNo)206 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
207 unsigned ResNo) {
208 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
209 return BitConvertToInteger(Op);
210 }
211
SoftenFloatRes_BUILD_PAIR(SDNode * N)212 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
213 // Convert the inputs to integers, and build a new pair out of them.
214 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
215 TLI.getTypeToTransformTo(*DAG.getContext(),
216 N->getValueType(0)),
217 BitConvertToInteger(N->getOperand(0)),
218 BitConvertToInteger(N->getOperand(1)));
219 }
220
SoftenFloatRes_ConstantFP(SDNode * N)221 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
222 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
223 // In ppcf128, the high 64 bits are always first in memory regardless
224 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
225 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
226 // way. However, APInt's are serialized in an Endian-sensitive fashion,
227 // so on big-Endian targets, the two doubles are output in the wrong
228 // order. Fix this by manually flipping the order of the high 64 bits
229 // and the low 64 bits here.
230 if (DAG.getDataLayout().isBigEndian() &&
231 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
232 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
233 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
234 APInt Val(128, words);
235 return DAG.getConstant(Val, SDLoc(CN),
236 TLI.getTypeToTransformTo(*DAG.getContext(),
237 CN->getValueType(0)));
238 } else {
239 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
240 TLI.getTypeToTransformTo(*DAG.getContext(),
241 CN->getValueType(0)));
242 }
243 }
244
SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode * N,unsigned ResNo)245 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
246 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
247 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
248 NewOp.getValueType().getVectorElementType(),
249 NewOp, N->getOperand(1));
250 }
251
SoftenFloatRes_FABS(SDNode * N)252 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
253 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
254 unsigned Size = NVT.getSizeInBits();
255
256 // Mask = ~(1 << (Size-1))
257 APInt API = APInt::getAllOnesValue(Size);
258 API.clearBit(Size - 1);
259 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
260 SDValue Op = GetSoftenedFloat(N->getOperand(0));
261 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
262 }
263
SoftenFloatRes_FMINNUM(SDNode * N)264 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
265 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
266 RTLIB::FMIN_F32,
267 RTLIB::FMIN_F64,
268 RTLIB::FMIN_F80,
269 RTLIB::FMIN_F128,
270 RTLIB::FMIN_PPCF128));
271 }
272
SoftenFloatRes_FMAXNUM(SDNode * N)273 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
274 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
275 RTLIB::FMAX_F32,
276 RTLIB::FMAX_F64,
277 RTLIB::FMAX_F80,
278 RTLIB::FMAX_F128,
279 RTLIB::FMAX_PPCF128));
280 }
281
SoftenFloatRes_FADD(SDNode * N)282 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
283 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
284 RTLIB::ADD_F32,
285 RTLIB::ADD_F64,
286 RTLIB::ADD_F80,
287 RTLIB::ADD_F128,
288 RTLIB::ADD_PPCF128));
289 }
290
SoftenFloatRes_FCBRT(SDNode * N)291 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
292 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
293 RTLIB::CBRT_F32,
294 RTLIB::CBRT_F64,
295 RTLIB::CBRT_F80,
296 RTLIB::CBRT_F128,
297 RTLIB::CBRT_PPCF128));
298 }
299
SoftenFloatRes_FCEIL(SDNode * N)300 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
301 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
302 RTLIB::CEIL_F32,
303 RTLIB::CEIL_F64,
304 RTLIB::CEIL_F80,
305 RTLIB::CEIL_F128,
306 RTLIB::CEIL_PPCF128));
307 }
308
SoftenFloatRes_FCOPYSIGN(SDNode * N)309 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
310 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
311 SDValue RHS = BitConvertToInteger(N->getOperand(1));
312 SDLoc dl(N);
313
314 EVT LVT = LHS.getValueType();
315 EVT RVT = RHS.getValueType();
316
317 unsigned LSize = LVT.getSizeInBits();
318 unsigned RSize = RVT.getSizeInBits();
319
320 // First get the sign bit of second operand.
321 SDValue SignBit = DAG.getNode(
322 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
323 DAG.getConstant(RSize - 1, dl,
324 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
325 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
326
327 // Shift right or sign-extend it if the two operands have different types.
328 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
329 if (SizeDiff > 0) {
330 SignBit =
331 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
332 DAG.getConstant(SizeDiff, dl,
333 TLI.getShiftAmountTy(SignBit.getValueType(),
334 DAG.getDataLayout())));
335 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
336 } else if (SizeDiff < 0) {
337 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
338 SignBit =
339 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
340 DAG.getConstant(-SizeDiff, dl,
341 TLI.getShiftAmountTy(SignBit.getValueType(),
342 DAG.getDataLayout())));
343 }
344
345 // Clear the sign bit of the first operand.
346 SDValue Mask = DAG.getNode(
347 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
348 DAG.getConstant(LSize - 1, dl,
349 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
350 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
351 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
352
353 // Or the value with the sign bit.
354 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
355 }
356
SoftenFloatRes_FCOS(SDNode * N)357 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
358 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
359 RTLIB::COS_F32,
360 RTLIB::COS_F64,
361 RTLIB::COS_F80,
362 RTLIB::COS_F128,
363 RTLIB::COS_PPCF128));
364 }
365
SoftenFloatRes_FDIV(SDNode * N)366 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
367 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
368 RTLIB::DIV_F32,
369 RTLIB::DIV_F64,
370 RTLIB::DIV_F80,
371 RTLIB::DIV_F128,
372 RTLIB::DIV_PPCF128));
373 }
374
SoftenFloatRes_FEXP(SDNode * N)375 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
376 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
377 RTLIB::EXP_F32,
378 RTLIB::EXP_F64,
379 RTLIB::EXP_F80,
380 RTLIB::EXP_F128,
381 RTLIB::EXP_PPCF128));
382 }
383
SoftenFloatRes_FEXP2(SDNode * N)384 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
385 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
386 RTLIB::EXP2_F32,
387 RTLIB::EXP2_F64,
388 RTLIB::EXP2_F80,
389 RTLIB::EXP2_F128,
390 RTLIB::EXP2_PPCF128));
391 }
392
SoftenFloatRes_FFLOOR(SDNode * N)393 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
394 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
395 RTLIB::FLOOR_F32,
396 RTLIB::FLOOR_F64,
397 RTLIB::FLOOR_F80,
398 RTLIB::FLOOR_F128,
399 RTLIB::FLOOR_PPCF128));
400 }
401
SoftenFloatRes_FLOG(SDNode * N)402 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
403 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
404 RTLIB::LOG_F32,
405 RTLIB::LOG_F64,
406 RTLIB::LOG_F80,
407 RTLIB::LOG_F128,
408 RTLIB::LOG_PPCF128));
409 }
410
SoftenFloatRes_FLOG2(SDNode * N)411 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
412 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
413 RTLIB::LOG2_F32,
414 RTLIB::LOG2_F64,
415 RTLIB::LOG2_F80,
416 RTLIB::LOG2_F128,
417 RTLIB::LOG2_PPCF128));
418 }
419
SoftenFloatRes_FLOG10(SDNode * N)420 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
421 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
422 RTLIB::LOG10_F32,
423 RTLIB::LOG10_F64,
424 RTLIB::LOG10_F80,
425 RTLIB::LOG10_F128,
426 RTLIB::LOG10_PPCF128));
427 }
428
SoftenFloatRes_FMA(SDNode * N)429 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
430 bool IsStrict = N->isStrictFPOpcode();
431 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
432 unsigned Offset = IsStrict ? 1 : 0;
433 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
434 GetSoftenedFloat(N->getOperand(1 + Offset)),
435 GetSoftenedFloat(N->getOperand(2 + Offset)) };
436 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
437 TargetLowering::MakeLibCallOptions CallOptions;
438 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
439 N->getOperand(1 + Offset).getValueType(),
440 N->getOperand(2 + Offset).getValueType() };
441 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
442 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
443 GetFPLibCall(N->getValueType(0),
444 RTLIB::FMA_F32,
445 RTLIB::FMA_F64,
446 RTLIB::FMA_F80,
447 RTLIB::FMA_F128,
448 RTLIB::FMA_PPCF128),
449 NVT, Ops, CallOptions, SDLoc(N), Chain);
450 if (IsStrict)
451 ReplaceValueWith(SDValue(N, 1), Tmp.second);
452 return Tmp.first;
453 }
454
SoftenFloatRes_FMUL(SDNode * N)455 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
456 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
457 RTLIB::MUL_F32,
458 RTLIB::MUL_F64,
459 RTLIB::MUL_F80,
460 RTLIB::MUL_F128,
461 RTLIB::MUL_PPCF128));
462 }
463
SoftenFloatRes_FNEARBYINT(SDNode * N)464 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
465 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
466 RTLIB::NEARBYINT_F32,
467 RTLIB::NEARBYINT_F64,
468 RTLIB::NEARBYINT_F80,
469 RTLIB::NEARBYINT_F128,
470 RTLIB::NEARBYINT_PPCF128));
471 }
472
SoftenFloatRes_FNEG(SDNode * N)473 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
474 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
475 SDLoc dl(N);
476
477 // Expand Y = FNEG(X) -> Y = X ^ sign mask
478 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
479 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
480 DAG.getConstant(SignMask, dl, NVT));
481 }
482
SoftenFloatRes_FP_EXTEND(SDNode * N)483 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
484 bool IsStrict = N->isStrictFPOpcode();
485 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
486 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
487
488 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
489
490 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
491 Op = GetPromotedFloat(Op);
492 // If the promotion did the FP_EXTEND to the destination type for us,
493 // there's nothing left to do here.
494 if (Op.getValueType() == N->getValueType(0))
495 return BitConvertToInteger(Op);
496 }
497
498 // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's
499 // entirely possible for both f16 and f32 to be legal, so use the fully
500 // hard-float FP_EXTEND rather than FP16_TO_FP.
501 if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) {
502 if (IsStrict) {
503 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
504 { MVT::f32, MVT::Other }, { Chain, Op });
505 Chain = Op.getValue(1);
506 } else {
507 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
508 }
509 }
510
511 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
512 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
513 TargetLowering::MakeLibCallOptions CallOptions;
514 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
515 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
516 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
517 CallOptions, SDLoc(N),
518 Chain);
519 if (IsStrict)
520 ReplaceValueWith(SDValue(N, 1), Tmp.second);
521 return Tmp.first;
522 }
523
524 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
525 // nodes?
SoftenFloatRes_FP16_TO_FP(SDNode * N)526 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
527 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
528 SDValue Op = N->getOperand(0);
529 TargetLowering::MakeLibCallOptions CallOptions;
530 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
531 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
532 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
533 CallOptions, SDLoc(N)).first;
534 if (N->getValueType(0) == MVT::f32)
535 return Res32;
536
537 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
538 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
539 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
540 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
541 }
542
SoftenFloatRes_FP_ROUND(SDNode * N)543 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
544 bool IsStrict = N->isStrictFPOpcode();
545 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
546 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
547 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
548 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
549 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
550 TargetLowering::MakeLibCallOptions CallOptions;
551 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
552 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
553 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
554 CallOptions, SDLoc(N),
555 Chain);
556 if (IsStrict)
557 ReplaceValueWith(SDValue(N, 1), Tmp.second);
558 return Tmp.first;
559 }
560
SoftenFloatRes_FPOW(SDNode * N)561 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
562 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
563 RTLIB::POW_F32,
564 RTLIB::POW_F64,
565 RTLIB::POW_F80,
566 RTLIB::POW_F128,
567 RTLIB::POW_PPCF128));
568 }
569
SoftenFloatRes_FPOWI(SDNode * N)570 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) {
571 bool IsStrict = N->isStrictFPOpcode();
572 unsigned Offset = IsStrict ? 1 : 0;
573 assert(N->getOperand(1 + Offset).getValueType() == MVT::i32 &&
574 "Unsupported power type!");
575 RTLIB::Libcall LC = GetFPLibCall(N->getValueType(0),
576 RTLIB::POWI_F32,
577 RTLIB::POWI_F64,
578 RTLIB::POWI_F80,
579 RTLIB::POWI_F128,
580 RTLIB::POWI_PPCF128);
581 if (!TLI.getLibcallName(LC)) {
582 // Some targets don't have a powi libcall; use pow instead.
583 // FIXME: Implement this if some target needs it.
584 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
585 return DAG.getUNDEF(N->getValueType(0));
586 }
587
588 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
589 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
590 N->getOperand(1 + Offset) };
591 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
592 TargetLowering::MakeLibCallOptions CallOptions;
593 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
594 N->getOperand(1 + Offset).getValueType() };
595 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
596 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
597 CallOptions, SDLoc(N),
598 Chain);
599 if (IsStrict)
600 ReplaceValueWith(SDValue(N, 1), Tmp.second);
601 return Tmp.first;
602 }
603
SoftenFloatRes_FREM(SDNode * N)604 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
605 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
606 RTLIB::REM_F32,
607 RTLIB::REM_F64,
608 RTLIB::REM_F80,
609 RTLIB::REM_F128,
610 RTLIB::REM_PPCF128));
611 }
612
SoftenFloatRes_FRINT(SDNode * N)613 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
614 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
615 RTLIB::RINT_F32,
616 RTLIB::RINT_F64,
617 RTLIB::RINT_F80,
618 RTLIB::RINT_F128,
619 RTLIB::RINT_PPCF128));
620 }
621
SoftenFloatRes_FROUND(SDNode * N)622 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
623 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
624 RTLIB::ROUND_F32,
625 RTLIB::ROUND_F64,
626 RTLIB::ROUND_F80,
627 RTLIB::ROUND_F128,
628 RTLIB::ROUND_PPCF128));
629 }
630
SoftenFloatRes_FROUNDEVEN(SDNode * N)631 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
632 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
633 RTLIB::ROUNDEVEN_F32,
634 RTLIB::ROUNDEVEN_F64,
635 RTLIB::ROUNDEVEN_F80,
636 RTLIB::ROUNDEVEN_F128,
637 RTLIB::ROUNDEVEN_PPCF128));
638 }
639
SoftenFloatRes_FSIN(SDNode * N)640 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
641 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
642 RTLIB::SIN_F32,
643 RTLIB::SIN_F64,
644 RTLIB::SIN_F80,
645 RTLIB::SIN_F128,
646 RTLIB::SIN_PPCF128));
647 }
648
SoftenFloatRes_FSQRT(SDNode * N)649 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
650 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
651 RTLIB::SQRT_F32,
652 RTLIB::SQRT_F64,
653 RTLIB::SQRT_F80,
654 RTLIB::SQRT_F128,
655 RTLIB::SQRT_PPCF128));
656 }
657
SoftenFloatRes_FSUB(SDNode * N)658 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
659 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
660 RTLIB::SUB_F32,
661 RTLIB::SUB_F64,
662 RTLIB::SUB_F80,
663 RTLIB::SUB_F128,
664 RTLIB::SUB_PPCF128));
665 }
666
SoftenFloatRes_FTRUNC(SDNode * N)667 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
668 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
669 RTLIB::TRUNC_F32,
670 RTLIB::TRUNC_F64,
671 RTLIB::TRUNC_F80,
672 RTLIB::TRUNC_F128,
673 RTLIB::TRUNC_PPCF128));
674 }
675
SoftenFloatRes_LOAD(SDNode * N)676 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
677 LoadSDNode *L = cast<LoadSDNode>(N);
678 EVT VT = N->getValueType(0);
679 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
680 SDLoc dl(N);
681
682 auto MMOFlags =
683 L->getMemOperand()->getFlags() &
684 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
685 SDValue NewL;
686 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
687 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
688 L->getChain(), L->getBasePtr(), L->getOffset(),
689 L->getPointerInfo(), NVT, L->getOriginalAlign(),
690 MMOFlags, L->getAAInfo());
691 // Legalized the chain result - switch anything that used the old chain to
692 // use the new one.
693 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
694 return NewL;
695 }
696
697 // Do a non-extending load followed by FP_EXTEND.
698 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
699 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
700 L->getPointerInfo(), L->getMemoryVT(),
701 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
702 // Legalized the chain result - switch anything that used the old chain to
703 // use the new one.
704 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
705 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
706 return BitConvertToInteger(ExtendNode);
707 }
708
SoftenFloatRes_SELECT(SDNode * N)709 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
710 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
711 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
712 return DAG.getSelect(SDLoc(N),
713 LHS.getValueType(), N->getOperand(0), LHS, RHS);
714 }
715
SoftenFloatRes_SELECT_CC(SDNode * N)716 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
717 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
718 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
719 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
720 LHS.getValueType(), N->getOperand(0),
721 N->getOperand(1), LHS, RHS, N->getOperand(4));
722 }
723
SoftenFloatRes_UNDEF(SDNode * N)724 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
725 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
726 N->getValueType(0)));
727 }
728
SoftenFloatRes_VAARG(SDNode * N)729 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
730 SDValue Chain = N->getOperand(0); // Get the chain.
731 SDValue Ptr = N->getOperand(1); // Get the pointer.
732 EVT VT = N->getValueType(0);
733 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
734 SDLoc dl(N);
735
736 SDValue NewVAARG;
737 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
738 N->getConstantOperandVal(3));
739
740 // Legalized the chain result - switch anything that used the old chain to
741 // use the new one.
742 if (N != NewVAARG.getValue(1).getNode())
743 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
744 return NewVAARG;
745 }
746
SoftenFloatRes_XINT_TO_FP(SDNode * N)747 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
748 bool IsStrict = N->isStrictFPOpcode();
749 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
750 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
751 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
752 EVT RVT = N->getValueType(0);
753 EVT NVT = EVT();
754 SDLoc dl(N);
755
756 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
757 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
758 // match. Look for an appropriate libcall.
759 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
760 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
761 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
762 NVT = (MVT::SimpleValueType)t;
763 // The source needs to big enough to hold the operand.
764 if (NVT.bitsGE(SVT))
765 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
766 }
767 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
768
769 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
770 // Sign/zero extend the argument if the libcall takes a larger type.
771 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
772 NVT, N->getOperand(IsStrict ? 1 : 0));
773 TargetLowering::MakeLibCallOptions CallOptions;
774 CallOptions.setSExt(Signed);
775 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
776 std::pair<SDValue, SDValue> Tmp =
777 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
778 Op, CallOptions, dl, Chain);
779
780 if (IsStrict)
781 ReplaceValueWith(SDValue(N, 1), Tmp.second);
782 return Tmp.first;
783 }
784
SoftenFloatRes_VECREDUCE(SDNode * N)785 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
786 // Expand and soften recursively.
787 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
788 return SDValue();
789 }
790
SoftenFloatRes_VECREDUCE_SEQ(SDNode * N)791 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
792 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
793 return SDValue();
794 }
795
796 //===----------------------------------------------------------------------===//
797 // Convert Float Operand to Integer
798 //===----------------------------------------------------------------------===//
799
SoftenFloatOperand(SDNode * N,unsigned OpNo)800 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
801 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
802 dbgs() << "\n");
803 SDValue Res = SDValue();
804
805 switch (N->getOpcode()) {
806 default:
807 #ifndef NDEBUG
808 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
809 N->dump(&DAG); dbgs() << "\n";
810 #endif
811 llvm_unreachable("Do not know how to soften this operator's operand!");
812
813 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
814 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
815 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
816 case ISD::STRICT_FP_ROUND:
817 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
818 case ISD::STRICT_FP_TO_SINT:
819 case ISD::STRICT_FP_TO_UINT:
820 case ISD::FP_TO_SINT:
821 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
822 case ISD::FP_TO_SINT_SAT:
823 case ISD::FP_TO_UINT_SAT:
824 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
825 case ISD::STRICT_LROUND:
826 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
827 case ISD::STRICT_LLROUND:
828 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
829 case ISD::STRICT_LRINT:
830 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
831 case ISD::STRICT_LLRINT:
832 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
833 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
834 case ISD::STRICT_FSETCC:
835 case ISD::STRICT_FSETCCS:
836 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
837 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
838 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
839 }
840
841 // If the result is null, the sub-method took care of registering results etc.
842 if (!Res.getNode()) return false;
843
844 // If the result is N, the sub-method updated N in place. Tell the legalizer
845 // core about this to re-analyze.
846 if (Res.getNode() == N)
847 return true;
848
849 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
850 "Invalid operand softening");
851
852 ReplaceValueWith(SDValue(N, 0), Res);
853 return false;
854 }
855
SoftenFloatOp_BITCAST(SDNode * N)856 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
857 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
858
859 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
860 }
861
SoftenFloatOp_FP_ROUND(SDNode * N)862 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
863 // We actually deal with the partially-softened FP_TO_FP16 node too, which
864 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
865 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
866 N->getOpcode() == ISD::STRICT_FP_ROUND);
867
868 bool IsStrict = N->isStrictFPOpcode();
869 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
870 EVT SVT = Op.getValueType();
871 EVT RVT = N->getValueType(0);
872 EVT FloatRVT = N->getOpcode() == ISD::FP_TO_FP16 ? MVT::f16 : RVT;
873
874 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
875 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
876
877 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
878 Op = GetSoftenedFloat(Op);
879 TargetLowering::MakeLibCallOptions CallOptions;
880 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
881 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
882 CallOptions, SDLoc(N),
883 Chain);
884 if (IsStrict) {
885 ReplaceValueWith(SDValue(N, 1), Tmp.second);
886 ReplaceValueWith(SDValue(N, 0), Tmp.first);
887 return SDValue();
888 }
889 return Tmp.first;
890 }
891
SoftenFloatOp_BR_CC(SDNode * N)892 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
893 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
894 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
895
896 EVT VT = NewLHS.getValueType();
897 NewLHS = GetSoftenedFloat(NewLHS);
898 NewRHS = GetSoftenedFloat(NewRHS);
899 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
900 N->getOperand(2), N->getOperand(3));
901
902 // If softenSetCCOperands returned a scalar, we need to compare the result
903 // against zero to select between true and false values.
904 if (!NewRHS.getNode()) {
905 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
906 CCCode = ISD::SETNE;
907 }
908
909 // Update N to have the operands specified.
910 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
911 DAG.getCondCode(CCCode), NewLHS, NewRHS,
912 N->getOperand(4)),
913 0);
914 }
915
916 // Even if the result type is legal, no libcall may exactly match. (e.g. We
917 // don't have FP-i8 conversions) This helper method looks for an appropriate
918 // promoted libcall.
findFPToIntLibcall(EVT SrcVT,EVT RetVT,EVT & Promoted,bool Signed)919 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
920 bool Signed) {
921 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
922 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
923 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
924 ++IntVT) {
925 Promoted = (MVT::SimpleValueType)IntVT;
926 // The type needs to big enough to hold the result.
927 if (Promoted.bitsGE(RetVT))
928 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
929 : RTLIB::getFPTOUINT(SrcVT, Promoted);
930 }
931 return LC;
932 }
933
SoftenFloatOp_FP_TO_XINT(SDNode * N)934 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
935 bool IsStrict = N->isStrictFPOpcode();
936 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
937 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
938
939 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
940 EVT SVT = Op.getValueType();
941 EVT RVT = N->getValueType(0);
942 EVT NVT = EVT();
943 SDLoc dl(N);
944
945 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
946 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
947 // match, eg. we don't have fp -> i8 conversions.
948 // Look for an appropriate libcall.
949 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
950 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
951 "Unsupported FP_TO_XINT!");
952
953 Op = GetSoftenedFloat(Op);
954 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
955 TargetLowering::MakeLibCallOptions CallOptions;
956 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
957 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
958 CallOptions, dl, Chain);
959
960 // Truncate the result if the libcall returns a larger type.
961 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
962
963 if (!IsStrict)
964 return Res;
965
966 ReplaceValueWith(SDValue(N, 1), Tmp.second);
967 ReplaceValueWith(SDValue(N, 0), Res);
968 return SDValue();
969 }
970
SoftenFloatOp_FP_TO_XINT_SAT(SDNode * N)971 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
972 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
973 return Res;
974 }
975
SoftenFloatOp_SELECT_CC(SDNode * N)976 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
977 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
978 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
979
980 EVT VT = NewLHS.getValueType();
981 NewLHS = GetSoftenedFloat(NewLHS);
982 NewRHS = GetSoftenedFloat(NewRHS);
983 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
984 N->getOperand(0), N->getOperand(1));
985
986 // If softenSetCCOperands returned a scalar, we need to compare the result
987 // against zero to select between true and false values.
988 if (!NewRHS.getNode()) {
989 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
990 CCCode = ISD::SETNE;
991 }
992
993 // Update N to have the operands specified.
994 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
995 N->getOperand(2), N->getOperand(3),
996 DAG.getCondCode(CCCode)),
997 0);
998 }
999
SoftenFloatOp_SETCC(SDNode * N)1000 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1001 bool IsStrict = N->isStrictFPOpcode();
1002 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1003 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1004 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1005 ISD::CondCode CCCode =
1006 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1007
1008 EVT VT = Op0.getValueType();
1009 SDValue NewLHS = GetSoftenedFloat(Op0);
1010 SDValue NewRHS = GetSoftenedFloat(Op1);
1011 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1012 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1013
1014 // Update N to have the operands specified.
1015 if (NewRHS.getNode()) {
1016 if (IsStrict)
1017 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1018 NewRHS, DAG.getCondCode(CCCode));
1019 else
1020 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1021 DAG.getCondCode(CCCode)), 0);
1022 }
1023
1024 // Otherwise, softenSetCCOperands returned a scalar, use it.
1025 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1026 "Unexpected setcc expansion!");
1027
1028 if (IsStrict) {
1029 ReplaceValueWith(SDValue(N, 0), NewLHS);
1030 ReplaceValueWith(SDValue(N, 1), Chain);
1031 return SDValue();
1032 }
1033 return NewLHS;
1034 }
1035
SoftenFloatOp_STORE(SDNode * N,unsigned OpNo)1036 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1037 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1038 assert(OpNo == 1 && "Can only soften the stored value!");
1039 StoreSDNode *ST = cast<StoreSDNode>(N);
1040 SDValue Val = ST->getValue();
1041 SDLoc dl(N);
1042
1043 if (ST->isTruncatingStore())
1044 // Do an FP_ROUND followed by a non-truncating store.
1045 Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(),
1046 Val, DAG.getIntPtrConstant(0, dl)));
1047 else
1048 Val = GetSoftenedFloat(Val);
1049
1050 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1051 ST->getMemOperand());
1052 }
1053
SoftenFloatOp_FCOPYSIGN(SDNode * N)1054 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1055 SDValue LHS = N->getOperand(0);
1056 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1057 SDLoc dl(N);
1058
1059 EVT LVT = LHS.getValueType();
1060 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1061 EVT RVT = RHS.getValueType();
1062
1063 unsigned LSize = LVT.getSizeInBits();
1064 unsigned RSize = RVT.getSizeInBits();
1065
1066 // Shift right or sign-extend it if the two operands have different types.
1067 int SizeDiff = RSize - LSize;
1068 if (SizeDiff > 0) {
1069 RHS =
1070 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1071 DAG.getConstant(SizeDiff, dl,
1072 TLI.getShiftAmountTy(RHS.getValueType(),
1073 DAG.getDataLayout())));
1074 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1075 } else if (SizeDiff < 0) {
1076 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1077 RHS =
1078 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1079 DAG.getConstant(-SizeDiff, dl,
1080 TLI.getShiftAmountTy(RHS.getValueType(),
1081 DAG.getDataLayout())));
1082 }
1083
1084 RHS = DAG.getBitcast(LVT, RHS);
1085 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1086 }
1087
SoftenFloatOp_Unary(SDNode * N,RTLIB::Libcall LC)1088 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1089 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1090 bool IsStrict = N->isStrictFPOpcode();
1091 unsigned Offset = IsStrict ? 1 : 0;
1092 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1093 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1094 TargetLowering::MakeLibCallOptions CallOptions;
1095 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1096 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1097 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1098 CallOptions, SDLoc(N),
1099 Chain);
1100 if (IsStrict) {
1101 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1102 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1103 return SDValue();
1104 }
1105
1106 return Tmp.first;
1107 }
1108
SoftenFloatOp_LROUND(SDNode * N)1109 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1110 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1111 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1112 RTLIB::LROUND_F32,
1113 RTLIB::LROUND_F64,
1114 RTLIB::LROUND_F80,
1115 RTLIB::LROUND_F128,
1116 RTLIB::LROUND_PPCF128));
1117 }
1118
SoftenFloatOp_LLROUND(SDNode * N)1119 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1120 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1121 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1122 RTLIB::LLROUND_F32,
1123 RTLIB::LLROUND_F64,
1124 RTLIB::LLROUND_F80,
1125 RTLIB::LLROUND_F128,
1126 RTLIB::LLROUND_PPCF128));
1127 }
1128
SoftenFloatOp_LRINT(SDNode * N)1129 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1130 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1131 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1132 RTLIB::LRINT_F32,
1133 RTLIB::LRINT_F64,
1134 RTLIB::LRINT_F80,
1135 RTLIB::LRINT_F128,
1136 RTLIB::LRINT_PPCF128));
1137 }
1138
SoftenFloatOp_LLRINT(SDNode * N)1139 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1140 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1141 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1142 RTLIB::LLRINT_F32,
1143 RTLIB::LLRINT_F64,
1144 RTLIB::LLRINT_F80,
1145 RTLIB::LLRINT_F128,
1146 RTLIB::LLRINT_PPCF128));
1147 }
1148
1149 //===----------------------------------------------------------------------===//
1150 // Float Result Expansion
1151 //===----------------------------------------------------------------------===//
1152
1153 /// ExpandFloatResult - This method is called when the specified result of the
1154 /// specified node is found to need expansion. At this point, the node may also
1155 /// have invalid operands or may have other results that need promotion, we just
1156 /// know that (at least) one result needs expansion.
ExpandFloatResult(SDNode * N,unsigned ResNo)1157 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1158 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1159 SDValue Lo, Hi;
1160 Lo = Hi = SDValue();
1161
1162 // See if the target wants to custom expand this node.
1163 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1164 return;
1165
1166 switch (N->getOpcode()) {
1167 default:
1168 #ifndef NDEBUG
1169 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1170 N->dump(&DAG); dbgs() << "\n";
1171 #endif
1172 llvm_unreachable("Do not know how to expand the result of this operator!");
1173
1174 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1175 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1176 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1177
1178 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1179 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1180 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1181 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1182 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1183 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1184
1185 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1186 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1187 case ISD::STRICT_FMINNUM:
1188 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1189 case ISD::STRICT_FMAXNUM:
1190 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1191 case ISD::STRICT_FADD:
1192 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1193 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1194 case ISD::STRICT_FCEIL:
1195 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1196 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1197 case ISD::STRICT_FCOS:
1198 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1199 case ISD::STRICT_FDIV:
1200 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1201 case ISD::STRICT_FEXP:
1202 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1203 case ISD::STRICT_FEXP2:
1204 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1205 case ISD::STRICT_FFLOOR:
1206 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1207 case ISD::STRICT_FLOG:
1208 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1209 case ISD::STRICT_FLOG2:
1210 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1211 case ISD::STRICT_FLOG10:
1212 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1213 case ISD::STRICT_FMA:
1214 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1215 case ISD::STRICT_FMUL:
1216 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1217 case ISD::STRICT_FNEARBYINT:
1218 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1219 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1220 case ISD::STRICT_FP_EXTEND:
1221 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1222 case ISD::STRICT_FPOW:
1223 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1224 case ISD::STRICT_FPOWI:
1225 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1226 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1227 case ISD::STRICT_FRINT:
1228 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1229 case ISD::STRICT_FROUND:
1230 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1231 case ISD::STRICT_FROUNDEVEN:
1232 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1233 case ISD::STRICT_FSIN:
1234 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1235 case ISD::STRICT_FSQRT:
1236 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1237 case ISD::STRICT_FSUB:
1238 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1239 case ISD::STRICT_FTRUNC:
1240 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1241 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1242 case ISD::STRICT_SINT_TO_FP:
1243 case ISD::STRICT_UINT_TO_FP:
1244 case ISD::SINT_TO_FP:
1245 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1246 case ISD::STRICT_FREM:
1247 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1248 }
1249
1250 // If Lo/Hi is null, the sub-method took care of registering results etc.
1251 if (Lo.getNode())
1252 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1253 }
1254
ExpandFloatRes_ConstantFP(SDNode * N,SDValue & Lo,SDValue & Hi)1255 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1256 SDValue &Hi) {
1257 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1258 assert(NVT.getSizeInBits() == 64 &&
1259 "Do not know how to expand this float constant!");
1260 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1261 SDLoc dl(N);
1262 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1263 APInt(64, C.getRawData()[1])),
1264 dl, NVT);
1265 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1266 APInt(64, C.getRawData()[0])),
1267 dl, NVT);
1268 }
1269
ExpandFloatRes_Unary(SDNode * N,RTLIB::Libcall LC,SDValue & Lo,SDValue & Hi)1270 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1271 SDValue &Lo, SDValue &Hi) {
1272 bool IsStrict = N->isStrictFPOpcode();
1273 unsigned Offset = IsStrict ? 1 : 0;
1274 SDValue Op = N->getOperand(0 + Offset);
1275 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1276 TargetLowering::MakeLibCallOptions CallOptions;
1277 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1278 Op, CallOptions, SDLoc(N),
1279 Chain);
1280 if (IsStrict)
1281 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1282 GetPairElements(Tmp.first, Lo, Hi);
1283 }
1284
ExpandFloatRes_Binary(SDNode * N,RTLIB::Libcall LC,SDValue & Lo,SDValue & Hi)1285 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1286 SDValue &Lo, SDValue &Hi) {
1287 bool IsStrict = N->isStrictFPOpcode();
1288 unsigned Offset = IsStrict ? 1 : 0;
1289 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1290 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1291 TargetLowering::MakeLibCallOptions CallOptions;
1292 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1293 Ops, CallOptions, SDLoc(N),
1294 Chain);
1295 if (IsStrict)
1296 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1297 GetPairElements(Tmp.first, Lo, Hi);
1298 }
1299
ExpandFloatRes_FABS(SDNode * N,SDValue & Lo,SDValue & Hi)1300 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1301 SDValue &Hi) {
1302 assert(N->getValueType(0) == MVT::ppcf128 &&
1303 "Logic only correct for ppcf128!");
1304 SDLoc dl(N);
1305 SDValue Tmp;
1306 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1307 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1308 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1309 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1310 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1311 ISD::SETEQ);
1312 }
1313
ExpandFloatRes_FMINNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1314 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1315 SDValue &Hi) {
1316 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1317 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1318 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1319 RTLIB::FMIN_PPCF128), Lo, Hi);
1320 }
1321
ExpandFloatRes_FMAXNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1322 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1323 SDValue &Hi) {
1324 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1325 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1326 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1327 RTLIB::FMAX_PPCF128), Lo, Hi);
1328 }
1329
ExpandFloatRes_FADD(SDNode * N,SDValue & Lo,SDValue & Hi)1330 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1331 SDValue &Hi) {
1332 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1333 RTLIB::ADD_F32, RTLIB::ADD_F64,
1334 RTLIB::ADD_F80, RTLIB::ADD_F128,
1335 RTLIB::ADD_PPCF128), Lo, Hi);
1336 }
1337
ExpandFloatRes_FCBRT(SDNode * N,SDValue & Lo,SDValue & Hi)1338 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1339 SDValue &Hi) {
1340 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1341 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1342 RTLIB::CBRT_F128,
1343 RTLIB::CBRT_PPCF128), Lo, Hi);
1344 }
1345
ExpandFloatRes_FCEIL(SDNode * N,SDValue & Lo,SDValue & Hi)1346 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1347 SDValue &Lo, SDValue &Hi) {
1348 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1349 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1350 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1351 RTLIB::CEIL_PPCF128), Lo, Hi);
1352 }
1353
ExpandFloatRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1354 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1355 SDValue &Lo, SDValue &Hi) {
1356 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1357 RTLIB::COPYSIGN_F32,
1358 RTLIB::COPYSIGN_F64,
1359 RTLIB::COPYSIGN_F80,
1360 RTLIB::COPYSIGN_F128,
1361 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1362 }
1363
ExpandFloatRes_FCOS(SDNode * N,SDValue & Lo,SDValue & Hi)1364 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1365 SDValue &Lo, SDValue &Hi) {
1366 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1367 RTLIB::COS_F32, RTLIB::COS_F64,
1368 RTLIB::COS_F80, RTLIB::COS_F128,
1369 RTLIB::COS_PPCF128), Lo, Hi);
1370 }
1371
ExpandFloatRes_FDIV(SDNode * N,SDValue & Lo,SDValue & Hi)1372 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1373 SDValue &Hi) {
1374 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1375 RTLIB::DIV_F32,
1376 RTLIB::DIV_F64,
1377 RTLIB::DIV_F80,
1378 RTLIB::DIV_F128,
1379 RTLIB::DIV_PPCF128), Lo, Hi);
1380 }
1381
ExpandFloatRes_FEXP(SDNode * N,SDValue & Lo,SDValue & Hi)1382 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1383 SDValue &Lo, SDValue &Hi) {
1384 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1385 RTLIB::EXP_F32, RTLIB::EXP_F64,
1386 RTLIB::EXP_F80, RTLIB::EXP_F128,
1387 RTLIB::EXP_PPCF128), Lo, Hi);
1388 }
1389
ExpandFloatRes_FEXP2(SDNode * N,SDValue & Lo,SDValue & Hi)1390 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1391 SDValue &Lo, SDValue &Hi) {
1392 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1393 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1394 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1395 RTLIB::EXP2_PPCF128), Lo, Hi);
1396 }
1397
ExpandFloatRes_FFLOOR(SDNode * N,SDValue & Lo,SDValue & Hi)1398 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1399 SDValue &Lo, SDValue &Hi) {
1400 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1401 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1402 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1403 RTLIB::FLOOR_PPCF128), Lo, Hi);
1404 }
1405
ExpandFloatRes_FLOG(SDNode * N,SDValue & Lo,SDValue & Hi)1406 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1407 SDValue &Lo, SDValue &Hi) {
1408 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1409 RTLIB::LOG_F32, RTLIB::LOG_F64,
1410 RTLIB::LOG_F80, RTLIB::LOG_F128,
1411 RTLIB::LOG_PPCF128), Lo, Hi);
1412 }
1413
ExpandFloatRes_FLOG2(SDNode * N,SDValue & Lo,SDValue & Hi)1414 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1415 SDValue &Lo, SDValue &Hi) {
1416 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1417 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1418 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1419 RTLIB::LOG2_PPCF128), Lo, Hi);
1420 }
1421
ExpandFloatRes_FLOG10(SDNode * N,SDValue & Lo,SDValue & Hi)1422 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1423 SDValue &Lo, SDValue &Hi) {
1424 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1425 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1426 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1427 RTLIB::LOG10_PPCF128), Lo, Hi);
1428 }
1429
ExpandFloatRes_FMA(SDNode * N,SDValue & Lo,SDValue & Hi)1430 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1431 SDValue &Hi) {
1432 bool IsStrict = N->isStrictFPOpcode();
1433 unsigned Offset = IsStrict ? 1 : 0;
1434 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1435 N->getOperand(2 + Offset) };
1436 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1437 TargetLowering::MakeLibCallOptions CallOptions;
1438 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1439 RTLIB::FMA_F32,
1440 RTLIB::FMA_F64,
1441 RTLIB::FMA_F80,
1442 RTLIB::FMA_F128,
1443 RTLIB::FMA_PPCF128),
1444 N->getValueType(0), Ops, CallOptions,
1445 SDLoc(N), Chain);
1446 if (IsStrict)
1447 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1448 GetPairElements(Tmp.first, Lo, Hi);
1449 }
1450
ExpandFloatRes_FMUL(SDNode * N,SDValue & Lo,SDValue & Hi)1451 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1452 SDValue &Hi) {
1453 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1454 RTLIB::MUL_F32,
1455 RTLIB::MUL_F64,
1456 RTLIB::MUL_F80,
1457 RTLIB::MUL_F128,
1458 RTLIB::MUL_PPCF128), Lo, Hi);
1459 }
1460
ExpandFloatRes_FNEARBYINT(SDNode * N,SDValue & Lo,SDValue & Hi)1461 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1462 SDValue &Lo, SDValue &Hi) {
1463 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1464 RTLIB::NEARBYINT_F32,
1465 RTLIB::NEARBYINT_F64,
1466 RTLIB::NEARBYINT_F80,
1467 RTLIB::NEARBYINT_F128,
1468 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1469 }
1470
ExpandFloatRes_FNEG(SDNode * N,SDValue & Lo,SDValue & Hi)1471 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1472 SDValue &Hi) {
1473 SDLoc dl(N);
1474 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1475 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1476 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1477 }
1478
ExpandFloatRes_FP_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)1479 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1480 SDValue &Hi) {
1481 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1482 SDLoc dl(N);
1483 bool IsStrict = N->isStrictFPOpcode();
1484
1485 SDValue Chain;
1486 if (IsStrict) {
1487 // If the expanded type is the same as the input type, just bypass the node.
1488 if (NVT == N->getOperand(1).getValueType()) {
1489 Hi = N->getOperand(1);
1490 Chain = N->getOperand(0);
1491 } else {
1492 // Other we need to extend.
1493 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1494 { N->getOperand(0), N->getOperand(1) });
1495 Chain = Hi.getValue(1);
1496 }
1497 } else {
1498 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1499 }
1500
1501 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1502 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1503
1504 if (IsStrict)
1505 ReplaceValueWith(SDValue(N, 1), Chain);
1506 }
1507
ExpandFloatRes_FPOW(SDNode * N,SDValue & Lo,SDValue & Hi)1508 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1509 SDValue &Lo, SDValue &Hi) {
1510 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1511 RTLIB::POW_F32, RTLIB::POW_F64,
1512 RTLIB::POW_F80, RTLIB::POW_F128,
1513 RTLIB::POW_PPCF128), Lo, Hi);
1514 }
1515
ExpandFloatRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)1516 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1517 SDValue &Lo, SDValue &Hi) {
1518 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1519 RTLIB::POWI_F32, RTLIB::POWI_F64,
1520 RTLIB::POWI_F80, RTLIB::POWI_F128,
1521 RTLIB::POWI_PPCF128), Lo, Hi);
1522 }
1523
ExpandFloatRes_FREEZE(SDNode * N,SDValue & Lo,SDValue & Hi)1524 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1525 SDValue &Lo, SDValue &Hi) {
1526 assert(N->getValueType(0) == MVT::ppcf128 &&
1527 "Logic only correct for ppcf128!");
1528
1529 SDLoc dl(N);
1530 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1531 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1532 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1533 }
1534
ExpandFloatRes_FREM(SDNode * N,SDValue & Lo,SDValue & Hi)1535 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1536 SDValue &Lo, SDValue &Hi) {
1537 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1538 RTLIB::REM_F32, RTLIB::REM_F64,
1539 RTLIB::REM_F80, RTLIB::REM_F128,
1540 RTLIB::REM_PPCF128), Lo, Hi);
1541 }
1542
ExpandFloatRes_FRINT(SDNode * N,SDValue & Lo,SDValue & Hi)1543 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1544 SDValue &Lo, SDValue &Hi) {
1545 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1546 RTLIB::RINT_F32, RTLIB::RINT_F64,
1547 RTLIB::RINT_F80, RTLIB::RINT_F128,
1548 RTLIB::RINT_PPCF128), Lo, Hi);
1549 }
1550
ExpandFloatRes_FROUND(SDNode * N,SDValue & Lo,SDValue & Hi)1551 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1552 SDValue &Lo, SDValue &Hi) {
1553 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1554 RTLIB::ROUND_F32,
1555 RTLIB::ROUND_F64,
1556 RTLIB::ROUND_F80,
1557 RTLIB::ROUND_F128,
1558 RTLIB::ROUND_PPCF128), Lo, Hi);
1559 }
1560
ExpandFloatRes_FROUNDEVEN(SDNode * N,SDValue & Lo,SDValue & Hi)1561 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1562 SDValue &Lo, SDValue &Hi) {
1563 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1564 RTLIB::ROUNDEVEN_F32,
1565 RTLIB::ROUNDEVEN_F64,
1566 RTLIB::ROUNDEVEN_F80,
1567 RTLIB::ROUNDEVEN_F128,
1568 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1569 }
1570
ExpandFloatRes_FSIN(SDNode * N,SDValue & Lo,SDValue & Hi)1571 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1572 SDValue &Lo, SDValue &Hi) {
1573 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1574 RTLIB::SIN_F32, RTLIB::SIN_F64,
1575 RTLIB::SIN_F80, RTLIB::SIN_F128,
1576 RTLIB::SIN_PPCF128), Lo, Hi);
1577 }
1578
ExpandFloatRes_FSQRT(SDNode * N,SDValue & Lo,SDValue & Hi)1579 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1580 SDValue &Lo, SDValue &Hi) {
1581 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1582 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1583 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1584 RTLIB::SQRT_PPCF128), Lo, Hi);
1585 }
1586
ExpandFloatRes_FSUB(SDNode * N,SDValue & Lo,SDValue & Hi)1587 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1588 SDValue &Hi) {
1589 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1590 RTLIB::SUB_F32,
1591 RTLIB::SUB_F64,
1592 RTLIB::SUB_F80,
1593 RTLIB::SUB_F128,
1594 RTLIB::SUB_PPCF128), Lo, Hi);
1595 }
1596
ExpandFloatRes_FTRUNC(SDNode * N,SDValue & Lo,SDValue & Hi)1597 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1598 SDValue &Lo, SDValue &Hi) {
1599 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1600 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1601 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1602 RTLIB::TRUNC_PPCF128), Lo, Hi);
1603 }
1604
ExpandFloatRes_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)1605 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1606 SDValue &Hi) {
1607 if (ISD::isNormalLoad(N)) {
1608 ExpandRes_NormalLoad(N, Lo, Hi);
1609 return;
1610 }
1611
1612 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1613 LoadSDNode *LD = cast<LoadSDNode>(N);
1614 SDValue Chain = LD->getChain();
1615 SDValue Ptr = LD->getBasePtr();
1616 SDLoc dl(N);
1617
1618 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1619 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1620 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1621
1622 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1623 LD->getMemoryVT(), LD->getMemOperand());
1624
1625 // Remember the chain.
1626 Chain = Hi.getValue(1);
1627
1628 // The low part is zero.
1629 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1630 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1631
1632 // Modified the chain - switch anything that used the old chain to use the
1633 // new one.
1634 ReplaceValueWith(SDValue(LD, 1), Chain);
1635 }
1636
ExpandFloatRes_XINT_TO_FP(SDNode * N,SDValue & Lo,SDValue & Hi)1637 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1638 SDValue &Hi) {
1639 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1640 EVT VT = N->getValueType(0);
1641 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1642 bool Strict = N->isStrictFPOpcode();
1643 SDValue Src = N->getOperand(Strict ? 1 : 0);
1644 EVT SrcVT = Src.getValueType();
1645 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1646 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1647 SDLoc dl(N);
1648 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1649
1650 // TODO: Any other flags to propagate?
1651 SDNodeFlags Flags;
1652 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1653
1654 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1655 // When promoting partial word types to i32 we must honor the signedness,
1656 // though.
1657 if (SrcVT.bitsLE(MVT::i32)) {
1658 // The integer can be represented exactly in an f64.
1659 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1660 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1661 if (Strict) {
1662 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1663 {Chain, Src}, Flags);
1664 Chain = Hi.getValue(1);
1665 } else
1666 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1667 } else {
1668 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1669 if (SrcVT.bitsLE(MVT::i64)) {
1670 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1671 MVT::i64, Src);
1672 LC = RTLIB::SINTTOFP_I64_PPCF128;
1673 } else if (SrcVT.bitsLE(MVT::i128)) {
1674 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1675 LC = RTLIB::SINTTOFP_I128_PPCF128;
1676 }
1677 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1678
1679 TargetLowering::MakeLibCallOptions CallOptions;
1680 CallOptions.setSExt(true);
1681 std::pair<SDValue, SDValue> Tmp =
1682 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1683 if (Strict)
1684 Chain = Tmp.second;
1685 GetPairElements(Tmp.first, Lo, Hi);
1686 }
1687
1688 // No need to complement for unsigned 32-bit integers
1689 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1690 if (Strict)
1691 ReplaceValueWith(SDValue(N, 1), Chain);
1692
1693 return;
1694 }
1695
1696 // Unsigned - fix up the SINT_TO_FP value just calculated.
1697 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1698 // keep semantics correctness if the integer is not exactly representable
1699 // here. See ExpandLegalINT_TO_FP.
1700 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1701 SrcVT = Src.getValueType();
1702
1703 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1704 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1705 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1706 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1707 ArrayRef<uint64_t> Parts;
1708
1709 switch (SrcVT.getSimpleVT().SimpleTy) {
1710 default:
1711 llvm_unreachable("Unsupported UINT_TO_FP!");
1712 case MVT::i32:
1713 Parts = TwoE32;
1714 break;
1715 case MVT::i64:
1716 Parts = TwoE64;
1717 break;
1718 case MVT::i128:
1719 Parts = TwoE128;
1720 break;
1721 }
1722
1723 // TODO: Are there other fast-math-flags to propagate to this FADD?
1724 SDValue NewLo = DAG.getConstantFP(
1725 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1726 if (Strict) {
1727 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1728 {Chain, Hi, NewLo}, Flags);
1729 Chain = Lo.getValue(1);
1730 ReplaceValueWith(SDValue(N, 1), Chain);
1731 } else
1732 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1733 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1734 Lo, Hi, ISD::SETLT);
1735 GetPairElements(Lo, Lo, Hi);
1736 }
1737
1738
1739 //===----------------------------------------------------------------------===//
1740 // Float Operand Expansion
1741 //===----------------------------------------------------------------------===//
1742
1743 /// ExpandFloatOperand - This method is called when the specified operand of the
1744 /// specified node is found to need expansion. At this point, all of the result
1745 /// types of the node are known to be legal, but other operands of the node may
1746 /// need promotion or expansion as well as the specified one.
ExpandFloatOperand(SDNode * N,unsigned OpNo)1747 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1748 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1749 SDValue Res = SDValue();
1750
1751 // See if the target wants to custom expand this node.
1752 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1753 return false;
1754
1755 switch (N->getOpcode()) {
1756 default:
1757 #ifndef NDEBUG
1758 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1759 N->dump(&DAG); dbgs() << "\n";
1760 #endif
1761 llvm_unreachable("Do not know how to expand this operator's operand!");
1762
1763 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1764 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1765 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1766
1767 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1768 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1769 case ISD::STRICT_FP_ROUND:
1770 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1771 case ISD::STRICT_FP_TO_SINT:
1772 case ISD::STRICT_FP_TO_UINT:
1773 case ISD::FP_TO_SINT:
1774 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1775 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1776 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1777 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1778 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1779 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1780 case ISD::STRICT_FSETCC:
1781 case ISD::STRICT_FSETCCS:
1782 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1783 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1784 OpNo); break;
1785 }
1786
1787 // If the result is null, the sub-method took care of registering results etc.
1788 if (!Res.getNode()) return false;
1789
1790 // If the result is N, the sub-method updated N in place. Tell the legalizer
1791 // core about this.
1792 if (Res.getNode() == N)
1793 return true;
1794
1795 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1796 "Invalid operand expansion");
1797
1798 ReplaceValueWith(SDValue(N, 0), Res);
1799 return false;
1800 }
1801
1802 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1803 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
FloatExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl,SDValue & Chain,bool IsSignaling)1804 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1805 SDValue &NewRHS,
1806 ISD::CondCode &CCCode,
1807 const SDLoc &dl, SDValue &Chain,
1808 bool IsSignaling) {
1809 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1810 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1811 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1812
1813 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1814
1815 // FIXME: This generated code sucks. We want to generate
1816 // FCMPU crN, hi1, hi2
1817 // BNE crN, L:
1818 // FCMPU crN, lo1, lo2
1819 // The following can be improved, but not that much.
1820 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1821 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1822 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1823 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1824 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1825 RHSLo, CCCode, OutputChain, IsSignaling);
1826 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1827 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1828 Tmp1 =
1829 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1830 ISD::SETUNE, OutputChain, IsSignaling);
1831 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1832 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1833 RHSHi, CCCode, OutputChain, IsSignaling);
1834 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1835 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1836 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1837 NewRHS = SDValue(); // LHS is the result, not a compare.
1838 Chain = OutputChain;
1839 }
1840
ExpandFloatOp_BR_CC(SDNode * N)1841 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1842 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1843 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1844 SDValue Chain;
1845 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1846
1847 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1848 // against zero to select between true and false values.
1849 if (!NewRHS.getNode()) {
1850 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1851 CCCode = ISD::SETNE;
1852 }
1853
1854 // Update N to have the operands specified.
1855 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1856 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1857 N->getOperand(4)), 0);
1858 }
1859
ExpandFloatOp_FCOPYSIGN(SDNode * N)1860 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1861 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1862 "Logic only correct for ppcf128!");
1863 SDValue Lo, Hi;
1864 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1865 // The ppcf128 value is providing only the sign; take it from the
1866 // higher-order double (which must have the larger magnitude).
1867 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1868 N->getValueType(0), N->getOperand(0), Hi);
1869 }
1870
ExpandFloatOp_FP_ROUND(SDNode * N)1871 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1872 bool IsStrict = N->isStrictFPOpcode();
1873 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1874 "Logic only correct for ppcf128!");
1875 SDValue Lo, Hi;
1876 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
1877
1878 if (!IsStrict)
1879 // Round it the rest of the way (e.g. to f32) if needed.
1880 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1881 N->getValueType(0), Hi, N->getOperand(1));
1882
1883 // Eliminate the node if the input float type is the same as the output float
1884 // type.
1885 if (Hi.getValueType() == N->getValueType(0)) {
1886 // Connect the output chain to the input chain, unlinking the node.
1887 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
1888 ReplaceValueWith(SDValue(N, 0), Hi);
1889 return SDValue();
1890 }
1891
1892 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
1893 {N->getValueType(0), MVT::Other},
1894 {N->getOperand(0), Hi, N->getOperand(2)});
1895 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
1896 ReplaceValueWith(SDValue(N, 0), Expansion);
1897 return SDValue();
1898 }
1899
ExpandFloatOp_FP_TO_XINT(SDNode * N)1900 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
1901 EVT RVT = N->getValueType(0);
1902 SDLoc dl(N);
1903
1904 bool IsStrict = N->isStrictFPOpcode();
1905 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1906 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1907 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1908 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1909
1910 EVT NVT;
1911 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
1912 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1913 "Unsupported FP_TO_XINT!");
1914 TargetLowering::MakeLibCallOptions CallOptions;
1915 std::pair<SDValue, SDValue> Tmp =
1916 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
1917 if (!IsStrict)
1918 return Tmp.first;
1919
1920 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1921 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1922 return SDValue();
1923 }
1924
ExpandFloatOp_SELECT_CC(SDNode * N)1925 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
1926 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1927 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1928 SDValue Chain;
1929 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1930
1931 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1932 // against zero to select between true and false values.
1933 if (!NewRHS.getNode()) {
1934 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1935 CCCode = ISD::SETNE;
1936 }
1937
1938 // Update N to have the operands specified.
1939 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1940 N->getOperand(2), N->getOperand(3),
1941 DAG.getCondCode(CCCode)), 0);
1942 }
1943
ExpandFloatOp_SETCC(SDNode * N)1944 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
1945 bool IsStrict = N->isStrictFPOpcode();
1946 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
1947 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
1948 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1949 ISD::CondCode CCCode =
1950 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1951 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
1952 N->getOpcode() == ISD::STRICT_FSETCCS);
1953
1954 // FloatExpandSetCCOperands always returned a scalar.
1955 assert(!NewRHS.getNode() && "Expect to return scalar");
1956 assert(NewLHS.getValueType() == N->getValueType(0) &&
1957 "Unexpected setcc expansion!");
1958 if (Chain) {
1959 ReplaceValueWith(SDValue(N, 0), NewLHS);
1960 ReplaceValueWith(SDValue(N, 1), Chain);
1961 return SDValue();
1962 }
1963 return NewLHS;
1964 }
1965
ExpandFloatOp_STORE(SDNode * N,unsigned OpNo)1966 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
1967 if (ISD::isNormalStore(N))
1968 return ExpandOp_NormalStore(N, OpNo);
1969
1970 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1971 assert(OpNo == 1 && "Can only expand the stored value so far");
1972 StoreSDNode *ST = cast<StoreSDNode>(N);
1973
1974 SDValue Chain = ST->getChain();
1975 SDValue Ptr = ST->getBasePtr();
1976
1977 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
1978 ST->getValue().getValueType());
1979 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1980 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1981 (void)NVT;
1982
1983 SDValue Lo, Hi;
1984 GetExpandedOp(ST->getValue(), Lo, Hi);
1985
1986 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
1987 ST->getMemoryVT(), ST->getMemOperand());
1988 }
1989
ExpandFloatOp_LROUND(SDNode * N)1990 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
1991 EVT RVT = N->getValueType(0);
1992 EVT RetVT = N->getOperand(0).getValueType();
1993 TargetLowering::MakeLibCallOptions CallOptions;
1994 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1995 RTLIB::LROUND_F32,
1996 RTLIB::LROUND_F64,
1997 RTLIB::LROUND_F80,
1998 RTLIB::LROUND_F128,
1999 RTLIB::LROUND_PPCF128),
2000 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2001 }
2002
ExpandFloatOp_LLROUND(SDNode * N)2003 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2004 EVT RVT = N->getValueType(0);
2005 EVT RetVT = N->getOperand(0).getValueType();
2006 TargetLowering::MakeLibCallOptions CallOptions;
2007 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2008 RTLIB::LLROUND_F32,
2009 RTLIB::LLROUND_F64,
2010 RTLIB::LLROUND_F80,
2011 RTLIB::LLROUND_F128,
2012 RTLIB::LLROUND_PPCF128),
2013 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2014 }
2015
ExpandFloatOp_LRINT(SDNode * N)2016 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2017 EVT RVT = N->getValueType(0);
2018 EVT RetVT = N->getOperand(0).getValueType();
2019 TargetLowering::MakeLibCallOptions CallOptions;
2020 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2021 RTLIB::LRINT_F32,
2022 RTLIB::LRINT_F64,
2023 RTLIB::LRINT_F80,
2024 RTLIB::LRINT_F128,
2025 RTLIB::LRINT_PPCF128),
2026 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2027 }
2028
ExpandFloatOp_LLRINT(SDNode * N)2029 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2030 EVT RVT = N->getValueType(0);
2031 EVT RetVT = N->getOperand(0).getValueType();
2032 TargetLowering::MakeLibCallOptions CallOptions;
2033 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2034 RTLIB::LLRINT_F32,
2035 RTLIB::LLRINT_F64,
2036 RTLIB::LLRINT_F80,
2037 RTLIB::LLRINT_F128,
2038 RTLIB::LLRINT_PPCF128),
2039 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2040 }
2041
2042 //===----------------------------------------------------------------------===//
2043 // Float Operand Promotion
2044 //===----------------------------------------------------------------------===//
2045 //
2046
GetPromotionOpcode(EVT OpVT,EVT RetVT)2047 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2048 if (OpVT == MVT::f16) {
2049 return ISD::FP16_TO_FP;
2050 } else if (RetVT == MVT::f16) {
2051 return ISD::FP_TO_FP16;
2052 }
2053
2054 report_fatal_error("Attempt at an invalid promotion-related conversion");
2055 }
2056
PromoteFloatOperand(SDNode * N,unsigned OpNo)2057 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2058 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
2059 dbgs() << "\n");
2060 SDValue R = SDValue();
2061
2062 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2063 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2064 return false;
2065 }
2066
2067 // Nodes that use a promotion-requiring floating point operand, but doesn't
2068 // produce a promotion-requiring floating point result, need to be legalized
2069 // to use the promoted float operand. Nodes that produce at least one
2070 // promotion-requiring floating point result have their operands legalized as
2071 // a part of PromoteFloatResult.
2072 switch (N->getOpcode()) {
2073 default:
2074 #ifndef NDEBUG
2075 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2076 N->dump(&DAG); dbgs() << "\n";
2077 #endif
2078 llvm_unreachable("Do not know how to promote this operator's operand!");
2079
2080 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2081 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2082 case ISD::FP_TO_SINT:
2083 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
2084 case ISD::FP_TO_SINT_SAT:
2085 case ISD::FP_TO_UINT_SAT:
2086 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2087 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2088 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2089 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2090 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2091 }
2092
2093 if (R.getNode())
2094 ReplaceValueWith(SDValue(N, 0), R);
2095 return false;
2096 }
2097
PromoteFloatOp_BITCAST(SDNode * N,unsigned OpNo)2098 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2099 SDValue Op = N->getOperand(0);
2100 EVT OpVT = Op->getValueType(0);
2101
2102 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2103 EVT PromotedVT = Promoted->getValueType(0);
2104
2105 // Convert the promoted float value to the desired IVT.
2106 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2107 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2108 IVT, Promoted);
2109 // The final result type might not be an scalar so we need a bitcast. The
2110 // bitcast will be further legalized if needed.
2111 return DAG.getBitcast(N->getValueType(0), Convert);
2112 }
2113
2114 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2115 // PromoteFloatRes_FCOPYSIGN.
PromoteFloatOp_FCOPYSIGN(SDNode * N,unsigned OpNo)2116 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2117 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2118 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2119
2120 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2121 N->getOperand(0), Op1);
2122 }
2123
2124 // Convert the promoted float value to the desired integer type
PromoteFloatOp_FP_TO_XINT(SDNode * N,unsigned OpNo)2125 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
2126 SDValue Op = GetPromotedFloat(N->getOperand(0));
2127 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2128 }
2129
PromoteFloatOp_FP_TO_XINT_SAT(SDNode * N,unsigned OpNo)2130 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2131 unsigned OpNo) {
2132 SDValue Op = GetPromotedFloat(N->getOperand(0));
2133 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2134 N->getOperand(1));
2135 }
2136
PromoteFloatOp_FP_EXTEND(SDNode * N,unsigned OpNo)2137 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2138 SDValue Op = GetPromotedFloat(N->getOperand(0));
2139 EVT VT = N->getValueType(0);
2140
2141 // Desired VT is same as promoted type. Use promoted float directly.
2142 if (VT == Op->getValueType(0))
2143 return Op;
2144
2145 // Else, extend the promoted float value to the desired VT.
2146 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2147 }
2148
2149 // Promote the float operands used for comparison. The true- and false-
2150 // operands have the same type as the result and are promoted, if needed, by
2151 // PromoteFloatRes_SELECT_CC
PromoteFloatOp_SELECT_CC(SDNode * N,unsigned OpNo)2152 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2153 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2154 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2155
2156 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2157 LHS, RHS, N->getOperand(2), N->getOperand(3),
2158 N->getOperand(4));
2159 }
2160
2161 // Construct a SETCC that compares the promoted values and sets the conditional
2162 // code.
PromoteFloatOp_SETCC(SDNode * N,unsigned OpNo)2163 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2164 EVT VT = N->getValueType(0);
2165 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2166 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2167 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2168
2169 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2170
2171 }
2172
2173 // Lower the promoted Float down to the integer value of same size and construct
2174 // a STORE of the integer value.
PromoteFloatOp_STORE(SDNode * N,unsigned OpNo)2175 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2176 StoreSDNode *ST = cast<StoreSDNode>(N);
2177 SDValue Val = ST->getValue();
2178 SDLoc DL(N);
2179
2180 SDValue Promoted = GetPromotedFloat(Val);
2181 EVT VT = ST->getOperand(1).getValueType();
2182 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2183
2184 SDValue NewVal;
2185 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2186 IVT, Promoted);
2187
2188 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2189 ST->getMemOperand());
2190 }
2191
2192 //===----------------------------------------------------------------------===//
2193 // Float Result Promotion
2194 //===----------------------------------------------------------------------===//
2195
PromoteFloatResult(SDNode * N,unsigned ResNo)2196 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2197 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2198 dbgs() << "\n");
2199 SDValue R = SDValue();
2200
2201 // See if the target wants to custom expand this node.
2202 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2203 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2204 return;
2205 }
2206
2207 switch (N->getOpcode()) {
2208 // These opcodes cannot appear if promotion of FP16 is done in the backend
2209 // instead of Clang
2210 case ISD::FP16_TO_FP:
2211 case ISD::FP_TO_FP16:
2212 default:
2213 #ifndef NDEBUG
2214 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2215 N->dump(&DAG); dbgs() << "\n";
2216 #endif
2217 llvm_unreachable("Do not know how to promote this operator's result!");
2218
2219 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2220 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2221 case ISD::EXTRACT_VECTOR_ELT:
2222 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2223 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2224
2225 // Unary FP Operations
2226 case ISD::FABS:
2227 case ISD::FCBRT:
2228 case ISD::FCEIL:
2229 case ISD::FCOS:
2230 case ISD::FEXP:
2231 case ISD::FEXP2:
2232 case ISD::FFLOOR:
2233 case ISD::FLOG:
2234 case ISD::FLOG2:
2235 case ISD::FLOG10:
2236 case ISD::FNEARBYINT:
2237 case ISD::FNEG:
2238 case ISD::FRINT:
2239 case ISD::FROUND:
2240 case ISD::FROUNDEVEN:
2241 case ISD::FSIN:
2242 case ISD::FSQRT:
2243 case ISD::FTRUNC:
2244 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2245
2246 // Binary FP Operations
2247 case ISD::FADD:
2248 case ISD::FDIV:
2249 case ISD::FMAXIMUM:
2250 case ISD::FMINIMUM:
2251 case ISD::FMAXNUM:
2252 case ISD::FMINNUM:
2253 case ISD::FMUL:
2254 case ISD::FPOW:
2255 case ISD::FREM:
2256 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2257
2258 case ISD::FMA: // FMA is same as FMAD
2259 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2260
2261 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break;
2262
2263 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2264 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2265 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2266 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2267
2268 case ISD::SINT_TO_FP:
2269 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2270 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2271 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2272 case ISD::VECREDUCE_FADD:
2273 case ISD::VECREDUCE_FMUL:
2274 case ISD::VECREDUCE_FMIN:
2275 case ISD::VECREDUCE_FMAX:
2276 R = PromoteFloatRes_VECREDUCE(N);
2277 break;
2278 case ISD::VECREDUCE_SEQ_FADD:
2279 case ISD::VECREDUCE_SEQ_FMUL:
2280 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2281 break;
2282 }
2283
2284 if (R.getNode())
2285 SetPromotedFloat(SDValue(N, ResNo), R);
2286 }
2287
2288 // Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2289 // At this point, it is not possible to determine if the bitcast value is
2290 // eventually stored to memory or promoted to f32 or promoted to a floating
2291 // point at a higher precision. Some of these cases are handled by FP_EXTEND,
2292 // STORE promotion handlers.
PromoteFloatRes_BITCAST(SDNode * N)2293 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2294 EVT VT = N->getValueType(0);
2295 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2296 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2297 // bitcast will be legalized further if necessary.
2298 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2299 N->getOperand(0).getValueType().getSizeInBits());
2300 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2301 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2302 }
2303
PromoteFloatRes_ConstantFP(SDNode * N)2304 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2305 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2306 EVT VT = N->getValueType(0);
2307 SDLoc DL(N);
2308
2309 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2310 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2311 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2312 IVT);
2313
2314 // Convert the Constant to the desired FP type
2315 // FIXME We might be able to do the conversion during compilation and get rid
2316 // of it from the object code
2317 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2318 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2319 }
2320
2321 // If the Index operand is a constant, try to redirect the extract operation to
2322 // the correct legalized vector. If not, bit-convert the input vector to
2323 // equivalent integer vector. Extract the element as an (bit-cast) integer
2324 // value and convert it to the promoted type.
PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode * N)2325 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2326 SDLoc DL(N);
2327
2328 // If the index is constant, try to extract the value from the legalized
2329 // vector type.
2330 if (isa<ConstantSDNode>(N->getOperand(1))) {
2331 SDValue Vec = N->getOperand(0);
2332 SDValue Idx = N->getOperand(1);
2333 EVT VecVT = Vec->getValueType(0);
2334 EVT EltVT = VecVT.getVectorElementType();
2335
2336 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2337
2338 switch (getTypeAction(VecVT)) {
2339 default: break;
2340 case TargetLowering::TypeScalarizeVector: {
2341 SDValue Res = GetScalarizedVector(N->getOperand(0));
2342 ReplaceValueWith(SDValue(N, 0), Res);
2343 return SDValue();
2344 }
2345 case TargetLowering::TypeWidenVector: {
2346 Vec = GetWidenedVector(Vec);
2347 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2348 ReplaceValueWith(SDValue(N, 0), Res);
2349 return SDValue();
2350 }
2351 case TargetLowering::TypeSplitVector: {
2352 SDValue Lo, Hi;
2353 GetSplitVector(Vec, Lo, Hi);
2354
2355 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2356 SDValue Res;
2357 if (IdxVal < LoElts)
2358 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2359 else
2360 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2361 DAG.getConstant(IdxVal - LoElts, DL,
2362 Idx.getValueType()));
2363 ReplaceValueWith(SDValue(N, 0), Res);
2364 return SDValue();
2365 }
2366
2367 }
2368 }
2369
2370 // Bit-convert the input vector to the equivalent integer vector
2371 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2372 EVT IVT = NewOp.getValueType().getVectorElementType();
2373
2374 // Extract the element as an (bit-cast) integer value
2375 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2376 NewOp, N->getOperand(1));
2377
2378 // Convert the element to the desired FP type
2379 EVT VT = N->getValueType(0);
2380 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2381 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2382 }
2383
2384 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2385 // needs promotion, so does the argument X. Note that Y, if needed, will be
2386 // handled during operand promotion.
PromoteFloatRes_FCOPYSIGN(SDNode * N)2387 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2388 EVT VT = N->getValueType(0);
2389 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2390 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2391
2392 SDValue Op1 = N->getOperand(1);
2393
2394 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2395 }
2396
2397 // Unary operation where the result and the operand have PromoteFloat type
2398 // action. Construct a new SDNode with the promoted float value of the old
2399 // operand.
PromoteFloatRes_UnaryOp(SDNode * N)2400 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2401 EVT VT = N->getValueType(0);
2402 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2403 SDValue Op = GetPromotedFloat(N->getOperand(0));
2404
2405 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2406 }
2407
2408 // Binary operations where the result and both operands have PromoteFloat type
2409 // action. Construct a new SDNode with the promoted float values of the old
2410 // operands.
PromoteFloatRes_BinOp(SDNode * N)2411 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2412 EVT VT = N->getValueType(0);
2413 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2414 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2415 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2416 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2417 }
2418
PromoteFloatRes_FMAD(SDNode * N)2419 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2420 EVT VT = N->getValueType(0);
2421 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2422 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2423 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2424 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2425
2426 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2427 }
2428
2429 // Promote the Float (first) operand and retain the Integer (second) operand
PromoteFloatRes_FPOWI(SDNode * N)2430 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
2431 EVT VT = N->getValueType(0);
2432 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2433 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2434 SDValue Op1 = N->getOperand(1);
2435
2436 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2437 }
2438
2439 // Explicit operation to reduce precision. Reduce the value to half precision
2440 // and promote it back to the legal type.
PromoteFloatRes_FP_ROUND(SDNode * N)2441 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2442 SDLoc DL(N);
2443
2444 SDValue Op = N->getOperand(0);
2445 EVT VT = N->getValueType(0);
2446 EVT OpVT = Op->getValueType(0);
2447 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2448 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2449
2450 // Round promoted float to desired precision
2451 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2452 // Promote it back to the legal output type
2453 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2454 }
2455
PromoteFloatRes_LOAD(SDNode * N)2456 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2457 LoadSDNode *L = cast<LoadSDNode>(N);
2458 EVT VT = N->getValueType(0);
2459
2460 // Load the value as an integer value with the same number of bits.
2461 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2462 SDValue newL = DAG.getLoad(
2463 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2464 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2465 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2466 // Legalize the chain result by replacing uses of the old value chain with the
2467 // new one
2468 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2469
2470 // Convert the integer value to the desired FP type
2471 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2472 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2473 }
2474
2475 // Construct a new SELECT node with the promoted true- and false- values.
PromoteFloatRes_SELECT(SDNode * N)2476 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2477 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2478 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2479
2480 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2481 N->getOperand(0), TrueVal, FalseVal);
2482 }
2483
2484 // Construct a new SELECT_CC node with the promoted true- and false- values.
2485 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
PromoteFloatRes_SELECT_CC(SDNode * N)2486 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2487 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2488 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2489
2490 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2491 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2492 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2493 }
2494
2495 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
2496 // float type.
PromoteFloatRes_XINT_TO_FP(SDNode * N)2497 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2498 SDLoc DL(N);
2499 EVT VT = N->getValueType(0);
2500 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2501 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2502 // Round the value to the desired precision (that of the source type).
2503 return DAG.getNode(
2504 ISD::FP_EXTEND, DL, NVT,
2505 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL)));
2506 }
2507
PromoteFloatRes_UNDEF(SDNode * N)2508 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2509 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2510 N->getValueType(0)));
2511 }
2512
PromoteFloatRes_VECREDUCE(SDNode * N)2513 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2514 // Expand and promote recursively.
2515 // TODO: This is non-optimal, but dealing with the concurrently happening
2516 // vector-legalization is non-trivial. We could do something similar to
2517 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2518 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2519 return SDValue();
2520 }
2521
PromoteFloatRes_VECREDUCE_SEQ(SDNode * N)2522 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2523 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2524 return SDValue();
2525 }
2526
BitcastToInt_ATOMIC_SWAP(SDNode * N)2527 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2528 EVT VT = N->getValueType(0);
2529
2530 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2531 SDLoc SL(N);
2532
2533 SDValue CastVal = BitConvertToInteger(AM->getVal());
2534 EVT CastVT = CastVal.getValueType();
2535
2536 SDValue NewAtomic
2537 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2538 DAG.getVTList(CastVT, MVT::Other),
2539 { AM->getChain(), AM->getBasePtr(), CastVal },
2540 AM->getMemOperand());
2541
2542 SDValue Result = NewAtomic;
2543
2544 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2545 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2546 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2547 NewAtomic);
2548 }
2549
2550 // Legalize the chain result by replacing uses of the old value chain with the
2551 // new one
2552 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2553
2554 return Result;
2555
2556 }
2557
2558 //===----------------------------------------------------------------------===//
2559 // Half Result Soft Promotion
2560 //===----------------------------------------------------------------------===//
2561
SoftPromoteHalfResult(SDNode * N,unsigned ResNo)2562 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2563 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2564 N->dump(&DAG); dbgs() << "\n");
2565 SDValue R = SDValue();
2566
2567 // See if the target wants to custom expand this node.
2568 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2569 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2570 return;
2571 }
2572
2573 switch (N->getOpcode()) {
2574 default:
2575 #ifndef NDEBUG
2576 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2577 N->dump(&DAG); dbgs() << "\n";
2578 #endif
2579 llvm_unreachable("Do not know how to soft promote this operator's result!");
2580
2581 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2582 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2583 case ISD::EXTRACT_VECTOR_ELT:
2584 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2585 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2586 case ISD::STRICT_FP_ROUND:
2587 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2588
2589 // Unary FP Operations
2590 case ISD::FABS:
2591 case ISD::FCBRT:
2592 case ISD::FCEIL:
2593 case ISD::FCOS:
2594 case ISD::FEXP:
2595 case ISD::FEXP2:
2596 case ISD::FFLOOR:
2597 case ISD::FLOG:
2598 case ISD::FLOG2:
2599 case ISD::FLOG10:
2600 case ISD::FNEARBYINT:
2601 case ISD::FNEG:
2602 case ISD::FREEZE:
2603 case ISD::FRINT:
2604 case ISD::FROUND:
2605 case ISD::FROUNDEVEN:
2606 case ISD::FSIN:
2607 case ISD::FSQRT:
2608 case ISD::FTRUNC:
2609 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2610
2611 // Binary FP Operations
2612 case ISD::FADD:
2613 case ISD::FDIV:
2614 case ISD::FMAXIMUM:
2615 case ISD::FMINIMUM:
2616 case ISD::FMAXNUM:
2617 case ISD::FMINNUM:
2618 case ISD::FMUL:
2619 case ISD::FPOW:
2620 case ISD::FREM:
2621 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2622
2623 case ISD::FMA: // FMA is same as FMAD
2624 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2625
2626 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break;
2627
2628 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2629 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2630 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2631 case ISD::SINT_TO_FP:
2632 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2633 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2634 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2635 case ISD::VECREDUCE_FADD:
2636 case ISD::VECREDUCE_FMUL:
2637 case ISD::VECREDUCE_FMIN:
2638 case ISD::VECREDUCE_FMAX:
2639 R = SoftPromoteHalfRes_VECREDUCE(N);
2640 break;
2641 case ISD::VECREDUCE_SEQ_FADD:
2642 case ISD::VECREDUCE_SEQ_FMUL:
2643 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2644 break;
2645 }
2646
2647 if (R.getNode())
2648 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2649 }
2650
SoftPromoteHalfRes_BITCAST(SDNode * N)2651 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2652 return BitConvertToInteger(N->getOperand(0));
2653 }
2654
SoftPromoteHalfRes_ConstantFP(SDNode * N)2655 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2656 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2657
2658 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2659 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2660 MVT::i16);
2661 }
2662
SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode * N)2663 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2664 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2665 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2666 NewOp.getValueType().getVectorElementType(), NewOp,
2667 N->getOperand(1));
2668 }
2669
SoftPromoteHalfRes_FCOPYSIGN(SDNode * N)2670 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2671 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2672 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2673 SDLoc dl(N);
2674
2675 EVT LVT = LHS.getValueType();
2676 EVT RVT = RHS.getValueType();
2677
2678 unsigned LSize = LVT.getSizeInBits();
2679 unsigned RSize = RVT.getSizeInBits();
2680
2681 // First get the sign bit of second operand.
2682 SDValue SignBit = DAG.getNode(
2683 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2684 DAG.getConstant(RSize - 1, dl,
2685 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2686 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2687
2688 // Shift right or sign-extend it if the two operands have different types.
2689 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2690 if (SizeDiff > 0) {
2691 SignBit =
2692 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2693 DAG.getConstant(SizeDiff, dl,
2694 TLI.getShiftAmountTy(SignBit.getValueType(),
2695 DAG.getDataLayout())));
2696 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2697 } else if (SizeDiff < 0) {
2698 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2699 SignBit =
2700 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2701 DAG.getConstant(-SizeDiff, dl,
2702 TLI.getShiftAmountTy(SignBit.getValueType(),
2703 DAG.getDataLayout())));
2704 }
2705
2706 // Clear the sign bit of the first operand.
2707 SDValue Mask = DAG.getNode(
2708 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2709 DAG.getConstant(LSize - 1, dl,
2710 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2711 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2712 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2713
2714 // Or the value with the sign bit.
2715 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2716 }
2717
SoftPromoteHalfRes_FMAD(SDNode * N)2718 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2719 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2720 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2721 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2722 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2723 SDLoc dl(N);
2724
2725 // Promote to the larger FP type.
2726 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2727 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2728 Op2 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op2);
2729
2730 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2731
2732 // Convert back to FP16 as an integer.
2733 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2734 }
2735
SoftPromoteHalfRes_FPOWI(SDNode * N)2736 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) {
2737 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2738 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2739 SDValue Op1 = N->getOperand(1);
2740 SDLoc dl(N);
2741
2742 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2743
2744 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2745
2746 // Convert back to FP16 as an integer.
2747 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2748 }
2749
SoftPromoteHalfRes_FP_ROUND(SDNode * N)2750 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2751 if (N->isStrictFPOpcode()) {
2752 SDValue Res =
2753 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
2754 {N->getOperand(0), N->getOperand(1)});
2755 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2756 return Res;
2757 }
2758
2759 return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), MVT::i16, N->getOperand(0));
2760 }
2761
SoftPromoteHalfRes_LOAD(SDNode * N)2762 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2763 LoadSDNode *L = cast<LoadSDNode>(N);
2764
2765 // Load the value as an integer value with the same number of bits.
2766 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2767 SDValue NewL =
2768 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2769 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2770 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2771 L->getMemOperand()->getFlags(), L->getAAInfo());
2772 // Legalize the chain result by replacing uses of the old value chain with the
2773 // new one
2774 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2775 return NewL;
2776 }
2777
SoftPromoteHalfRes_SELECT(SDNode * N)2778 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2779 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2780 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2781 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2782 Op2);
2783 }
2784
SoftPromoteHalfRes_SELECT_CC(SDNode * N)2785 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2786 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2787 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2788 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2789 N->getOperand(0), N->getOperand(1), Op2, Op3,
2790 N->getOperand(4));
2791 }
2792
SoftPromoteHalfRes_XINT_TO_FP(SDNode * N)2793 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2794 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2795 SDLoc dl(N);
2796
2797 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2798
2799 // Round the value to the softened type.
2800 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2801 }
2802
SoftPromoteHalfRes_UNDEF(SDNode * N)2803 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2804 return DAG.getUNDEF(MVT::i16);
2805 }
2806
SoftPromoteHalfRes_UnaryOp(SDNode * N)2807 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2808 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2809 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2810 SDLoc dl(N);
2811
2812 // Promote to the larger FP type.
2813 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2814
2815 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2816
2817 // Convert back to FP16 as an integer.
2818 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2819 }
2820
SoftPromoteHalfRes_BinOp(SDNode * N)2821 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2822 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2823 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2824 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2825 SDLoc dl(N);
2826
2827 // Promote to the larger FP type.
2828 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2829 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2830
2831 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2832
2833 // Convert back to FP16 as an integer.
2834 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2835 }
2836
SoftPromoteHalfRes_VECREDUCE(SDNode * N)2837 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2838 // Expand and soften recursively.
2839 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2840 return SDValue();
2841 }
2842
SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode * N)2843 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
2844 // Expand and soften.
2845 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2846 return SDValue();
2847 }
2848
2849 //===----------------------------------------------------------------------===//
2850 // Half Operand Soft Promotion
2851 //===----------------------------------------------------------------------===//
2852
SoftPromoteHalfOperand(SDNode * N,unsigned OpNo)2853 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2854 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2855 N->dump(&DAG); dbgs() << "\n");
2856 SDValue Res = SDValue();
2857
2858 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2859 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2860 return false;
2861 }
2862
2863 // Nodes that use a promotion-requiring floating point operand, but doesn't
2864 // produce a soft promotion-requiring floating point result, need to be
2865 // legalized to use the soft promoted float operand. Nodes that produce at
2866 // least one soft promotion-requiring floating point result have their
2867 // operands legalized as a part of PromoteFloatResult.
2868 switch (N->getOpcode()) {
2869 default:
2870 #ifndef NDEBUG
2871 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2872 N->dump(&DAG); dbgs() << "\n";
2873 #endif
2874 llvm_unreachable("Do not know how to soft promote this operator's operand!");
2875
2876 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2877 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
2878 case ISD::FP_TO_SINT:
2879 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
2880 case ISD::FP_TO_SINT_SAT:
2881 case ISD::FP_TO_UINT_SAT:
2882 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
2883 case ISD::STRICT_FP_EXTEND:
2884 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
2885 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
2886 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
2887 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
2888 }
2889
2890 if (!Res.getNode())
2891 return false;
2892
2893 assert(Res.getNode() != N && "Expected a new node!");
2894
2895 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2896 "Invalid operand expansion");
2897
2898 ReplaceValueWith(SDValue(N, 0), Res);
2899 return false;
2900 }
2901
SoftPromoteHalfOp_BITCAST(SDNode * N)2902 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
2903 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2904
2905 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
2906 }
2907
SoftPromoteHalfOp_FCOPYSIGN(SDNode * N,unsigned OpNo)2908 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
2909 unsigned OpNo) {
2910 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2911 SDValue Op1 = N->getOperand(1);
2912 SDLoc dl(N);
2913
2914 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
2915
2916 Op1 = GetSoftPromotedHalf(Op1);
2917 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2918
2919 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
2920 Op1);
2921 }
2922
SoftPromoteHalfOp_FP_EXTEND(SDNode * N)2923 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
2924 bool IsStrict = N->isStrictFPOpcode();
2925 SDValue Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
2926
2927 if (IsStrict) {
2928 SDValue Res =
2929 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
2930 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
2931 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2932 ReplaceValueWith(SDValue(N, 0), Res);
2933 return SDValue();
2934 }
2935
2936 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), Op);
2937 }
2938
SoftPromoteHalfOp_FP_TO_XINT(SDNode * N)2939 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
2940 SDValue Op = N->getOperand(0);
2941 SDLoc dl(N);
2942
2943 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2944
2945 Op = GetSoftPromotedHalf(Op);
2946
2947 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2948
2949 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
2950 }
2951
SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode * N)2952 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
2953 SDValue Op = N->getOperand(0);
2954 SDLoc dl(N);
2955
2956 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2957
2958 Op = GetSoftPromotedHalf(Op);
2959
2960 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2961
2962 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
2963 N->getOperand(1));
2964 }
2965
SoftPromoteHalfOp_SELECT_CC(SDNode * N,unsigned OpNo)2966 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
2967 unsigned OpNo) {
2968 assert(OpNo == 0 && "Can only soften the comparison values");
2969 SDValue Op0 = N->getOperand(0);
2970 SDValue Op1 = N->getOperand(1);
2971 SDLoc dl(N);
2972
2973 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
2974
2975 Op0 = GetSoftPromotedHalf(Op0);
2976 Op1 = GetSoftPromotedHalf(Op1);
2977
2978 // Promote to the larger FP type.
2979 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2980 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2981
2982 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
2983 N->getOperand(2), N->getOperand(3), N->getOperand(4));
2984 }
2985
SoftPromoteHalfOp_SETCC(SDNode * N)2986 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
2987 SDValue Op0 = N->getOperand(0);
2988 SDValue Op1 = N->getOperand(1);
2989 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2990 SDLoc dl(N);
2991
2992 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
2993
2994 Op0 = GetSoftPromotedHalf(Op0);
2995 Op1 = GetSoftPromotedHalf(Op1);
2996
2997 // Promote to the larger FP type.
2998 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2999 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
3000
3001 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3002 }
3003
SoftPromoteHalfOp_STORE(SDNode * N,unsigned OpNo)3004 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3005 assert(OpNo == 1 && "Can only soften the stored value!");
3006 StoreSDNode *ST = cast<StoreSDNode>(N);
3007 SDValue Val = ST->getValue();
3008 SDLoc dl(N);
3009
3010 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3011 SDValue Promoted = GetSoftPromotedHalf(Val);
3012 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3013 ST->getMemOperand());
3014 }
3015