1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements float type expansion and softening for LegalizeTypes.
10 // Softening is the act of turning a computation in an illegal floating point
11 // type into a computation in an integer type of the same size; also known as
12 // "soft float". For example, turning f32 arithmetic into operations using i32.
13 // The resulting integer value is the same as what you would get by performing
14 // the floating point operation and bitcasting the result to the integer type.
15 // Expansion is the act of changing a computation in an illegal type to be a
16 // computation in two identical registers of a smaller type. For example,
17 // implementing ppcf128 arithmetic in two f64 registers.
18 //
19 //===----------------------------------------------------------------------===//
20
21 #include "LegalizeTypes.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 /// GetFPLibCall - Return the right libcall for the given floating point type.
30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31 /// refactored away (see RTLIB::getPOWI for an example).
GetFPLibCall(EVT VT,RTLIB::Libcall Call_F32,RTLIB::Libcall Call_F64,RTLIB::Libcall Call_F80,RTLIB::Libcall Call_F128,RTLIB::Libcall Call_PPCF128)32 static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45 }
46
47 //===----------------------------------------------------------------------===//
48 // Convert Float Results to Integer
49 //===----------------------------------------------------------------------===//
50
SoftenFloatResult(SDNode * N,unsigned ResNo)51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 default:
57 #ifndef NDEBUG
58 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
59 N->dump(&DAG); dbgs() << "\n";
60 #endif
61 report_fatal_error("Do not know how to soften the result of this "
62 "operator!");
63
64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
69 case ISD::EXTRACT_VECTOR_ELT:
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
72 case ISD::STRICT_FMINNUM:
73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
74 case ISD::STRICT_FMAXNUM:
75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
76 case ISD::STRICT_FADD:
77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
79 case ISD::STRICT_FCEIL:
80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
82 case ISD::STRICT_FCOS:
83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
84 case ISD::STRICT_FDIV:
85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
86 case ISD::STRICT_FEXP:
87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
88 case ISD::STRICT_FEXP2:
89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
90 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
91 case ISD::STRICT_FFLOOR:
92 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
93 case ISD::STRICT_FLOG:
94 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
95 case ISD::STRICT_FLOG2:
96 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
97 case ISD::STRICT_FLOG10:
98 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
99 case ISD::STRICT_FMA:
100 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
101 case ISD::STRICT_FMUL:
102 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
103 case ISD::STRICT_FNEARBYINT:
104 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
105 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
106 case ISD::STRICT_FP_EXTEND:
107 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
108 case ISD::STRICT_FP_ROUND:
109 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
110 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
111 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
112 case ISD::STRICT_FPOW:
113 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
114 case ISD::STRICT_FPOWI:
115 case ISD::FPOWI:
116 case ISD::FLDEXP:
117 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
118 case ISD::FFREXP:
119 R = SoftenFloatRes_FFREXP(N);
120 break;
121 case ISD::STRICT_FREM:
122 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
123 case ISD::STRICT_FRINT:
124 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
125 case ISD::STRICT_FROUND:
126 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
127 case ISD::STRICT_FROUNDEVEN:
128 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
129 case ISD::STRICT_FSIN:
130 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
131 case ISD::STRICT_FSQRT:
132 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
133 case ISD::STRICT_FSUB:
134 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
135 case ISD::STRICT_FTRUNC:
136 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
137 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
138 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
139 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
140 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
141 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
142 case ISD::STRICT_SINT_TO_FP:
143 case ISD::STRICT_UINT_TO_FP:
144 case ISD::SINT_TO_FP:
145 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
146 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
147 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
148 case ISD::VECREDUCE_FADD:
149 case ISD::VECREDUCE_FMUL:
150 case ISD::VECREDUCE_FMIN:
151 case ISD::VECREDUCE_FMAX:
152 case ISD::VECREDUCE_FMAXIMUM:
153 case ISD::VECREDUCE_FMINIMUM:
154 R = SoftenFloatRes_VECREDUCE(N);
155 break;
156 case ISD::VECREDUCE_SEQ_FADD:
157 case ISD::VECREDUCE_SEQ_FMUL:
158 R = SoftenFloatRes_VECREDUCE_SEQ(N);
159 break;
160 }
161
162 // If R is null, the sub-method took care of registering the result.
163 if (R.getNode()) {
164 assert(R.getNode() != N);
165 SetSoftenedFloat(SDValue(N, ResNo), R);
166 }
167 }
168
SoftenFloatRes_Unary(SDNode * N,RTLIB::Libcall LC)169 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
170 bool IsStrict = N->isStrictFPOpcode();
171 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
172 unsigned Offset = IsStrict ? 1 : 0;
173 assert(N->getNumOperands() == (1 + Offset) &&
174 "Unexpected number of operands!");
175 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
176 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
177 TargetLowering::MakeLibCallOptions CallOptions;
178 EVT OpVT = N->getOperand(0 + Offset).getValueType();
179 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
180 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
181 CallOptions, SDLoc(N),
182 Chain);
183 if (IsStrict)
184 ReplaceValueWith(SDValue(N, 1), Tmp.second);
185 return Tmp.first;
186 }
187
SoftenFloatRes_Binary(SDNode * N,RTLIB::Libcall LC)188 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
189 bool IsStrict = N->isStrictFPOpcode();
190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
191 unsigned Offset = IsStrict ? 1 : 0;
192 assert(N->getNumOperands() == (2 + Offset) &&
193 "Unexpected number of operands!");
194 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
195 GetSoftenedFloat(N->getOperand(1 + Offset)) };
196 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
197 TargetLowering::MakeLibCallOptions CallOptions;
198 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
199 N->getOperand(1 + Offset).getValueType() };
200 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
202 CallOptions, SDLoc(N),
203 Chain);
204 if (IsStrict)
205 ReplaceValueWith(SDValue(N, 1), Tmp.second);
206 return Tmp.first;
207 }
208
SoftenFloatRes_BITCAST(SDNode * N)209 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
210 return BitConvertToInteger(N->getOperand(0));
211 }
212
SoftenFloatRes_FREEZE(SDNode * N)213 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
214 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
215 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
216 GetSoftenedFloat(N->getOperand(0)));
217 }
218
SoftenFloatRes_ARITH_FENCE(SDNode * N)219 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
220 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
221 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
222 GetSoftenedFloat(N->getOperand(0)));
223 return NewFence;
224 }
225
SoftenFloatRes_MERGE_VALUES(SDNode * N,unsigned ResNo)226 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
227 unsigned ResNo) {
228 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
229 return BitConvertToInteger(Op);
230 }
231
SoftenFloatRes_BUILD_PAIR(SDNode * N)232 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
233 // Convert the inputs to integers, and build a new pair out of them.
234 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
235 TLI.getTypeToTransformTo(*DAG.getContext(),
236 N->getValueType(0)),
237 BitConvertToInteger(N->getOperand(0)),
238 BitConvertToInteger(N->getOperand(1)));
239 }
240
SoftenFloatRes_ConstantFP(SDNode * N)241 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
242 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
243 // In ppcf128, the high 64 bits are always first in memory regardless
244 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
245 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
246 // way. However, APInt's are serialized in an Endian-sensitive fashion,
247 // so on big-Endian targets, the two doubles are output in the wrong
248 // order. Fix this by manually flipping the order of the high 64 bits
249 // and the low 64 bits here.
250 if (DAG.getDataLayout().isBigEndian() &&
251 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
252 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
253 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
254 APInt Val(128, words);
255 return DAG.getConstant(Val, SDLoc(CN),
256 TLI.getTypeToTransformTo(*DAG.getContext(),
257 CN->getValueType(0)));
258 } else {
259 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
260 TLI.getTypeToTransformTo(*DAG.getContext(),
261 CN->getValueType(0)));
262 }
263 }
264
SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode * N,unsigned ResNo)265 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
266 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
267 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
268 NewOp.getValueType().getVectorElementType(),
269 NewOp, N->getOperand(1));
270 }
271
SoftenFloatRes_FABS(SDNode * N)272 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
273 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
274 unsigned Size = NVT.getSizeInBits();
275
276 // Mask = ~(1 << (Size-1))
277 APInt API = APInt::getAllOnes(Size);
278 API.clearBit(Size - 1);
279 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
280 SDValue Op = GetSoftenedFloat(N->getOperand(0));
281 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
282 }
283
SoftenFloatRes_FMINNUM(SDNode * N)284 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
285 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
286 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
287 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
288 RTLIB::FMIN_F32,
289 RTLIB::FMIN_F64,
290 RTLIB::FMIN_F80,
291 RTLIB::FMIN_F128,
292 RTLIB::FMIN_PPCF128));
293 }
294
SoftenFloatRes_FMAXNUM(SDNode * N)295 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
296 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
297 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
299 RTLIB::FMAX_F32,
300 RTLIB::FMAX_F64,
301 RTLIB::FMAX_F80,
302 RTLIB::FMAX_F128,
303 RTLIB::FMAX_PPCF128));
304 }
305
SoftenFloatRes_FADD(SDNode * N)306 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
308 RTLIB::ADD_F32,
309 RTLIB::ADD_F64,
310 RTLIB::ADD_F80,
311 RTLIB::ADD_F128,
312 RTLIB::ADD_PPCF128));
313 }
314
SoftenFloatRes_FCBRT(SDNode * N)315 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
317 RTLIB::CBRT_F32,
318 RTLIB::CBRT_F64,
319 RTLIB::CBRT_F80,
320 RTLIB::CBRT_F128,
321 RTLIB::CBRT_PPCF128));
322 }
323
SoftenFloatRes_FCEIL(SDNode * N)324 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
325 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
326 RTLIB::CEIL_F32,
327 RTLIB::CEIL_F64,
328 RTLIB::CEIL_F80,
329 RTLIB::CEIL_F128,
330 RTLIB::CEIL_PPCF128));
331 }
332
SoftenFloatRes_FCOPYSIGN(SDNode * N)333 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
334 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
335 SDValue RHS = BitConvertToInteger(N->getOperand(1));
336 SDLoc dl(N);
337
338 EVT LVT = LHS.getValueType();
339 EVT RVT = RHS.getValueType();
340
341 unsigned LSize = LVT.getSizeInBits();
342 unsigned RSize = RVT.getSizeInBits();
343
344 // First get the sign bit of second operand.
345 SDValue SignBit = DAG.getNode(
346 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
347 DAG.getConstant(RSize - 1, dl,
348 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
349 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
350
351 // Shift right or sign-extend it if the two operands have different types.
352 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
353 if (SizeDiff > 0) {
354 SignBit =
355 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
356 DAG.getConstant(SizeDiff, dl,
357 TLI.getShiftAmountTy(SignBit.getValueType(),
358 DAG.getDataLayout())));
359 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
360 } else if (SizeDiff < 0) {
361 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
362 SignBit =
363 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
364 DAG.getConstant(-SizeDiff, dl,
365 TLI.getShiftAmountTy(SignBit.getValueType(),
366 DAG.getDataLayout())));
367 }
368
369 // Clear the sign bit of the first operand.
370 SDValue Mask = DAG.getNode(
371 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
372 DAG.getConstant(LSize - 1, dl,
373 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
374 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
375 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
376
377 // Or the value with the sign bit.
378 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
379 }
380
SoftenFloatRes_FCOS(SDNode * N)381 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
382 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
383 RTLIB::COS_F32,
384 RTLIB::COS_F64,
385 RTLIB::COS_F80,
386 RTLIB::COS_F128,
387 RTLIB::COS_PPCF128));
388 }
389
SoftenFloatRes_FDIV(SDNode * N)390 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
391 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
392 RTLIB::DIV_F32,
393 RTLIB::DIV_F64,
394 RTLIB::DIV_F80,
395 RTLIB::DIV_F128,
396 RTLIB::DIV_PPCF128));
397 }
398
SoftenFloatRes_FEXP(SDNode * N)399 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
401 RTLIB::EXP_F32,
402 RTLIB::EXP_F64,
403 RTLIB::EXP_F80,
404 RTLIB::EXP_F128,
405 RTLIB::EXP_PPCF128));
406 }
407
SoftenFloatRes_FEXP2(SDNode * N)408 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
410 RTLIB::EXP2_F32,
411 RTLIB::EXP2_F64,
412 RTLIB::EXP2_F80,
413 RTLIB::EXP2_F128,
414 RTLIB::EXP2_PPCF128));
415 }
416
SoftenFloatRes_FEXP10(SDNode * N)417 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
418 return SoftenFloatRes_Unary(
419 N,
420 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
421 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
422 }
423
SoftenFloatRes_FFLOOR(SDNode * N)424 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
425 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
426 RTLIB::FLOOR_F32,
427 RTLIB::FLOOR_F64,
428 RTLIB::FLOOR_F80,
429 RTLIB::FLOOR_F128,
430 RTLIB::FLOOR_PPCF128));
431 }
432
SoftenFloatRes_FLOG(SDNode * N)433 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
434 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
435 RTLIB::LOG_F32,
436 RTLIB::LOG_F64,
437 RTLIB::LOG_F80,
438 RTLIB::LOG_F128,
439 RTLIB::LOG_PPCF128));
440 }
441
SoftenFloatRes_FLOG2(SDNode * N)442 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
443 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
444 RTLIB::LOG2_F32,
445 RTLIB::LOG2_F64,
446 RTLIB::LOG2_F80,
447 RTLIB::LOG2_F128,
448 RTLIB::LOG2_PPCF128));
449 }
450
SoftenFloatRes_FLOG10(SDNode * N)451 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
452 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
453 RTLIB::LOG10_F32,
454 RTLIB::LOG10_F64,
455 RTLIB::LOG10_F80,
456 RTLIB::LOG10_F128,
457 RTLIB::LOG10_PPCF128));
458 }
459
SoftenFloatRes_FMA(SDNode * N)460 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
461 bool IsStrict = N->isStrictFPOpcode();
462 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
463 unsigned Offset = IsStrict ? 1 : 0;
464 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
465 GetSoftenedFloat(N->getOperand(1 + Offset)),
466 GetSoftenedFloat(N->getOperand(2 + Offset)) };
467 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
468 TargetLowering::MakeLibCallOptions CallOptions;
469 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
470 N->getOperand(1 + Offset).getValueType(),
471 N->getOperand(2 + Offset).getValueType() };
472 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
473 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
474 GetFPLibCall(N->getValueType(0),
475 RTLIB::FMA_F32,
476 RTLIB::FMA_F64,
477 RTLIB::FMA_F80,
478 RTLIB::FMA_F128,
479 RTLIB::FMA_PPCF128),
480 NVT, Ops, CallOptions, SDLoc(N), Chain);
481 if (IsStrict)
482 ReplaceValueWith(SDValue(N, 1), Tmp.second);
483 return Tmp.first;
484 }
485
SoftenFloatRes_FMUL(SDNode * N)486 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
487 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
488 RTLIB::MUL_F32,
489 RTLIB::MUL_F64,
490 RTLIB::MUL_F80,
491 RTLIB::MUL_F128,
492 RTLIB::MUL_PPCF128));
493 }
494
SoftenFloatRes_FNEARBYINT(SDNode * N)495 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
496 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
497 RTLIB::NEARBYINT_F32,
498 RTLIB::NEARBYINT_F64,
499 RTLIB::NEARBYINT_F80,
500 RTLIB::NEARBYINT_F128,
501 RTLIB::NEARBYINT_PPCF128));
502 }
503
SoftenFloatRes_FNEG(SDNode * N)504 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
506 SDLoc dl(N);
507
508 // Expand Y = FNEG(X) -> Y = X ^ sign mask
509 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
510 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
511 DAG.getConstant(SignMask, dl, NVT));
512 }
513
SoftenFloatRes_FP_EXTEND(SDNode * N)514 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
515 bool IsStrict = N->isStrictFPOpcode();
516 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
517 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
518
519 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
520
521 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
522 Op = GetPromotedFloat(Op);
523 // If the promotion did the FP_EXTEND to the destination type for us,
524 // there's nothing left to do here.
525 if (Op.getValueType() == N->getValueType(0)) {
526 if (IsStrict)
527 ReplaceValueWith(SDValue(N, 1), Chain);
528 return BitConvertToInteger(Op);
529 }
530 }
531
532 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
533 // -> f32, so proceed in two stages. Also, it's entirely possible for both
534 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
535 // than FP16_TO_FP.
536 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
537 N->getValueType(0) != MVT::f32) {
538 if (IsStrict) {
539 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
540 { MVT::f32, MVT::Other }, { Chain, Op });
541 Chain = Op.getValue(1);
542 } else {
543 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
544 }
545 }
546
547 if (Op.getValueType() == MVT::bf16) {
548 // FIXME: Need ReplaceValueWith on chain in strict case
549 return SoftenFloatRes_BF16_TO_FP(N);
550 }
551
552 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
553 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
554 TargetLowering::MakeLibCallOptions CallOptions;
555 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
556 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
557 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
558 CallOptions, SDLoc(N),
559 Chain);
560 if (IsStrict)
561 ReplaceValueWith(SDValue(N, 1), Tmp.second);
562 return Tmp.first;
563 }
564
565 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
566 // nodes?
SoftenFloatRes_FP16_TO_FP(SDNode * N)567 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
568 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
569 SDValue Op = N->getOperand(0);
570 TargetLowering::MakeLibCallOptions CallOptions;
571 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
572 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
573 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
574 CallOptions, SDLoc(N)).first;
575 if (N->getValueType(0) == MVT::f32)
576 return Res32;
577
578 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
579 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
580 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
581 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
582 }
583
584 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
585 // nodes?
SoftenFloatRes_BF16_TO_FP(SDNode * N)586 SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
587 assert(N->getValueType(0) == MVT::f32 &&
588 "Can only soften BF16_TO_FP with f32 result");
589 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
590 SDValue Op = N->getOperand(0);
591 SDLoc DL(N);
592 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
593 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
594 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
595 DAG.getShiftAmountConstant(16, NVT, DL));
596 return Res;
597 }
598
SoftenFloatRes_FP_ROUND(SDNode * N)599 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
600 bool IsStrict = N->isStrictFPOpcode();
601 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
602 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
603 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
604 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
605 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
606 TargetLowering::MakeLibCallOptions CallOptions;
607 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
608 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
609 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
610 CallOptions, SDLoc(N),
611 Chain);
612 if (IsStrict)
613 ReplaceValueWith(SDValue(N, 1), Tmp.second);
614 return Tmp.first;
615 }
616
SoftenFloatRes_FPOW(SDNode * N)617 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
618 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
619 RTLIB::POW_F32,
620 RTLIB::POW_F64,
621 RTLIB::POW_F80,
622 RTLIB::POW_F128,
623 RTLIB::POW_PPCF128));
624 }
625
SoftenFloatRes_ExpOp(SDNode * N)626 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
627 bool IsStrict = N->isStrictFPOpcode();
628 unsigned Offset = IsStrict ? 1 : 0;
629 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
630 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
631 "Unsupported power type!");
632 bool IsPowI =
633 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
634
635 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
636 : RTLIB::getLDEXP(N->getValueType(0));
637 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
638 if (!TLI.getLibcallName(LC)) {
639 // Some targets don't have a powi libcall; use pow instead.
640 // FIXME: Implement this if some target needs it.
641 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
642 return DAG.getUNDEF(N->getValueType(0));
643 }
644
645 if (DAG.getLibInfo().getIntSize() !=
646 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
647 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
648 // would use the wrong type for the argument.
649 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
650 return DAG.getUNDEF(N->getValueType(0));
651 }
652
653 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
654 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
655 N->getOperand(1 + Offset) };
656 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
657 TargetLowering::MakeLibCallOptions CallOptions;
658 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
659 N->getOperand(1 + Offset).getValueType() };
660 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
661 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
662 CallOptions, SDLoc(N),
663 Chain);
664 if (IsStrict)
665 ReplaceValueWith(SDValue(N, 1), Tmp.second);
666 return Tmp.first;
667 }
668
SoftenFloatRes_FFREXP(SDNode * N)669 SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
670 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
671 EVT VT0 = N->getValueType(0);
672 EVT VT1 = N->getValueType(1);
673 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
674
675 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
676 // If the exponent does not match with sizeof(int) a libcall would use the
677 // wrong type for the argument.
678 // TODO: Should be able to handle mismatches.
679 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
680 return DAG.getUNDEF(N->getValueType(0));
681 }
682
683 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
684 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
685
686 SDLoc DL(N);
687
688 TargetLowering::MakeLibCallOptions CallOptions;
689 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
690 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
691
692 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
693 // but we only really need to handle the 0th one for softening anyway.
694 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
695
696 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
697 /*Chain=*/SDValue());
698 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
699 auto PtrInfo =
700 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
701
702 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
703
704 ReplaceValueWith(SDValue(N, 1), LoadExp);
705 return ReturnVal;
706 }
707
SoftenFloatRes_FREM(SDNode * N)708 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
709 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
710 RTLIB::REM_F32,
711 RTLIB::REM_F64,
712 RTLIB::REM_F80,
713 RTLIB::REM_F128,
714 RTLIB::REM_PPCF128));
715 }
716
SoftenFloatRes_FRINT(SDNode * N)717 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
718 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
719 RTLIB::RINT_F32,
720 RTLIB::RINT_F64,
721 RTLIB::RINT_F80,
722 RTLIB::RINT_F128,
723 RTLIB::RINT_PPCF128));
724 }
725
SoftenFloatRes_FROUND(SDNode * N)726 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
727 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
728 RTLIB::ROUND_F32,
729 RTLIB::ROUND_F64,
730 RTLIB::ROUND_F80,
731 RTLIB::ROUND_F128,
732 RTLIB::ROUND_PPCF128));
733 }
734
SoftenFloatRes_FROUNDEVEN(SDNode * N)735 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
736 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
737 RTLIB::ROUNDEVEN_F32,
738 RTLIB::ROUNDEVEN_F64,
739 RTLIB::ROUNDEVEN_F80,
740 RTLIB::ROUNDEVEN_F128,
741 RTLIB::ROUNDEVEN_PPCF128));
742 }
743
SoftenFloatRes_FSIN(SDNode * N)744 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
745 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
746 RTLIB::SIN_F32,
747 RTLIB::SIN_F64,
748 RTLIB::SIN_F80,
749 RTLIB::SIN_F128,
750 RTLIB::SIN_PPCF128));
751 }
752
SoftenFloatRes_FSQRT(SDNode * N)753 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
754 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
755 RTLIB::SQRT_F32,
756 RTLIB::SQRT_F64,
757 RTLIB::SQRT_F80,
758 RTLIB::SQRT_F128,
759 RTLIB::SQRT_PPCF128));
760 }
761
SoftenFloatRes_FSUB(SDNode * N)762 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
763 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
764 RTLIB::SUB_F32,
765 RTLIB::SUB_F64,
766 RTLIB::SUB_F80,
767 RTLIB::SUB_F128,
768 RTLIB::SUB_PPCF128));
769 }
770
SoftenFloatRes_FTRUNC(SDNode * N)771 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
772 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
773 RTLIB::TRUNC_F32,
774 RTLIB::TRUNC_F64,
775 RTLIB::TRUNC_F80,
776 RTLIB::TRUNC_F128,
777 RTLIB::TRUNC_PPCF128));
778 }
779
SoftenFloatRes_LOAD(SDNode * N)780 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
781 LoadSDNode *L = cast<LoadSDNode>(N);
782 EVT VT = N->getValueType(0);
783 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
784 SDLoc dl(N);
785
786 auto MMOFlags =
787 L->getMemOperand()->getFlags() &
788 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
789 SDValue NewL;
790 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
791 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
792 L->getChain(), L->getBasePtr(), L->getOffset(),
793 L->getPointerInfo(), NVT, L->getOriginalAlign(),
794 MMOFlags, L->getAAInfo());
795 // Legalized the chain result - switch anything that used the old chain to
796 // use the new one.
797 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
798 return NewL;
799 }
800
801 // Do a non-extending load followed by FP_EXTEND.
802 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
803 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
804 L->getPointerInfo(), L->getMemoryVT(),
805 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
806 // Legalized the chain result - switch anything that used the old chain to
807 // use the new one.
808 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
809 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
810 return BitConvertToInteger(ExtendNode);
811 }
812
SoftenFloatRes_SELECT(SDNode * N)813 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
814 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
815 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
816 return DAG.getSelect(SDLoc(N),
817 LHS.getValueType(), N->getOperand(0), LHS, RHS);
818 }
819
SoftenFloatRes_SELECT_CC(SDNode * N)820 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
821 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
822 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
823 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
824 LHS.getValueType(), N->getOperand(0),
825 N->getOperand(1), LHS, RHS, N->getOperand(4));
826 }
827
SoftenFloatRes_UNDEF(SDNode * N)828 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
829 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
830 N->getValueType(0)));
831 }
832
SoftenFloatRes_VAARG(SDNode * N)833 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
834 SDValue Chain = N->getOperand(0); // Get the chain.
835 SDValue Ptr = N->getOperand(1); // Get the pointer.
836 EVT VT = N->getValueType(0);
837 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
838 SDLoc dl(N);
839
840 SDValue NewVAARG;
841 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
842 N->getConstantOperandVal(3));
843
844 // Legalized the chain result - switch anything that used the old chain to
845 // use the new one.
846 if (N != NewVAARG.getValue(1).getNode())
847 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
848 return NewVAARG;
849 }
850
SoftenFloatRes_XINT_TO_FP(SDNode * N)851 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
852 bool IsStrict = N->isStrictFPOpcode();
853 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
854 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
855 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
856 EVT RVT = N->getValueType(0);
857 EVT NVT = EVT();
858 SDLoc dl(N);
859
860 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
861 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
862 // match. Look for an appropriate libcall.
863 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
864 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
865 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
866 NVT = (MVT::SimpleValueType)t;
867 // The source needs to big enough to hold the operand.
868 if (NVT.bitsGE(SVT))
869 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
870 }
871 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
872
873 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
874 // Sign/zero extend the argument if the libcall takes a larger type.
875 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
876 NVT, N->getOperand(IsStrict ? 1 : 0));
877 TargetLowering::MakeLibCallOptions CallOptions;
878 CallOptions.setSExt(Signed);
879 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
880 std::pair<SDValue, SDValue> Tmp =
881 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
882 Op, CallOptions, dl, Chain);
883
884 if (IsStrict)
885 ReplaceValueWith(SDValue(N, 1), Tmp.second);
886 return Tmp.first;
887 }
888
SoftenFloatRes_VECREDUCE(SDNode * N)889 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
890 // Expand and soften recursively.
891 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
892 return SDValue();
893 }
894
SoftenFloatRes_VECREDUCE_SEQ(SDNode * N)895 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
896 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
897 return SDValue();
898 }
899
900 //===----------------------------------------------------------------------===//
901 // Convert Float Operand to Integer
902 //===----------------------------------------------------------------------===//
903
SoftenFloatOperand(SDNode * N,unsigned OpNo)904 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
905 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
906 SDValue Res = SDValue();
907
908 switch (N->getOpcode()) {
909 default:
910 #ifndef NDEBUG
911 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
912 N->dump(&DAG); dbgs() << "\n";
913 #endif
914 report_fatal_error("Do not know how to soften this operator's operand!");
915
916 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
917 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
918 case ISD::STRICT_FP_TO_FP16:
919 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
920 case ISD::FP_TO_BF16:
921 case ISD::STRICT_FP_ROUND:
922 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
923 case ISD::STRICT_FP_TO_SINT:
924 case ISD::STRICT_FP_TO_UINT:
925 case ISD::FP_TO_SINT:
926 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
927 case ISD::FP_TO_SINT_SAT:
928 case ISD::FP_TO_UINT_SAT:
929 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
930 case ISD::STRICT_LROUND:
931 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
932 case ISD::STRICT_LLROUND:
933 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
934 case ISD::STRICT_LRINT:
935 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
936 case ISD::STRICT_LLRINT:
937 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
938 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
939 case ISD::STRICT_FSETCC:
940 case ISD::STRICT_FSETCCS:
941 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
942 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
943 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
944 }
945
946 // If the result is null, the sub-method took care of registering results etc.
947 if (!Res.getNode()) return false;
948
949 // If the result is N, the sub-method updated N in place. Tell the legalizer
950 // core about this to re-analyze.
951 if (Res.getNode() == N)
952 return true;
953
954 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
955 "Invalid operand softening");
956
957 ReplaceValueWith(SDValue(N, 0), Res);
958 return false;
959 }
960
SoftenFloatOp_BITCAST(SDNode * N)961 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
962 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
963
964 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
965 }
966
SoftenFloatOp_FP_ROUND(SDNode * N)967 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
968 // We actually deal with the partially-softened FP_TO_FP16 node too, which
969 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
970 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
971 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
972 N->getOpcode() == ISD::FP_TO_BF16 ||
973 N->getOpcode() == ISD::STRICT_FP_ROUND);
974
975 bool IsStrict = N->isStrictFPOpcode();
976 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
977 EVT SVT = Op.getValueType();
978 EVT RVT = N->getValueType(0);
979 EVT FloatRVT = RVT;
980 if (N->getOpcode() == ISD::FP_TO_FP16 ||
981 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
982 FloatRVT = MVT::f16;
983 else if (N->getOpcode() == ISD::FP_TO_BF16)
984 FloatRVT = MVT::bf16;
985
986 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
987 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
988
989 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
990 Op = GetSoftenedFloat(Op);
991 TargetLowering::MakeLibCallOptions CallOptions;
992 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
993 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
994 CallOptions, SDLoc(N),
995 Chain);
996 if (IsStrict) {
997 ReplaceValueWith(SDValue(N, 1), Tmp.second);
998 ReplaceValueWith(SDValue(N, 0), Tmp.first);
999 return SDValue();
1000 }
1001 return Tmp.first;
1002 }
1003
SoftenFloatOp_BR_CC(SDNode * N)1004 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1005 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1006 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1007
1008 EVT VT = NewLHS.getValueType();
1009 NewLHS = GetSoftenedFloat(NewLHS);
1010 NewRHS = GetSoftenedFloat(NewRHS);
1011 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1012 N->getOperand(2), N->getOperand(3));
1013
1014 // If softenSetCCOperands returned a scalar, we need to compare the result
1015 // against zero to select between true and false values.
1016 if (!NewRHS.getNode()) {
1017 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1018 CCCode = ISD::SETNE;
1019 }
1020
1021 // Update N to have the operands specified.
1022 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1023 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1024 N->getOperand(4)),
1025 0);
1026 }
1027
1028 // Even if the result type is legal, no libcall may exactly match. (e.g. We
1029 // don't have FP-i8 conversions) This helper method looks for an appropriate
1030 // promoted libcall.
findFPToIntLibcall(EVT SrcVT,EVT RetVT,EVT & Promoted,bool Signed)1031 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1032 bool Signed) {
1033 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1034 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1035 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1036 ++IntVT) {
1037 Promoted = (MVT::SimpleValueType)IntVT;
1038 // The type needs to big enough to hold the result.
1039 if (Promoted.bitsGE(RetVT))
1040 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1041 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1042 }
1043 return LC;
1044 }
1045
SoftenFloatOp_FP_TO_XINT(SDNode * N)1046 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1047 bool IsStrict = N->isStrictFPOpcode();
1048 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1049 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1050
1051 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1052 EVT SVT = Op.getValueType();
1053 EVT RVT = N->getValueType(0);
1054 EVT NVT = EVT();
1055 SDLoc dl(N);
1056
1057 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1058 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1059 // match, eg. we don't have fp -> i8 conversions.
1060 // Look for an appropriate libcall.
1061 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1062 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1063 "Unsupported FP_TO_XINT!");
1064
1065 Op = GetSoftenedFloat(Op);
1066 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1067 TargetLowering::MakeLibCallOptions CallOptions;
1068 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1069 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1070 CallOptions, dl, Chain);
1071
1072 // Truncate the result if the libcall returns a larger type.
1073 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1074
1075 if (!IsStrict)
1076 return Res;
1077
1078 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1079 ReplaceValueWith(SDValue(N, 0), Res);
1080 return SDValue();
1081 }
1082
SoftenFloatOp_FP_TO_XINT_SAT(SDNode * N)1083 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1084 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1085 return Res;
1086 }
1087
SoftenFloatOp_SELECT_CC(SDNode * N)1088 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1089 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1090 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1091
1092 EVT VT = NewLHS.getValueType();
1093 NewLHS = GetSoftenedFloat(NewLHS);
1094 NewRHS = GetSoftenedFloat(NewRHS);
1095 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1096 N->getOperand(0), N->getOperand(1));
1097
1098 // If softenSetCCOperands returned a scalar, we need to compare the result
1099 // against zero to select between true and false values.
1100 if (!NewRHS.getNode()) {
1101 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1102 CCCode = ISD::SETNE;
1103 }
1104
1105 // Update N to have the operands specified.
1106 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1107 N->getOperand(2), N->getOperand(3),
1108 DAG.getCondCode(CCCode)),
1109 0);
1110 }
1111
SoftenFloatOp_SETCC(SDNode * N)1112 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1113 bool IsStrict = N->isStrictFPOpcode();
1114 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1115 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1116 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1117 ISD::CondCode CCCode =
1118 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1119
1120 EVT VT = Op0.getValueType();
1121 SDValue NewLHS = GetSoftenedFloat(Op0);
1122 SDValue NewRHS = GetSoftenedFloat(Op1);
1123 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1124 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1125
1126 // Update N to have the operands specified.
1127 if (NewRHS.getNode()) {
1128 if (IsStrict)
1129 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1130 NewRHS, DAG.getCondCode(CCCode));
1131 else
1132 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1133 DAG.getCondCode(CCCode)), 0);
1134 }
1135
1136 // Otherwise, softenSetCCOperands returned a scalar, use it.
1137 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1138 "Unexpected setcc expansion!");
1139
1140 if (IsStrict) {
1141 ReplaceValueWith(SDValue(N, 0), NewLHS);
1142 ReplaceValueWith(SDValue(N, 1), Chain);
1143 return SDValue();
1144 }
1145 return NewLHS;
1146 }
1147
SoftenFloatOp_STORE(SDNode * N,unsigned OpNo)1148 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1149 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1150 assert(OpNo == 1 && "Can only soften the stored value!");
1151 StoreSDNode *ST = cast<StoreSDNode>(N);
1152 SDValue Val = ST->getValue();
1153 SDLoc dl(N);
1154
1155 if (ST->isTruncatingStore())
1156 // Do an FP_ROUND followed by a non-truncating store.
1157 Val = BitConvertToInteger(
1158 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1159 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1160 else
1161 Val = GetSoftenedFloat(Val);
1162
1163 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1164 ST->getMemOperand());
1165 }
1166
SoftenFloatOp_FCOPYSIGN(SDNode * N)1167 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1168 SDValue LHS = N->getOperand(0);
1169 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1170 SDLoc dl(N);
1171
1172 EVT LVT = LHS.getValueType();
1173 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1174 EVT RVT = RHS.getValueType();
1175
1176 unsigned LSize = LVT.getSizeInBits();
1177 unsigned RSize = RVT.getSizeInBits();
1178
1179 // Shift right or sign-extend it if the two operands have different types.
1180 int SizeDiff = RSize - LSize;
1181 if (SizeDiff > 0) {
1182 RHS =
1183 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1184 DAG.getConstant(SizeDiff, dl,
1185 TLI.getShiftAmountTy(RHS.getValueType(),
1186 DAG.getDataLayout())));
1187 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1188 } else if (SizeDiff < 0) {
1189 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1190 RHS =
1191 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1192 DAG.getConstant(-SizeDiff, dl,
1193 TLI.getShiftAmountTy(RHS.getValueType(),
1194 DAG.getDataLayout())));
1195 }
1196
1197 RHS = DAG.getBitcast(LVT, RHS);
1198 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1199 }
1200
SoftenFloatOp_Unary(SDNode * N,RTLIB::Libcall LC)1201 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1202 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1203 bool IsStrict = N->isStrictFPOpcode();
1204 unsigned Offset = IsStrict ? 1 : 0;
1205 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1206 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1207 TargetLowering::MakeLibCallOptions CallOptions;
1208 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1209 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1210 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1211 CallOptions, SDLoc(N),
1212 Chain);
1213 if (IsStrict) {
1214 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1215 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1216 return SDValue();
1217 }
1218
1219 return Tmp.first;
1220 }
1221
SoftenFloatOp_LROUND(SDNode * N)1222 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1223 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1224 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1225 RTLIB::LROUND_F32,
1226 RTLIB::LROUND_F64,
1227 RTLIB::LROUND_F80,
1228 RTLIB::LROUND_F128,
1229 RTLIB::LROUND_PPCF128));
1230 }
1231
SoftenFloatOp_LLROUND(SDNode * N)1232 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1233 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1234 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1235 RTLIB::LLROUND_F32,
1236 RTLIB::LLROUND_F64,
1237 RTLIB::LLROUND_F80,
1238 RTLIB::LLROUND_F128,
1239 RTLIB::LLROUND_PPCF128));
1240 }
1241
SoftenFloatOp_LRINT(SDNode * N)1242 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1243 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1244 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1245 RTLIB::LRINT_F32,
1246 RTLIB::LRINT_F64,
1247 RTLIB::LRINT_F80,
1248 RTLIB::LRINT_F128,
1249 RTLIB::LRINT_PPCF128));
1250 }
1251
SoftenFloatOp_LLRINT(SDNode * N)1252 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1253 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1254 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1255 RTLIB::LLRINT_F32,
1256 RTLIB::LLRINT_F64,
1257 RTLIB::LLRINT_F80,
1258 RTLIB::LLRINT_F128,
1259 RTLIB::LLRINT_PPCF128));
1260 }
1261
1262 //===----------------------------------------------------------------------===//
1263 // Float Result Expansion
1264 //===----------------------------------------------------------------------===//
1265
1266 /// ExpandFloatResult - This method is called when the specified result of the
1267 /// specified node is found to need expansion. At this point, the node may also
1268 /// have invalid operands or may have other results that need promotion, we just
1269 /// know that (at least) one result needs expansion.
ExpandFloatResult(SDNode * N,unsigned ResNo)1270 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1271 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1272 SDValue Lo, Hi;
1273 Lo = Hi = SDValue();
1274
1275 // See if the target wants to custom expand this node.
1276 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1277 return;
1278
1279 switch (N->getOpcode()) {
1280 default:
1281 #ifndef NDEBUG
1282 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1283 N->dump(&DAG); dbgs() << "\n";
1284 #endif
1285 report_fatal_error("Do not know how to expand the result of this "
1286 "operator!");
1287
1288 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1289 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1290 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1291
1292 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1293 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1294 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1295 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1296 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1297 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1298
1299 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1300 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1301 case ISD::STRICT_FMINNUM:
1302 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1303 case ISD::STRICT_FMAXNUM:
1304 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1305 case ISD::STRICT_FADD:
1306 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1307 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1308 case ISD::STRICT_FCEIL:
1309 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1310 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1311 case ISD::STRICT_FCOS:
1312 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1313 case ISD::STRICT_FDIV:
1314 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1315 case ISD::STRICT_FEXP:
1316 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1317 case ISD::STRICT_FEXP2:
1318 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1319 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1320 case ISD::STRICT_FFLOOR:
1321 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1322 case ISD::STRICT_FLOG:
1323 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1324 case ISD::STRICT_FLOG2:
1325 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1326 case ISD::STRICT_FLOG10:
1327 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1328 case ISD::STRICT_FMA:
1329 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1330 case ISD::STRICT_FMUL:
1331 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1332 case ISD::STRICT_FNEARBYINT:
1333 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1334 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1335 case ISD::STRICT_FP_EXTEND:
1336 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1337 case ISD::STRICT_FPOW:
1338 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1339 case ISD::STRICT_FPOWI:
1340 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1341 case ISD::FLDEXP:
1342 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1343 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1344 case ISD::STRICT_FRINT:
1345 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1346 case ISD::STRICT_FROUND:
1347 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1348 case ISD::STRICT_FROUNDEVEN:
1349 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1350 case ISD::STRICT_FSIN:
1351 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1352 case ISD::STRICT_FSQRT:
1353 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1354 case ISD::STRICT_FSUB:
1355 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1356 case ISD::STRICT_FTRUNC:
1357 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1358 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1359 case ISD::STRICT_SINT_TO_FP:
1360 case ISD::STRICT_UINT_TO_FP:
1361 case ISD::SINT_TO_FP:
1362 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1363 case ISD::STRICT_FREM:
1364 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1365 }
1366
1367 // If Lo/Hi is null, the sub-method took care of registering results etc.
1368 if (Lo.getNode())
1369 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1370 }
1371
ExpandFloatRes_ConstantFP(SDNode * N,SDValue & Lo,SDValue & Hi)1372 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1373 SDValue &Hi) {
1374 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1375 assert(NVT.getSizeInBits() == 64 &&
1376 "Do not know how to expand this float constant!");
1377 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1378 SDLoc dl(N);
1379 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1380 APInt(64, C.getRawData()[1])),
1381 dl, NVT);
1382 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1383 APInt(64, C.getRawData()[0])),
1384 dl, NVT);
1385 }
1386
ExpandFloatRes_Unary(SDNode * N,RTLIB::Libcall LC,SDValue & Lo,SDValue & Hi)1387 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1388 SDValue &Lo, SDValue &Hi) {
1389 bool IsStrict = N->isStrictFPOpcode();
1390 unsigned Offset = IsStrict ? 1 : 0;
1391 SDValue Op = N->getOperand(0 + Offset);
1392 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1393 TargetLowering::MakeLibCallOptions CallOptions;
1394 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1395 Op, CallOptions, SDLoc(N),
1396 Chain);
1397 if (IsStrict)
1398 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1399 GetPairElements(Tmp.first, Lo, Hi);
1400 }
1401
ExpandFloatRes_Binary(SDNode * N,RTLIB::Libcall LC,SDValue & Lo,SDValue & Hi)1402 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1403 SDValue &Lo, SDValue &Hi) {
1404 bool IsStrict = N->isStrictFPOpcode();
1405 unsigned Offset = IsStrict ? 1 : 0;
1406 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1407 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1408 TargetLowering::MakeLibCallOptions CallOptions;
1409 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1410 Ops, CallOptions, SDLoc(N),
1411 Chain);
1412 if (IsStrict)
1413 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1414 GetPairElements(Tmp.first, Lo, Hi);
1415 }
1416
ExpandFloatRes_FABS(SDNode * N,SDValue & Lo,SDValue & Hi)1417 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1418 SDValue &Hi) {
1419 assert(N->getValueType(0) == MVT::ppcf128 &&
1420 "Logic only correct for ppcf128!");
1421 SDLoc dl(N);
1422 SDValue Tmp;
1423 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1424 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1425 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1426 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1427 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1428 ISD::SETEQ);
1429 }
1430
ExpandFloatRes_FMINNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1431 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1432 SDValue &Hi) {
1433 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1434 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1435 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1436 RTLIB::FMIN_PPCF128), Lo, Hi);
1437 }
1438
ExpandFloatRes_FMAXNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1439 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1440 SDValue &Hi) {
1441 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1442 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1443 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1444 RTLIB::FMAX_PPCF128), Lo, Hi);
1445 }
1446
ExpandFloatRes_FADD(SDNode * N,SDValue & Lo,SDValue & Hi)1447 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1448 SDValue &Hi) {
1449 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1450 RTLIB::ADD_F32, RTLIB::ADD_F64,
1451 RTLIB::ADD_F80, RTLIB::ADD_F128,
1452 RTLIB::ADD_PPCF128), Lo, Hi);
1453 }
1454
ExpandFloatRes_FCBRT(SDNode * N,SDValue & Lo,SDValue & Hi)1455 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1456 SDValue &Hi) {
1457 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1458 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1459 RTLIB::CBRT_F128,
1460 RTLIB::CBRT_PPCF128), Lo, Hi);
1461 }
1462
ExpandFloatRes_FCEIL(SDNode * N,SDValue & Lo,SDValue & Hi)1463 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1464 SDValue &Lo, SDValue &Hi) {
1465 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1466 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1467 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1468 RTLIB::CEIL_PPCF128), Lo, Hi);
1469 }
1470
ExpandFloatRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1471 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1472 SDValue &Lo, SDValue &Hi) {
1473 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1474 RTLIB::COPYSIGN_F32,
1475 RTLIB::COPYSIGN_F64,
1476 RTLIB::COPYSIGN_F80,
1477 RTLIB::COPYSIGN_F128,
1478 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1479 }
1480
ExpandFloatRes_FCOS(SDNode * N,SDValue & Lo,SDValue & Hi)1481 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1482 SDValue &Lo, SDValue &Hi) {
1483 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1484 RTLIB::COS_F32, RTLIB::COS_F64,
1485 RTLIB::COS_F80, RTLIB::COS_F128,
1486 RTLIB::COS_PPCF128), Lo, Hi);
1487 }
1488
ExpandFloatRes_FDIV(SDNode * N,SDValue & Lo,SDValue & Hi)1489 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1490 SDValue &Hi) {
1491 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1492 RTLIB::DIV_F32,
1493 RTLIB::DIV_F64,
1494 RTLIB::DIV_F80,
1495 RTLIB::DIV_F128,
1496 RTLIB::DIV_PPCF128), Lo, Hi);
1497 }
1498
ExpandFloatRes_FEXP(SDNode * N,SDValue & Lo,SDValue & Hi)1499 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1500 SDValue &Lo, SDValue &Hi) {
1501 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1502 RTLIB::EXP_F32, RTLIB::EXP_F64,
1503 RTLIB::EXP_F80, RTLIB::EXP_F128,
1504 RTLIB::EXP_PPCF128), Lo, Hi);
1505 }
1506
ExpandFloatRes_FEXP2(SDNode * N,SDValue & Lo,SDValue & Hi)1507 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1508 SDValue &Lo, SDValue &Hi) {
1509 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1510 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1511 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1512 RTLIB::EXP2_PPCF128), Lo, Hi);
1513 }
1514
ExpandFloatRes_FEXP10(SDNode * N,SDValue & Lo,SDValue & Hi)1515 void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1516 SDValue &Hi) {
1517 ExpandFloatRes_Unary(N,
1518 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1519 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1520 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1521 Lo, Hi);
1522 }
1523
ExpandFloatRes_FFLOOR(SDNode * N,SDValue & Lo,SDValue & Hi)1524 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1525 SDValue &Lo, SDValue &Hi) {
1526 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1527 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1528 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1529 RTLIB::FLOOR_PPCF128), Lo, Hi);
1530 }
1531
ExpandFloatRes_FLOG(SDNode * N,SDValue & Lo,SDValue & Hi)1532 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1533 SDValue &Lo, SDValue &Hi) {
1534 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1535 RTLIB::LOG_F32, RTLIB::LOG_F64,
1536 RTLIB::LOG_F80, RTLIB::LOG_F128,
1537 RTLIB::LOG_PPCF128), Lo, Hi);
1538 }
1539
ExpandFloatRes_FLOG2(SDNode * N,SDValue & Lo,SDValue & Hi)1540 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1541 SDValue &Lo, SDValue &Hi) {
1542 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1543 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1544 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1545 RTLIB::LOG2_PPCF128), Lo, Hi);
1546 }
1547
ExpandFloatRes_FLOG10(SDNode * N,SDValue & Lo,SDValue & Hi)1548 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1549 SDValue &Lo, SDValue &Hi) {
1550 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1551 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1552 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1553 RTLIB::LOG10_PPCF128), Lo, Hi);
1554 }
1555
ExpandFloatRes_FMA(SDNode * N,SDValue & Lo,SDValue & Hi)1556 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1557 SDValue &Hi) {
1558 bool IsStrict = N->isStrictFPOpcode();
1559 unsigned Offset = IsStrict ? 1 : 0;
1560 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1561 N->getOperand(2 + Offset) };
1562 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1563 TargetLowering::MakeLibCallOptions CallOptions;
1564 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1565 RTLIB::FMA_F32,
1566 RTLIB::FMA_F64,
1567 RTLIB::FMA_F80,
1568 RTLIB::FMA_F128,
1569 RTLIB::FMA_PPCF128),
1570 N->getValueType(0), Ops, CallOptions,
1571 SDLoc(N), Chain);
1572 if (IsStrict)
1573 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1574 GetPairElements(Tmp.first, Lo, Hi);
1575 }
1576
ExpandFloatRes_FMUL(SDNode * N,SDValue & Lo,SDValue & Hi)1577 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1578 SDValue &Hi) {
1579 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1580 RTLIB::MUL_F32,
1581 RTLIB::MUL_F64,
1582 RTLIB::MUL_F80,
1583 RTLIB::MUL_F128,
1584 RTLIB::MUL_PPCF128), Lo, Hi);
1585 }
1586
ExpandFloatRes_FNEARBYINT(SDNode * N,SDValue & Lo,SDValue & Hi)1587 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1588 SDValue &Lo, SDValue &Hi) {
1589 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1590 RTLIB::NEARBYINT_F32,
1591 RTLIB::NEARBYINT_F64,
1592 RTLIB::NEARBYINT_F80,
1593 RTLIB::NEARBYINT_F128,
1594 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1595 }
1596
ExpandFloatRes_FNEG(SDNode * N,SDValue & Lo,SDValue & Hi)1597 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1598 SDValue &Hi) {
1599 SDLoc dl(N);
1600 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1601 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1602 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1603 }
1604
ExpandFloatRes_FP_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)1605 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1606 SDValue &Hi) {
1607 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1608 SDLoc dl(N);
1609 bool IsStrict = N->isStrictFPOpcode();
1610
1611 SDValue Chain;
1612 if (IsStrict) {
1613 // If the expanded type is the same as the input type, just bypass the node.
1614 if (NVT == N->getOperand(1).getValueType()) {
1615 Hi = N->getOperand(1);
1616 Chain = N->getOperand(0);
1617 } else {
1618 // Other we need to extend.
1619 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1620 { N->getOperand(0), N->getOperand(1) });
1621 Chain = Hi.getValue(1);
1622 }
1623 } else {
1624 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1625 }
1626
1627 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1628 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1629
1630 if (IsStrict)
1631 ReplaceValueWith(SDValue(N, 1), Chain);
1632 }
1633
ExpandFloatRes_FPOW(SDNode * N,SDValue & Lo,SDValue & Hi)1634 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1635 SDValue &Lo, SDValue &Hi) {
1636 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1637 RTLIB::POW_F32, RTLIB::POW_F64,
1638 RTLIB::POW_F80, RTLIB::POW_F128,
1639 RTLIB::POW_PPCF128), Lo, Hi);
1640 }
1641
ExpandFloatRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)1642 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1643 SDValue &Lo, SDValue &Hi) {
1644 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1645 }
1646
ExpandFloatRes_FLDEXP(SDNode * N,SDValue & Lo,SDValue & Hi)1647 void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1648 SDValue &Hi) {
1649 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1650 }
1651
ExpandFloatRes_FREEZE(SDNode * N,SDValue & Lo,SDValue & Hi)1652 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1653 SDValue &Lo, SDValue &Hi) {
1654 assert(N->getValueType(0) == MVT::ppcf128 &&
1655 "Logic only correct for ppcf128!");
1656
1657 SDLoc dl(N);
1658 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1659 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1660 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1661 }
1662
ExpandFloatRes_FREM(SDNode * N,SDValue & Lo,SDValue & Hi)1663 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1664 SDValue &Lo, SDValue &Hi) {
1665 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1666 RTLIB::REM_F32, RTLIB::REM_F64,
1667 RTLIB::REM_F80, RTLIB::REM_F128,
1668 RTLIB::REM_PPCF128), Lo, Hi);
1669 }
1670
ExpandFloatRes_FRINT(SDNode * N,SDValue & Lo,SDValue & Hi)1671 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1672 SDValue &Lo, SDValue &Hi) {
1673 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1674 RTLIB::RINT_F32, RTLIB::RINT_F64,
1675 RTLIB::RINT_F80, RTLIB::RINT_F128,
1676 RTLIB::RINT_PPCF128), Lo, Hi);
1677 }
1678
ExpandFloatRes_FROUND(SDNode * N,SDValue & Lo,SDValue & Hi)1679 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1680 SDValue &Lo, SDValue &Hi) {
1681 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1682 RTLIB::ROUND_F32,
1683 RTLIB::ROUND_F64,
1684 RTLIB::ROUND_F80,
1685 RTLIB::ROUND_F128,
1686 RTLIB::ROUND_PPCF128), Lo, Hi);
1687 }
1688
ExpandFloatRes_FROUNDEVEN(SDNode * N,SDValue & Lo,SDValue & Hi)1689 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1690 SDValue &Lo, SDValue &Hi) {
1691 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1692 RTLIB::ROUNDEVEN_F32,
1693 RTLIB::ROUNDEVEN_F64,
1694 RTLIB::ROUNDEVEN_F80,
1695 RTLIB::ROUNDEVEN_F128,
1696 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1697 }
1698
ExpandFloatRes_FSIN(SDNode * N,SDValue & Lo,SDValue & Hi)1699 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1700 SDValue &Lo, SDValue &Hi) {
1701 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1702 RTLIB::SIN_F32, RTLIB::SIN_F64,
1703 RTLIB::SIN_F80, RTLIB::SIN_F128,
1704 RTLIB::SIN_PPCF128), Lo, Hi);
1705 }
1706
ExpandFloatRes_FSQRT(SDNode * N,SDValue & Lo,SDValue & Hi)1707 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1708 SDValue &Lo, SDValue &Hi) {
1709 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1710 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1711 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1712 RTLIB::SQRT_PPCF128), Lo, Hi);
1713 }
1714
ExpandFloatRes_FSUB(SDNode * N,SDValue & Lo,SDValue & Hi)1715 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1716 SDValue &Hi) {
1717 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1718 RTLIB::SUB_F32,
1719 RTLIB::SUB_F64,
1720 RTLIB::SUB_F80,
1721 RTLIB::SUB_F128,
1722 RTLIB::SUB_PPCF128), Lo, Hi);
1723 }
1724
ExpandFloatRes_FTRUNC(SDNode * N,SDValue & Lo,SDValue & Hi)1725 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1726 SDValue &Lo, SDValue &Hi) {
1727 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1728 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1729 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1730 RTLIB::TRUNC_PPCF128), Lo, Hi);
1731 }
1732
ExpandFloatRes_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)1733 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1734 SDValue &Hi) {
1735 if (ISD::isNormalLoad(N)) {
1736 ExpandRes_NormalLoad(N, Lo, Hi);
1737 return;
1738 }
1739
1740 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1741 LoadSDNode *LD = cast<LoadSDNode>(N);
1742 SDValue Chain = LD->getChain();
1743 SDValue Ptr = LD->getBasePtr();
1744 SDLoc dl(N);
1745
1746 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1747 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1748 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1749
1750 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1751 LD->getMemoryVT(), LD->getMemOperand());
1752
1753 // Remember the chain.
1754 Chain = Hi.getValue(1);
1755
1756 // The low part is zero.
1757 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1758 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1759
1760 // Modified the chain - switch anything that used the old chain to use the
1761 // new one.
1762 ReplaceValueWith(SDValue(LD, 1), Chain);
1763 }
1764
ExpandFloatRes_XINT_TO_FP(SDNode * N,SDValue & Lo,SDValue & Hi)1765 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1766 SDValue &Hi) {
1767 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1768 EVT VT = N->getValueType(0);
1769 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1770 bool Strict = N->isStrictFPOpcode();
1771 SDValue Src = N->getOperand(Strict ? 1 : 0);
1772 EVT SrcVT = Src.getValueType();
1773 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1774 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1775 SDLoc dl(N);
1776 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1777
1778 // TODO: Any other flags to propagate?
1779 SDNodeFlags Flags;
1780 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1781
1782 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1783 // When promoting partial word types to i32 we must honor the signedness,
1784 // though.
1785 if (SrcVT.bitsLE(MVT::i32)) {
1786 // The integer can be represented exactly in an f64.
1787 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1788 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1789 if (Strict) {
1790 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1791 {Chain, Src}, Flags);
1792 Chain = Hi.getValue(1);
1793 } else
1794 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1795 } else {
1796 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1797 if (SrcVT.bitsLE(MVT::i64)) {
1798 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1799 MVT::i64, Src);
1800 LC = RTLIB::SINTTOFP_I64_PPCF128;
1801 } else if (SrcVT.bitsLE(MVT::i128)) {
1802 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1803 LC = RTLIB::SINTTOFP_I128_PPCF128;
1804 }
1805 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1806
1807 TargetLowering::MakeLibCallOptions CallOptions;
1808 CallOptions.setSExt(true);
1809 std::pair<SDValue, SDValue> Tmp =
1810 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1811 if (Strict)
1812 Chain = Tmp.second;
1813 GetPairElements(Tmp.first, Lo, Hi);
1814 }
1815
1816 // No need to complement for unsigned 32-bit integers
1817 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1818 if (Strict)
1819 ReplaceValueWith(SDValue(N, 1), Chain);
1820
1821 return;
1822 }
1823
1824 // Unsigned - fix up the SINT_TO_FP value just calculated.
1825 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1826 // keep semantics correctness if the integer is not exactly representable
1827 // here. See ExpandLegalINT_TO_FP.
1828 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1829 SrcVT = Src.getValueType();
1830
1831 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1832 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1833 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1834 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1835 ArrayRef<uint64_t> Parts;
1836
1837 switch (SrcVT.getSimpleVT().SimpleTy) {
1838 default:
1839 llvm_unreachable("Unsupported UINT_TO_FP!");
1840 case MVT::i32:
1841 Parts = TwoE32;
1842 break;
1843 case MVT::i64:
1844 Parts = TwoE64;
1845 break;
1846 case MVT::i128:
1847 Parts = TwoE128;
1848 break;
1849 }
1850
1851 // TODO: Are there other fast-math-flags to propagate to this FADD?
1852 SDValue NewLo = DAG.getConstantFP(
1853 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1854 if (Strict) {
1855 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1856 {Chain, Hi, NewLo}, Flags);
1857 Chain = Lo.getValue(1);
1858 ReplaceValueWith(SDValue(N, 1), Chain);
1859 } else
1860 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1861 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1862 Lo, Hi, ISD::SETLT);
1863 GetPairElements(Lo, Lo, Hi);
1864 }
1865
1866
1867 //===----------------------------------------------------------------------===//
1868 // Float Operand Expansion
1869 //===----------------------------------------------------------------------===//
1870
1871 /// ExpandFloatOperand - This method is called when the specified operand of the
1872 /// specified node is found to need expansion. At this point, all of the result
1873 /// types of the node are known to be legal, but other operands of the node may
1874 /// need promotion or expansion as well as the specified one.
ExpandFloatOperand(SDNode * N,unsigned OpNo)1875 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1876 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1877 SDValue Res = SDValue();
1878
1879 // See if the target wants to custom expand this node.
1880 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1881 return false;
1882
1883 switch (N->getOpcode()) {
1884 default:
1885 #ifndef NDEBUG
1886 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1887 N->dump(&DAG); dbgs() << "\n";
1888 #endif
1889 report_fatal_error("Do not know how to expand this operator's operand!");
1890
1891 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1892 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1893 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1894
1895 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1896 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1897 case ISD::STRICT_FP_ROUND:
1898 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1899 case ISD::STRICT_FP_TO_SINT:
1900 case ISD::STRICT_FP_TO_UINT:
1901 case ISD::FP_TO_SINT:
1902 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1903 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1904 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1905 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1906 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1907 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1908 case ISD::STRICT_FSETCC:
1909 case ISD::STRICT_FSETCCS:
1910 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1911 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1912 OpNo); break;
1913 }
1914
1915 // If the result is null, the sub-method took care of registering results etc.
1916 if (!Res.getNode()) return false;
1917
1918 // If the result is N, the sub-method updated N in place. Tell the legalizer
1919 // core about this.
1920 if (Res.getNode() == N)
1921 return true;
1922
1923 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1924 "Invalid operand expansion");
1925
1926 ReplaceValueWith(SDValue(N, 0), Res);
1927 return false;
1928 }
1929
1930 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1931 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
FloatExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl,SDValue & Chain,bool IsSignaling)1932 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1933 SDValue &NewRHS,
1934 ISD::CondCode &CCCode,
1935 const SDLoc &dl, SDValue &Chain,
1936 bool IsSignaling) {
1937 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1938 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1939 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1940
1941 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1942
1943 // FIXME: This generated code sucks. We want to generate
1944 // FCMPU crN, hi1, hi2
1945 // BNE crN, L:
1946 // FCMPU crN, lo1, lo2
1947 // The following can be improved, but not that much.
1948 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1949 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1950 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1951 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1952 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1953 RHSLo, CCCode, OutputChain, IsSignaling);
1954 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1955 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1956 Tmp1 =
1957 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1958 ISD::SETUNE, OutputChain, IsSignaling);
1959 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1960 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1961 RHSHi, CCCode, OutputChain, IsSignaling);
1962 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1963 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1964 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1965 NewRHS = SDValue(); // LHS is the result, not a compare.
1966 Chain = OutputChain;
1967 }
1968
ExpandFloatOp_BR_CC(SDNode * N)1969 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1970 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1971 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1972 SDValue Chain;
1973 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1974
1975 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1976 // against zero to select between true and false values.
1977 if (!NewRHS.getNode()) {
1978 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1979 CCCode = ISD::SETNE;
1980 }
1981
1982 // Update N to have the operands specified.
1983 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1984 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1985 N->getOperand(4)), 0);
1986 }
1987
ExpandFloatOp_FCOPYSIGN(SDNode * N)1988 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1989 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1990 "Logic only correct for ppcf128!");
1991 SDValue Lo, Hi;
1992 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1993 // The ppcf128 value is providing only the sign; take it from the
1994 // higher-order double (which must have the larger magnitude).
1995 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1996 N->getValueType(0), N->getOperand(0), Hi);
1997 }
1998
ExpandFloatOp_FP_ROUND(SDNode * N)1999 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2000 bool IsStrict = N->isStrictFPOpcode();
2001 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2002 "Logic only correct for ppcf128!");
2003 SDValue Lo, Hi;
2004 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2005
2006 if (!IsStrict)
2007 // Round it the rest of the way (e.g. to f32) if needed.
2008 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2009 N->getValueType(0), Hi, N->getOperand(1));
2010
2011 // Eliminate the node if the input float type is the same as the output float
2012 // type.
2013 if (Hi.getValueType() == N->getValueType(0)) {
2014 // Connect the output chain to the input chain, unlinking the node.
2015 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2016 ReplaceValueWith(SDValue(N, 0), Hi);
2017 return SDValue();
2018 }
2019
2020 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2021 {N->getValueType(0), MVT::Other},
2022 {N->getOperand(0), Hi, N->getOperand(2)});
2023 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2024 ReplaceValueWith(SDValue(N, 0), Expansion);
2025 return SDValue();
2026 }
2027
ExpandFloatOp_FP_TO_XINT(SDNode * N)2028 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2029 EVT RVT = N->getValueType(0);
2030 SDLoc dl(N);
2031
2032 bool IsStrict = N->isStrictFPOpcode();
2033 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2034 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2035 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2036 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2037
2038 EVT NVT;
2039 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2040 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2041 "Unsupported FP_TO_XINT!");
2042 TargetLowering::MakeLibCallOptions CallOptions;
2043 std::pair<SDValue, SDValue> Tmp =
2044 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2045 if (!IsStrict)
2046 return Tmp.first;
2047
2048 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2049 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2050 return SDValue();
2051 }
2052
ExpandFloatOp_SELECT_CC(SDNode * N)2053 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2054 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2055 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2056 SDValue Chain;
2057 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2058
2059 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2060 // against zero to select between true and false values.
2061 if (!NewRHS.getNode()) {
2062 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2063 CCCode = ISD::SETNE;
2064 }
2065
2066 // Update N to have the operands specified.
2067 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2068 N->getOperand(2), N->getOperand(3),
2069 DAG.getCondCode(CCCode)), 0);
2070 }
2071
ExpandFloatOp_SETCC(SDNode * N)2072 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2073 bool IsStrict = N->isStrictFPOpcode();
2074 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2075 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2076 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2077 ISD::CondCode CCCode =
2078 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2079 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2080 N->getOpcode() == ISD::STRICT_FSETCCS);
2081
2082 // FloatExpandSetCCOperands always returned a scalar.
2083 assert(!NewRHS.getNode() && "Expect to return scalar");
2084 assert(NewLHS.getValueType() == N->getValueType(0) &&
2085 "Unexpected setcc expansion!");
2086 if (Chain) {
2087 ReplaceValueWith(SDValue(N, 0), NewLHS);
2088 ReplaceValueWith(SDValue(N, 1), Chain);
2089 return SDValue();
2090 }
2091 return NewLHS;
2092 }
2093
ExpandFloatOp_STORE(SDNode * N,unsigned OpNo)2094 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2095 if (ISD::isNormalStore(N))
2096 return ExpandOp_NormalStore(N, OpNo);
2097
2098 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2099 assert(OpNo == 1 && "Can only expand the stored value so far");
2100 StoreSDNode *ST = cast<StoreSDNode>(N);
2101
2102 SDValue Chain = ST->getChain();
2103 SDValue Ptr = ST->getBasePtr();
2104
2105 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2106 ST->getValue().getValueType());
2107 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2108 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2109 (void)NVT;
2110
2111 SDValue Lo, Hi;
2112 GetExpandedOp(ST->getValue(), Lo, Hi);
2113
2114 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2115 ST->getMemoryVT(), ST->getMemOperand());
2116 }
2117
ExpandFloatOp_LROUND(SDNode * N)2118 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2119 EVT RVT = N->getValueType(0);
2120 EVT RetVT = N->getOperand(0).getValueType();
2121 TargetLowering::MakeLibCallOptions CallOptions;
2122 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2123 RTLIB::LROUND_F32,
2124 RTLIB::LROUND_F64,
2125 RTLIB::LROUND_F80,
2126 RTLIB::LROUND_F128,
2127 RTLIB::LROUND_PPCF128),
2128 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2129 }
2130
ExpandFloatOp_LLROUND(SDNode * N)2131 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2132 EVT RVT = N->getValueType(0);
2133 EVT RetVT = N->getOperand(0).getValueType();
2134 TargetLowering::MakeLibCallOptions CallOptions;
2135 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2136 RTLIB::LLROUND_F32,
2137 RTLIB::LLROUND_F64,
2138 RTLIB::LLROUND_F80,
2139 RTLIB::LLROUND_F128,
2140 RTLIB::LLROUND_PPCF128),
2141 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2142 }
2143
ExpandFloatOp_LRINT(SDNode * N)2144 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2145 EVT RVT = N->getValueType(0);
2146 EVT RetVT = N->getOperand(0).getValueType();
2147 TargetLowering::MakeLibCallOptions CallOptions;
2148 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2149 RTLIB::LRINT_F32,
2150 RTLIB::LRINT_F64,
2151 RTLIB::LRINT_F80,
2152 RTLIB::LRINT_F128,
2153 RTLIB::LRINT_PPCF128),
2154 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2155 }
2156
ExpandFloatOp_LLRINT(SDNode * N)2157 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2158 EVT RVT = N->getValueType(0);
2159 EVT RetVT = N->getOperand(0).getValueType();
2160 TargetLowering::MakeLibCallOptions CallOptions;
2161 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2162 RTLIB::LLRINT_F32,
2163 RTLIB::LLRINT_F64,
2164 RTLIB::LLRINT_F80,
2165 RTLIB::LLRINT_F128,
2166 RTLIB::LLRINT_PPCF128),
2167 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2168 }
2169
2170 //===----------------------------------------------------------------------===//
2171 // Float Operand Promotion
2172 //===----------------------------------------------------------------------===//
2173 //
2174
GetPromotionOpcode(EVT OpVT,EVT RetVT)2175 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2176 if (OpVT == MVT::f16) {
2177 return ISD::FP16_TO_FP;
2178 } else if (RetVT == MVT::f16) {
2179 return ISD::FP_TO_FP16;
2180 } else if (OpVT == MVT::bf16) {
2181 return ISD::BF16_TO_FP;
2182 } else if (RetVT == MVT::bf16) {
2183 return ISD::FP_TO_BF16;
2184 }
2185
2186 report_fatal_error("Attempt at an invalid promotion-related conversion");
2187 }
2188
GetPromotionOpcodeStrict(EVT OpVT,EVT RetVT)2189 static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2190 if (OpVT == MVT::f16)
2191 return ISD::STRICT_FP16_TO_FP;
2192
2193 if (RetVT == MVT::f16)
2194 return ISD::STRICT_FP_TO_FP16;
2195
2196 if (OpVT == MVT::bf16) {
2197 // TODO: return ISD::STRICT_BF16_TO_FP;
2198 }
2199
2200 if (RetVT == MVT::bf16) {
2201 // TODO: return ISD::STRICT_FP_TO_BF16;
2202 }
2203
2204 report_fatal_error("Attempt at an invalid promotion-related conversion");
2205 }
2206
PromoteFloatOperand(SDNode * N,unsigned OpNo)2207 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2208 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2209 SDValue R = SDValue();
2210
2211 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2212 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2213 return false;
2214 }
2215
2216 // Nodes that use a promotion-requiring floating point operand, but doesn't
2217 // produce a promotion-requiring floating point result, need to be legalized
2218 // to use the promoted float operand. Nodes that produce at least one
2219 // promotion-requiring floating point result have their operands legalized as
2220 // a part of PromoteFloatResult.
2221 // clang-format off
2222 switch (N->getOpcode()) {
2223 default:
2224 #ifndef NDEBUG
2225 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2226 N->dump(&DAG); dbgs() << "\n";
2227 #endif
2228 report_fatal_error("Do not know how to promote this operator's operand!");
2229
2230 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2231 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2232 case ISD::FP_TO_SINT:
2233 case ISD::FP_TO_UINT:
2234 case ISD::LRINT:
2235 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2236 case ISD::FP_TO_SINT_SAT:
2237 case ISD::FP_TO_UINT_SAT:
2238 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2239 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2240 case ISD::STRICT_FP_EXTEND:
2241 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2242 break;
2243 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2244 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2245 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2246 }
2247 // clang-format on
2248
2249 if (R.getNode())
2250 ReplaceValueWith(SDValue(N, 0), R);
2251 return false;
2252 }
2253
PromoteFloatOp_BITCAST(SDNode * N,unsigned OpNo)2254 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2255 SDValue Op = N->getOperand(0);
2256 EVT OpVT = Op->getValueType(0);
2257
2258 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2259 EVT PromotedVT = Promoted->getValueType(0);
2260
2261 // Convert the promoted float value to the desired IVT.
2262 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2263 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2264 IVT, Promoted);
2265 // The final result type might not be an scalar so we need a bitcast. The
2266 // bitcast will be further legalized if needed.
2267 return DAG.getBitcast(N->getValueType(0), Convert);
2268 }
2269
2270 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2271 // PromoteFloatRes_FCOPYSIGN.
PromoteFloatOp_FCOPYSIGN(SDNode * N,unsigned OpNo)2272 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2273 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2274 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2275
2276 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2277 N->getOperand(0), Op1);
2278 }
2279
2280 // Convert the promoted float value to the desired integer type
PromoteFloatOp_UnaryOp(SDNode * N,unsigned OpNo)2281 SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2282 SDValue Op = GetPromotedFloat(N->getOperand(0));
2283 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2284 }
2285
PromoteFloatOp_FP_TO_XINT_SAT(SDNode * N,unsigned OpNo)2286 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2287 unsigned OpNo) {
2288 SDValue Op = GetPromotedFloat(N->getOperand(0));
2289 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2290 N->getOperand(1));
2291 }
2292
PromoteFloatOp_FP_EXTEND(SDNode * N,unsigned OpNo)2293 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2294 SDValue Op = GetPromotedFloat(N->getOperand(0));
2295 EVT VT = N->getValueType(0);
2296
2297 // Desired VT is same as promoted type. Use promoted float directly.
2298 if (VT == Op->getValueType(0))
2299 return Op;
2300
2301 // Else, extend the promoted float value to the desired VT.
2302 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2303 }
2304
PromoteFloatOp_STRICT_FP_EXTEND(SDNode * N,unsigned OpNo)2305 SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2306 unsigned OpNo) {
2307 assert(OpNo == 1 && "Promoting unpromotable operand");
2308
2309 SDValue Op = GetPromotedFloat(N->getOperand(1));
2310 EVT VT = N->getValueType(0);
2311
2312 // Desired VT is same as promoted type. Use promoted float directly.
2313 if (VT == Op->getValueType(0)) {
2314 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2315 return Op;
2316 }
2317
2318 // Else, extend the promoted float value to the desired VT.
2319 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2320 N->getOperand(0), Op);
2321 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2322 return Res;
2323 }
2324
2325 // Promote the float operands used for comparison. The true- and false-
2326 // operands have the same type as the result and are promoted, if needed, by
2327 // PromoteFloatRes_SELECT_CC
PromoteFloatOp_SELECT_CC(SDNode * N,unsigned OpNo)2328 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2329 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2330 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2331
2332 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2333 LHS, RHS, N->getOperand(2), N->getOperand(3),
2334 N->getOperand(4));
2335 }
2336
2337 // Construct a SETCC that compares the promoted values and sets the conditional
2338 // code.
PromoteFloatOp_SETCC(SDNode * N,unsigned OpNo)2339 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2340 EVT VT = N->getValueType(0);
2341 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2342 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2343 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2344
2345 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2346
2347 }
2348
2349 // Lower the promoted Float down to the integer value of same size and construct
2350 // a STORE of the integer value.
PromoteFloatOp_STORE(SDNode * N,unsigned OpNo)2351 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2352 StoreSDNode *ST = cast<StoreSDNode>(N);
2353 SDValue Val = ST->getValue();
2354 SDLoc DL(N);
2355
2356 SDValue Promoted = GetPromotedFloat(Val);
2357 EVT VT = ST->getOperand(1).getValueType();
2358 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2359
2360 SDValue NewVal;
2361 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2362 IVT, Promoted);
2363
2364 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2365 ST->getMemOperand());
2366 }
2367
2368 //===----------------------------------------------------------------------===//
2369 // Float Result Promotion
2370 //===----------------------------------------------------------------------===//
2371
PromoteFloatResult(SDNode * N,unsigned ResNo)2372 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2373 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2374 SDValue R = SDValue();
2375
2376 // See if the target wants to custom expand this node.
2377 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2378 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2379 return;
2380 }
2381
2382 switch (N->getOpcode()) {
2383 // These opcodes cannot appear if promotion of FP16 is done in the backend
2384 // instead of Clang
2385 case ISD::FP16_TO_FP:
2386 case ISD::FP_TO_FP16:
2387 default:
2388 #ifndef NDEBUG
2389 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2390 N->dump(&DAG); dbgs() << "\n";
2391 #endif
2392 report_fatal_error("Do not know how to promote this operator's result!");
2393
2394 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2395 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2396 case ISD::EXTRACT_VECTOR_ELT:
2397 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2398 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2399
2400 // Unary FP Operations
2401 case ISD::FABS:
2402 case ISD::FCBRT:
2403 case ISD::FCEIL:
2404 case ISD::FCOS:
2405 case ISD::FEXP:
2406 case ISD::FEXP2:
2407 case ISD::FEXP10:
2408 case ISD::FFLOOR:
2409 case ISD::FLOG:
2410 case ISD::FLOG2:
2411 case ISD::FLOG10:
2412 case ISD::FNEARBYINT:
2413 case ISD::FNEG:
2414 case ISD::FRINT:
2415 case ISD::FROUND:
2416 case ISD::FROUNDEVEN:
2417 case ISD::FSIN:
2418 case ISD::FSQRT:
2419 case ISD::FTRUNC:
2420 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2421
2422 // Binary FP Operations
2423 case ISD::FADD:
2424 case ISD::FDIV:
2425 case ISD::FMAXIMUM:
2426 case ISD::FMINIMUM:
2427 case ISD::FMAXNUM:
2428 case ISD::FMINNUM:
2429 case ISD::FMUL:
2430 case ISD::FPOW:
2431 case ISD::FREM:
2432 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2433
2434 case ISD::FMA: // FMA is same as FMAD
2435 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2436
2437 case ISD::FPOWI:
2438 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2439 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2440
2441 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2442 case ISD::STRICT_FP_ROUND:
2443 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2444 break;
2445 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2446 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2447 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2448
2449 case ISD::SINT_TO_FP:
2450 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2451 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2452 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2453 case ISD::VECREDUCE_FADD:
2454 case ISD::VECREDUCE_FMUL:
2455 case ISD::VECREDUCE_FMIN:
2456 case ISD::VECREDUCE_FMAX:
2457 case ISD::VECREDUCE_FMAXIMUM:
2458 case ISD::VECREDUCE_FMINIMUM:
2459 R = PromoteFloatRes_VECREDUCE(N);
2460 break;
2461 case ISD::VECREDUCE_SEQ_FADD:
2462 case ISD::VECREDUCE_SEQ_FMUL:
2463 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2464 break;
2465 }
2466
2467 if (R.getNode())
2468 SetPromotedFloat(SDValue(N, ResNo), R);
2469 }
2470
2471 // Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2472 // At this point, it is not possible to determine if the bitcast value is
2473 // eventually stored to memory or promoted to f32 or promoted to a floating
2474 // point at a higher precision. Some of these cases are handled by FP_EXTEND,
2475 // STORE promotion handlers.
PromoteFloatRes_BITCAST(SDNode * N)2476 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2477 EVT VT = N->getValueType(0);
2478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2479 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2480 // bitcast will be legalized further if necessary.
2481 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2482 N->getOperand(0).getValueType().getSizeInBits());
2483 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2484 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2485 }
2486
PromoteFloatRes_ConstantFP(SDNode * N)2487 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2488 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2489 EVT VT = N->getValueType(0);
2490 SDLoc DL(N);
2491
2492 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2493 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2494 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2495 IVT);
2496
2497 // Convert the Constant to the desired FP type
2498 // FIXME We might be able to do the conversion during compilation and get rid
2499 // of it from the object code
2500 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2501 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2502 }
2503
2504 // If the Index operand is a constant, try to redirect the extract operation to
2505 // the correct legalized vector. If not, bit-convert the input vector to
2506 // equivalent integer vector. Extract the element as an (bit-cast) integer
2507 // value and convert it to the promoted type.
PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode * N)2508 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2509 SDLoc DL(N);
2510
2511 // If the index is constant, try to extract the value from the legalized
2512 // vector type.
2513 if (isa<ConstantSDNode>(N->getOperand(1))) {
2514 SDValue Vec = N->getOperand(0);
2515 SDValue Idx = N->getOperand(1);
2516 EVT VecVT = Vec->getValueType(0);
2517 EVT EltVT = VecVT.getVectorElementType();
2518
2519 uint64_t IdxVal = Idx->getAsZExtVal();
2520
2521 switch (getTypeAction(VecVT)) {
2522 default: break;
2523 case TargetLowering::TypeScalarizeVector: {
2524 SDValue Res = GetScalarizedVector(N->getOperand(0));
2525 ReplaceValueWith(SDValue(N, 0), Res);
2526 return SDValue();
2527 }
2528 case TargetLowering::TypeWidenVector: {
2529 Vec = GetWidenedVector(Vec);
2530 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2531 ReplaceValueWith(SDValue(N, 0), Res);
2532 return SDValue();
2533 }
2534 case TargetLowering::TypeSplitVector: {
2535 SDValue Lo, Hi;
2536 GetSplitVector(Vec, Lo, Hi);
2537
2538 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2539 SDValue Res;
2540 if (IdxVal < LoElts)
2541 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2542 else
2543 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2544 DAG.getConstant(IdxVal - LoElts, DL,
2545 Idx.getValueType()));
2546 ReplaceValueWith(SDValue(N, 0), Res);
2547 return SDValue();
2548 }
2549
2550 }
2551 }
2552
2553 // Bit-convert the input vector to the equivalent integer vector
2554 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2555 EVT IVT = NewOp.getValueType().getVectorElementType();
2556
2557 // Extract the element as an (bit-cast) integer value
2558 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2559 NewOp, N->getOperand(1));
2560
2561 // Convert the element to the desired FP type
2562 EVT VT = N->getValueType(0);
2563 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2564 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2565 }
2566
2567 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2568 // needs promotion, so does the argument X. Note that Y, if needed, will be
2569 // handled during operand promotion.
PromoteFloatRes_FCOPYSIGN(SDNode * N)2570 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2571 EVT VT = N->getValueType(0);
2572 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2573 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2574
2575 SDValue Op1 = N->getOperand(1);
2576
2577 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2578 }
2579
2580 // Unary operation where the result and the operand have PromoteFloat type
2581 // action. Construct a new SDNode with the promoted float value of the old
2582 // operand.
PromoteFloatRes_UnaryOp(SDNode * N)2583 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2584 EVT VT = N->getValueType(0);
2585 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2586 SDValue Op = GetPromotedFloat(N->getOperand(0));
2587
2588 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2589 }
2590
2591 // Binary operations where the result and both operands have PromoteFloat type
2592 // action. Construct a new SDNode with the promoted float values of the old
2593 // operands.
PromoteFloatRes_BinOp(SDNode * N)2594 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2595 EVT VT = N->getValueType(0);
2596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2597 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2598 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2599 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2600 }
2601
PromoteFloatRes_FMAD(SDNode * N)2602 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2603 EVT VT = N->getValueType(0);
2604 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2605 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2606 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2607 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2608
2609 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2610 }
2611
2612 // Promote the Float (first) operand and retain the Integer (second) operand
PromoteFloatRes_ExpOp(SDNode * N)2613 SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2614 EVT VT = N->getValueType(0);
2615 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2616 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2617 SDValue Op1 = N->getOperand(1);
2618
2619 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2620 }
2621
PromoteFloatRes_FFREXP(SDNode * N)2622 SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2623 EVT VT = N->getValueType(0);
2624 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2625 SDValue Op = GetPromotedFloat(N->getOperand(0));
2626 SDValue Res =
2627 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2628
2629 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2630 return Res;
2631 }
2632
2633 // Explicit operation to reduce precision. Reduce the value to half precision
2634 // and promote it back to the legal type.
PromoteFloatRes_FP_ROUND(SDNode * N)2635 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2636 SDLoc DL(N);
2637
2638 SDValue Op = N->getOperand(0);
2639 EVT VT = N->getValueType(0);
2640 EVT OpVT = Op->getValueType(0);
2641 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2642 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2643
2644 // Round promoted float to desired precision
2645 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2646 // Promote it back to the legal output type
2647 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2648 }
2649
2650 // Explicit operation to reduce precision. Reduce the value to half precision
2651 // and promote it back to the legal type.
PromoteFloatRes_STRICT_FP_ROUND(SDNode * N)2652 SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2653 SDLoc DL(N);
2654
2655 SDValue Chain = N->getOperand(0);
2656 SDValue Op = N->getOperand(1);
2657 EVT VT = N->getValueType(0);
2658 EVT OpVT = Op->getValueType(0);
2659 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2660 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2661
2662 // Round promoted float to desired precision
2663 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2664 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2665 // Promote it back to the legal output type
2666 SDValue Res =
2667 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2668 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2669 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2670 return Res;
2671 }
2672
PromoteFloatRes_LOAD(SDNode * N)2673 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2674 LoadSDNode *L = cast<LoadSDNode>(N);
2675 EVT VT = N->getValueType(0);
2676
2677 // Load the value as an integer value with the same number of bits.
2678 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2679 SDValue newL = DAG.getLoad(
2680 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2681 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2682 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2683 // Legalize the chain result by replacing uses of the old value chain with the
2684 // new one
2685 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2686
2687 // Convert the integer value to the desired FP type
2688 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2689 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2690 }
2691
2692 // Construct a new SELECT node with the promoted true- and false- values.
PromoteFloatRes_SELECT(SDNode * N)2693 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2694 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2695 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2696
2697 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2698 N->getOperand(0), TrueVal, FalseVal);
2699 }
2700
2701 // Construct a new SELECT_CC node with the promoted true- and false- values.
2702 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
PromoteFloatRes_SELECT_CC(SDNode * N)2703 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2704 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2705 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2706
2707 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2708 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2709 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2710 }
2711
2712 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
2713 // float type.
PromoteFloatRes_XINT_TO_FP(SDNode * N)2714 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2715 SDLoc DL(N);
2716 EVT VT = N->getValueType(0);
2717 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2718 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2719 // Round the value to the desired precision (that of the source type).
2720 return DAG.getNode(
2721 ISD::FP_EXTEND, DL, NVT,
2722 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2723 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2724 }
2725
PromoteFloatRes_UNDEF(SDNode * N)2726 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2727 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2728 N->getValueType(0)));
2729 }
2730
PromoteFloatRes_VECREDUCE(SDNode * N)2731 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2732 // Expand and promote recursively.
2733 // TODO: This is non-optimal, but dealing with the concurrently happening
2734 // vector-legalization is non-trivial. We could do something similar to
2735 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2736 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2737 return SDValue();
2738 }
2739
PromoteFloatRes_VECREDUCE_SEQ(SDNode * N)2740 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2741 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2742 return SDValue();
2743 }
2744
BitcastToInt_ATOMIC_SWAP(SDNode * N)2745 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2746 EVT VT = N->getValueType(0);
2747
2748 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2749 SDLoc SL(N);
2750
2751 SDValue CastVal = BitConvertToInteger(AM->getVal());
2752 EVT CastVT = CastVal.getValueType();
2753
2754 SDValue NewAtomic
2755 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2756 DAG.getVTList(CastVT, MVT::Other),
2757 { AM->getChain(), AM->getBasePtr(), CastVal },
2758 AM->getMemOperand());
2759
2760 SDValue Result = NewAtomic;
2761
2762 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2763 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2764 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2765 NewAtomic);
2766 }
2767
2768 // Legalize the chain result by replacing uses of the old value chain with the
2769 // new one
2770 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2771
2772 return Result;
2773
2774 }
2775
2776 //===----------------------------------------------------------------------===//
2777 // Half Result Soft Promotion
2778 //===----------------------------------------------------------------------===//
2779
SoftPromoteHalfResult(SDNode * N,unsigned ResNo)2780 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2781 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2782 N->dump(&DAG));
2783 SDValue R = SDValue();
2784
2785 // See if the target wants to custom expand this node.
2786 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2787 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2788 return;
2789 }
2790
2791 switch (N->getOpcode()) {
2792 default:
2793 #ifndef NDEBUG
2794 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2795 N->dump(&DAG); dbgs() << "\n";
2796 #endif
2797 report_fatal_error("Do not know how to soft promote this operator's "
2798 "result!");
2799
2800 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2801 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2802 case ISD::EXTRACT_VECTOR_ELT:
2803 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2804 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2805 case ISD::STRICT_FP_ROUND:
2806 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2807
2808 // Unary FP Operations
2809 case ISD::FABS:
2810 case ISD::FCBRT:
2811 case ISD::FCEIL:
2812 case ISD::FCOS:
2813 case ISD::FEXP:
2814 case ISD::FEXP2:
2815 case ISD::FEXP10:
2816 case ISD::FFLOOR:
2817 case ISD::FLOG:
2818 case ISD::FLOG2:
2819 case ISD::FLOG10:
2820 case ISD::FNEARBYINT:
2821 case ISD::FNEG:
2822 case ISD::FREEZE:
2823 case ISD::FRINT:
2824 case ISD::FROUND:
2825 case ISD::FROUNDEVEN:
2826 case ISD::FSIN:
2827 case ISD::FSQRT:
2828 case ISD::FTRUNC:
2829 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2830
2831 // Binary FP Operations
2832 case ISD::FADD:
2833 case ISD::FDIV:
2834 case ISD::FMAXIMUM:
2835 case ISD::FMINIMUM:
2836 case ISD::FMAXNUM:
2837 case ISD::FMINNUM:
2838 case ISD::FMUL:
2839 case ISD::FPOW:
2840 case ISD::FREM:
2841 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2842
2843 case ISD::FMA: // FMA is same as FMAD
2844 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2845
2846 case ISD::FPOWI:
2847 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2848
2849 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2850
2851 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2852 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2853 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2854 case ISD::SINT_TO_FP:
2855 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2856 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2857 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2858 case ISD::VECREDUCE_FADD:
2859 case ISD::VECREDUCE_FMUL:
2860 case ISD::VECREDUCE_FMIN:
2861 case ISD::VECREDUCE_FMAX:
2862 case ISD::VECREDUCE_FMAXIMUM:
2863 case ISD::VECREDUCE_FMINIMUM:
2864 R = SoftPromoteHalfRes_VECREDUCE(N);
2865 break;
2866 case ISD::VECREDUCE_SEQ_FADD:
2867 case ISD::VECREDUCE_SEQ_FMUL:
2868 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2869 break;
2870 }
2871
2872 if (R.getNode())
2873 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2874 }
2875
SoftPromoteHalfRes_BITCAST(SDNode * N)2876 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2877 return BitConvertToInteger(N->getOperand(0));
2878 }
2879
SoftPromoteHalfRes_ConstantFP(SDNode * N)2880 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2881 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2882
2883 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2884 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2885 MVT::i16);
2886 }
2887
SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode * N)2888 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2889 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2890 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2891 NewOp.getValueType().getVectorElementType(), NewOp,
2892 N->getOperand(1));
2893 }
2894
SoftPromoteHalfRes_FCOPYSIGN(SDNode * N)2895 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2896 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2897 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2898 SDLoc dl(N);
2899
2900 EVT LVT = LHS.getValueType();
2901 EVT RVT = RHS.getValueType();
2902
2903 unsigned LSize = LVT.getSizeInBits();
2904 unsigned RSize = RVT.getSizeInBits();
2905
2906 // First get the sign bit of second operand.
2907 SDValue SignBit = DAG.getNode(
2908 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2909 DAG.getConstant(RSize - 1, dl,
2910 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2911 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2912
2913 // Shift right or sign-extend it if the two operands have different types.
2914 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2915 if (SizeDiff > 0) {
2916 SignBit =
2917 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2918 DAG.getConstant(SizeDiff, dl,
2919 TLI.getShiftAmountTy(SignBit.getValueType(),
2920 DAG.getDataLayout())));
2921 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2922 } else if (SizeDiff < 0) {
2923 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2924 SignBit =
2925 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2926 DAG.getConstant(-SizeDiff, dl,
2927 TLI.getShiftAmountTy(SignBit.getValueType(),
2928 DAG.getDataLayout())));
2929 }
2930
2931 // Clear the sign bit of the first operand.
2932 SDValue Mask = DAG.getNode(
2933 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2934 DAG.getConstant(LSize - 1, dl,
2935 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2936 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2937 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2938
2939 // Or the value with the sign bit.
2940 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2941 }
2942
SoftPromoteHalfRes_FMAD(SDNode * N)2943 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2944 EVT OVT = N->getValueType(0);
2945 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2946 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2947 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2948 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2949 SDLoc dl(N);
2950
2951 // Promote to the larger FP type.
2952 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2953 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2954 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2955 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2956
2957 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2958
2959 // Convert back to FP16 as an integer.
2960 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2961 }
2962
SoftPromoteHalfRes_ExpOp(SDNode * N)2963 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2964 EVT OVT = N->getValueType(0);
2965 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2966 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2967 SDValue Op1 = N->getOperand(1);
2968 SDLoc dl(N);
2969
2970 // Promote to the larger FP type.
2971 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2972
2973 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2974
2975 // Convert back to FP16 as an integer.
2976 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2977 }
2978
SoftPromoteHalfRes_FFREXP(SDNode * N)2979 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2980 EVT OVT = N->getValueType(0);
2981 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2982 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2983 SDLoc dl(N);
2984
2985 // Promote to the larger FP type.
2986 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2987
2988 SDValue Res = DAG.getNode(N->getOpcode(), dl,
2989 DAG.getVTList(NVT, N->getValueType(1)), Op);
2990
2991 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2992
2993 // Convert back to FP16 as an integer.
2994 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2995 }
2996
SoftPromoteHalfRes_FP_ROUND(SDNode * N)2997 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2998 EVT RVT = N->getValueType(0);
2999 EVT SVT = N->getOperand(0).getValueType();
3000
3001 if (N->isStrictFPOpcode()) {
3002 assert(RVT == MVT::f16);
3003 SDValue Res =
3004 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
3005 {N->getOperand(0), N->getOperand(1)});
3006 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3007 return Res;
3008 }
3009
3010 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3011 N->getOperand(0));
3012 }
3013
SoftPromoteHalfRes_LOAD(SDNode * N)3014 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3015 LoadSDNode *L = cast<LoadSDNode>(N);
3016
3017 // Load the value as an integer value with the same number of bits.
3018 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3019 SDValue NewL =
3020 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3021 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3022 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3023 L->getMemOperand()->getFlags(), L->getAAInfo());
3024 // Legalize the chain result by replacing uses of the old value chain with the
3025 // new one
3026 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3027 return NewL;
3028 }
3029
SoftPromoteHalfRes_SELECT(SDNode * N)3030 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3031 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3032 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3033 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3034 Op2);
3035 }
3036
SoftPromoteHalfRes_SELECT_CC(SDNode * N)3037 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3038 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3039 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3040 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3041 N->getOperand(0), N->getOperand(1), Op2, Op3,
3042 N->getOperand(4));
3043 }
3044
SoftPromoteHalfRes_XINT_TO_FP(SDNode * N)3045 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3046 EVT OVT = N->getValueType(0);
3047 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3048 SDLoc dl(N);
3049
3050 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3051
3052 // Round the value to the softened type.
3053 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3054 }
3055
SoftPromoteHalfRes_UNDEF(SDNode * N)3056 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3057 return DAG.getUNDEF(MVT::i16);
3058 }
3059
SoftPromoteHalfRes_UnaryOp(SDNode * N)3060 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3061 EVT OVT = N->getValueType(0);
3062 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3063 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3064 SDLoc dl(N);
3065
3066 // Promote to the larger FP type.
3067 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3068
3069 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3070
3071 // Convert back to FP16 as an integer.
3072 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3073 }
3074
SoftPromoteHalfRes_BinOp(SDNode * N)3075 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3076 EVT OVT = N->getValueType(0);
3077 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3078 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3079 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3080 SDLoc dl(N);
3081
3082 // Promote to the larger FP type.
3083 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3084 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3085 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3086
3087 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3088
3089 // Convert back to FP16 as an integer.
3090 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3091 }
3092
SoftPromoteHalfRes_VECREDUCE(SDNode * N)3093 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3094 // Expand and soften recursively.
3095 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3096 return SDValue();
3097 }
3098
SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode * N)3099 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3100 // Expand and soften.
3101 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3102 return SDValue();
3103 }
3104
3105 //===----------------------------------------------------------------------===//
3106 // Half Operand Soft Promotion
3107 //===----------------------------------------------------------------------===//
3108
SoftPromoteHalfOperand(SDNode * N,unsigned OpNo)3109 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3110 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3111 N->dump(&DAG));
3112 SDValue Res = SDValue();
3113
3114 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3115 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3116 return false;
3117 }
3118
3119 // Nodes that use a promotion-requiring floating point operand, but doesn't
3120 // produce a soft promotion-requiring floating point result, need to be
3121 // legalized to use the soft promoted float operand. Nodes that produce at
3122 // least one soft promotion-requiring floating point result have their
3123 // operands legalized as a part of PromoteFloatResult.
3124 switch (N->getOpcode()) {
3125 default:
3126 #ifndef NDEBUG
3127 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3128 N->dump(&DAG); dbgs() << "\n";
3129 #endif
3130 report_fatal_error("Do not know how to soft promote this operator's "
3131 "operand!");
3132
3133 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3134 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3135 case ISD::FP_TO_SINT:
3136 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3137 case ISD::FP_TO_SINT_SAT:
3138 case ISD::FP_TO_UINT_SAT:
3139 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3140 case ISD::STRICT_FP_EXTEND:
3141 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3142 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3143 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3144 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3145 case ISD::STACKMAP:
3146 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3147 break;
3148 case ISD::PATCHPOINT:
3149 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3150 break;
3151 }
3152
3153 if (!Res.getNode())
3154 return false;
3155
3156 assert(Res.getNode() != N && "Expected a new node!");
3157
3158 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3159 "Invalid operand expansion");
3160
3161 ReplaceValueWith(SDValue(N, 0), Res);
3162 return false;
3163 }
3164
SoftPromoteHalfOp_BITCAST(SDNode * N)3165 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3166 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3167
3168 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3169 }
3170
SoftPromoteHalfOp_FCOPYSIGN(SDNode * N,unsigned OpNo)3171 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3172 unsigned OpNo) {
3173 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3174 SDValue Op1 = N->getOperand(1);
3175 EVT RVT = Op1.getValueType();
3176 SDLoc dl(N);
3177
3178 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3179
3180 Op1 = GetSoftPromotedHalf(Op1);
3181 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3182
3183 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3184 Op1);
3185 }
3186
SoftPromoteHalfOp_FP_EXTEND(SDNode * N)3187 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3188 EVT RVT = N->getValueType(0);
3189 bool IsStrict = N->isStrictFPOpcode();
3190 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3191 EVT SVT = Op.getValueType();
3192 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3193
3194 if (IsStrict) {
3195 assert(SVT == MVT::f16);
3196 SDValue Res =
3197 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
3198 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
3199 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3200 ReplaceValueWith(SDValue(N, 0), Res);
3201 return SDValue();
3202 }
3203
3204 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3205 }
3206
SoftPromoteHalfOp_FP_TO_XINT(SDNode * N)3207 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3208 EVT RVT = N->getValueType(0);
3209 SDValue Op = N->getOperand(0);
3210 EVT SVT = Op.getValueType();
3211 SDLoc dl(N);
3212
3213 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3214
3215 Op = GetSoftPromotedHalf(Op);
3216
3217 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3218
3219 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3220 }
3221
SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode * N)3222 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3223 EVT RVT = N->getValueType(0);
3224 SDValue Op = N->getOperand(0);
3225 EVT SVT = Op.getValueType();
3226 SDLoc dl(N);
3227
3228 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3229
3230 Op = GetSoftPromotedHalf(Op);
3231
3232 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3233
3234 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3235 N->getOperand(1));
3236 }
3237
SoftPromoteHalfOp_SELECT_CC(SDNode * N,unsigned OpNo)3238 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3239 unsigned OpNo) {
3240 assert(OpNo == 0 && "Can only soften the comparison values");
3241 SDValue Op0 = N->getOperand(0);
3242 SDValue Op1 = N->getOperand(1);
3243 SDLoc dl(N);
3244
3245 EVT SVT = Op0.getValueType();
3246 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3247
3248 Op0 = GetSoftPromotedHalf(Op0);
3249 Op1 = GetSoftPromotedHalf(Op1);
3250
3251 // Promote to the larger FP type.
3252 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3253 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3254 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3255
3256 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3257 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3258 }
3259
SoftPromoteHalfOp_SETCC(SDNode * N)3260 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3261 SDValue Op0 = N->getOperand(0);
3262 SDValue Op1 = N->getOperand(1);
3263 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3264 SDLoc dl(N);
3265
3266 EVT SVT = Op0.getValueType();
3267 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3268
3269 Op0 = GetSoftPromotedHalf(Op0);
3270 Op1 = GetSoftPromotedHalf(Op1);
3271
3272 // Promote to the larger FP type.
3273 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3274 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3275 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3276
3277 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3278 }
3279
SoftPromoteHalfOp_STORE(SDNode * N,unsigned OpNo)3280 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3281 assert(OpNo == 1 && "Can only soften the stored value!");
3282 StoreSDNode *ST = cast<StoreSDNode>(N);
3283 SDValue Val = ST->getValue();
3284 SDLoc dl(N);
3285
3286 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3287 SDValue Promoted = GetSoftPromotedHalf(Val);
3288 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3289 ST->getMemOperand());
3290 }
3291
SoftPromoteHalfOp_STACKMAP(SDNode * N,unsigned OpNo)3292 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3293 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3294 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3295 SDValue Op = N->getOperand(OpNo);
3296 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3297 SDValue NewNode =
3298 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3299
3300 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3301 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3302
3303 return SDValue(); // Signal that we replaced the node ourselves.
3304 }
3305
SoftPromoteHalfOp_PATCHPOINT(SDNode * N,unsigned OpNo)3306 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3307 unsigned OpNo) {
3308 assert(OpNo >= 7);
3309 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3310 SDValue Op = N->getOperand(OpNo);
3311 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3312 SDValue NewNode =
3313 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3314
3315 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3316 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3317
3318 return SDValue(); // Signal that we replaced the node ourselves.
3319 }
3320