1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements float type expansion and softening for LegalizeTypes. 10 // Softening is the act of turning a computation in an illegal floating point 11 // type into a computation in an integer type of the same size; also known as 12 // "soft float". For example, turning f32 arithmetic into operations using i32. 13 // The resulting integer value is the same as what you would get by performing 14 // the floating point operation and bitcasting the result to the integer type. 15 // Expansion is the act of changing a computation in an illegal type to be a 16 // computation in two identical registers of a smaller type. For example, 17 // implementing ppcf128 arithmetic in two f64 registers. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/Analysis/TargetLibraryInfo.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 /// GetFPLibCall - Return the right libcall for the given floating point type. 30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be 31 /// refactored away (see RTLIB::getPOWI for an example). 32 static RTLIB::Libcall GetFPLibCall(EVT VT, 33 RTLIB::Libcall Call_F32, 34 RTLIB::Libcall Call_F64, 35 RTLIB::Libcall Call_F80, 36 RTLIB::Libcall Call_F128, 37 RTLIB::Libcall Call_PPCF128) { 38 return 39 VT == MVT::f32 ? Call_F32 : 40 VT == MVT::f64 ? Call_F64 : 41 VT == MVT::f80 ? Call_F80 : 42 VT == MVT::f128 ? Call_F128 : 43 VT == MVT::ppcf128 ? Call_PPCF128 : 44 RTLIB::UNKNOWN_LIBCALL; 45 } 46 47 //===----------------------------------------------------------------------===// 48 // Convert Float Results to Integer 49 //===----------------------------------------------------------------------===// 50 51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { 52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG)); 53 SDValue R = SDValue(); 54 55 switch (N->getOpcode()) { 56 default: 57 #ifndef NDEBUG 58 dbgs() << "SoftenFloatResult #" << ResNo << ": "; 59 N->dump(&DAG); dbgs() << "\n"; 60 #endif 61 report_fatal_error("Do not know how to soften the result of this " 62 "operator!"); 63 64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break; 65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; 66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; 67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; 68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; 69 case ISD::EXTRACT_VECTOR_ELT: 70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; 71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break; 72 case ISD::STRICT_FMINNUM: 73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; 74 case ISD::STRICT_FMAXNUM: 75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; 76 case ISD::STRICT_FADD: 77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break; 78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; 79 case ISD::STRICT_FCEIL: 80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; 81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; 82 case ISD::STRICT_FCOS: 83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; 84 case ISD::STRICT_FDIV: 85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; 86 case ISD::STRICT_FEXP: 87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; 88 case ISD::STRICT_FEXP2: 89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; 90 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break; 91 case ISD::STRICT_FFLOOR: 92 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; 93 case ISD::STRICT_FLOG: 94 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; 95 case ISD::STRICT_FLOG2: 96 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; 97 case ISD::STRICT_FLOG10: 98 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; 99 case ISD::STRICT_FMA: 100 case ISD::FMA: R = SoftenFloatRes_FMA(N); break; 101 case ISD::STRICT_FMUL: 102 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; 103 case ISD::STRICT_FNEARBYINT: 104 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; 105 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; 106 case ISD::STRICT_FP_EXTEND: 107 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; 108 case ISD::STRICT_FP_ROUND: 109 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; 110 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; 111 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break; 112 case ISD::STRICT_FPOW: 113 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; 114 case ISD::STRICT_FPOWI: 115 case ISD::FPOWI: 116 case ISD::FLDEXP: 117 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break; 118 case ISD::FFREXP: 119 R = SoftenFloatRes_FFREXP(N); 120 break; 121 case ISD::STRICT_FREM: 122 case ISD::FREM: R = SoftenFloatRes_FREM(N); break; 123 case ISD::STRICT_FRINT: 124 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; 125 case ISD::STRICT_FROUND: 126 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; 127 case ISD::STRICT_FROUNDEVEN: 128 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; 129 case ISD::STRICT_FSIN: 130 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; 131 case ISD::STRICT_FSQRT: 132 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; 133 case ISD::STRICT_FSUB: 134 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; 135 case ISD::STRICT_FTRUNC: 136 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; 137 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; 138 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 139 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; 140 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; 141 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; 142 case ISD::STRICT_SINT_TO_FP: 143 case ISD::STRICT_UINT_TO_FP: 144 case ISD::SINT_TO_FP: 145 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; 146 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; 147 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; 148 case ISD::VECREDUCE_FADD: 149 case ISD::VECREDUCE_FMUL: 150 case ISD::VECREDUCE_FMIN: 151 case ISD::VECREDUCE_FMAX: 152 case ISD::VECREDUCE_FMAXIMUM: 153 case ISD::VECREDUCE_FMINIMUM: 154 R = SoftenFloatRes_VECREDUCE(N); 155 break; 156 case ISD::VECREDUCE_SEQ_FADD: 157 case ISD::VECREDUCE_SEQ_FMUL: 158 R = SoftenFloatRes_VECREDUCE_SEQ(N); 159 break; 160 } 161 162 // If R is null, the sub-method took care of registering the result. 163 if (R.getNode()) { 164 assert(R.getNode() != N); 165 SetSoftenedFloat(SDValue(N, ResNo), R); 166 } 167 } 168 169 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { 170 bool IsStrict = N->isStrictFPOpcode(); 171 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 172 unsigned Offset = IsStrict ? 1 : 0; 173 assert(N->getNumOperands() == (1 + Offset) && 174 "Unexpected number of operands!"); 175 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 176 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 177 TargetLowering::MakeLibCallOptions CallOptions; 178 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 179 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 180 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 181 CallOptions, SDLoc(N), 182 Chain); 183 if (IsStrict) 184 ReplaceValueWith(SDValue(N, 1), Tmp.second); 185 return Tmp.first; 186 } 187 188 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { 189 bool IsStrict = N->isStrictFPOpcode(); 190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 191 unsigned Offset = IsStrict ? 1 : 0; 192 assert(N->getNumOperands() == (2 + Offset) && 193 "Unexpected number of operands!"); 194 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 195 GetSoftenedFloat(N->getOperand(1 + Offset)) }; 196 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 197 TargetLowering::MakeLibCallOptions CallOptions; 198 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 199 N->getOperand(1 + Offset).getValueType() }; 200 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 202 CallOptions, SDLoc(N), 203 Chain); 204 if (IsStrict) 205 ReplaceValueWith(SDValue(N, 1), Tmp.second); 206 return Tmp.first; 207 } 208 209 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { 210 return BitConvertToInteger(N->getOperand(0)); 211 } 212 213 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { 214 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 215 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty, 216 GetSoftenedFloat(N->getOperand(0))); 217 } 218 219 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) { 220 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 221 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty, 222 GetSoftenedFloat(N->getOperand(0))); 223 return NewFence; 224 } 225 226 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, 227 unsigned ResNo) { 228 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 229 return BitConvertToInteger(Op); 230 } 231 232 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { 233 // Convert the inputs to integers, and build a new pair out of them. 234 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), 235 TLI.getTypeToTransformTo(*DAG.getContext(), 236 N->getValueType(0)), 237 BitConvertToInteger(N->getOperand(0)), 238 BitConvertToInteger(N->getOperand(1))); 239 } 240 241 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { 242 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 243 // In ppcf128, the high 64 bits are always first in memory regardless 244 // of Endianness. LLVM's APFloat representation is not Endian sensitive, 245 // and so always converts into a 128-bit APInt in a non-Endian-sensitive 246 // way. However, APInt's are serialized in an Endian-sensitive fashion, 247 // so on big-Endian targets, the two doubles are output in the wrong 248 // order. Fix this by manually flipping the order of the high 64 bits 249 // and the low 64 bits here. 250 if (DAG.getDataLayout().isBigEndian() && 251 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) { 252 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], 253 CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; 254 APInt Val(128, words); 255 return DAG.getConstant(Val, SDLoc(CN), 256 TLI.getTypeToTransformTo(*DAG.getContext(), 257 CN->getValueType(0))); 258 } else { 259 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 260 TLI.getTypeToTransformTo(*DAG.getContext(), 261 CN->getValueType(0))); 262 } 263 } 264 265 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) { 266 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 267 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 268 NewOp.getValueType().getVectorElementType(), 269 NewOp, N->getOperand(1)); 270 } 271 272 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { 273 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 274 unsigned Size = NVT.getSizeInBits(); 275 276 // Mask = ~(1 << (Size-1)) 277 APInt API = APInt::getAllOnes(Size); 278 API.clearBit(Size - 1); 279 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT); 280 SDValue Op = GetSoftenedFloat(N->getOperand(0)); 281 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask); 282 } 283 284 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { 285 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 286 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 287 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 288 RTLIB::FMIN_F32, 289 RTLIB::FMIN_F64, 290 RTLIB::FMIN_F80, 291 RTLIB::FMIN_F128, 292 RTLIB::FMIN_PPCF128)); 293 } 294 295 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { 296 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 297 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 299 RTLIB::FMAX_F32, 300 RTLIB::FMAX_F64, 301 RTLIB::FMAX_F80, 302 RTLIB::FMAX_F128, 303 RTLIB::FMAX_PPCF128)); 304 } 305 306 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { 307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 308 RTLIB::ADD_F32, 309 RTLIB::ADD_F64, 310 RTLIB::ADD_F80, 311 RTLIB::ADD_F128, 312 RTLIB::ADD_PPCF128)); 313 } 314 315 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { 316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 317 RTLIB::CBRT_F32, 318 RTLIB::CBRT_F64, 319 RTLIB::CBRT_F80, 320 RTLIB::CBRT_F128, 321 RTLIB::CBRT_PPCF128)); 322 } 323 324 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { 325 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 326 RTLIB::CEIL_F32, 327 RTLIB::CEIL_F64, 328 RTLIB::CEIL_F80, 329 RTLIB::CEIL_F128, 330 RTLIB::CEIL_PPCF128)); 331 } 332 333 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { 334 SDValue LHS = GetSoftenedFloat(N->getOperand(0)); 335 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 336 SDLoc dl(N); 337 338 EVT LVT = LHS.getValueType(); 339 EVT RVT = RHS.getValueType(); 340 341 unsigned LSize = LVT.getSizeInBits(); 342 unsigned RSize = RVT.getSizeInBits(); 343 344 // First get the sign bit of second operand. 345 SDValue SignBit = DAG.getNode( 346 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 347 DAG.getConstant(RSize - 1, dl, 348 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 349 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 350 351 // Shift right or sign-extend it if the two operands have different types. 352 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 353 if (SizeDiff > 0) { 354 SignBit = 355 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 356 DAG.getConstant(SizeDiff, dl, 357 TLI.getShiftAmountTy(SignBit.getValueType(), 358 DAG.getDataLayout()))); 359 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 360 } else if (SizeDiff < 0) { 361 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 362 SignBit = 363 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 364 DAG.getConstant(-SizeDiff, dl, 365 TLI.getShiftAmountTy(SignBit.getValueType(), 366 DAG.getDataLayout()))); 367 } 368 369 // Clear the sign bit of the first operand. 370 SDValue Mask = DAG.getNode( 371 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 372 DAG.getConstant(LSize - 1, dl, 373 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 374 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 375 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 376 377 // Or the value with the sign bit. 378 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 379 } 380 381 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { 382 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 383 RTLIB::COS_F32, 384 RTLIB::COS_F64, 385 RTLIB::COS_F80, 386 RTLIB::COS_F128, 387 RTLIB::COS_PPCF128)); 388 } 389 390 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { 391 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 392 RTLIB::DIV_F32, 393 RTLIB::DIV_F64, 394 RTLIB::DIV_F80, 395 RTLIB::DIV_F128, 396 RTLIB::DIV_PPCF128)); 397 } 398 399 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { 400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 401 RTLIB::EXP_F32, 402 RTLIB::EXP_F64, 403 RTLIB::EXP_F80, 404 RTLIB::EXP_F128, 405 RTLIB::EXP_PPCF128)); 406 } 407 408 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { 409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 410 RTLIB::EXP2_F32, 411 RTLIB::EXP2_F64, 412 RTLIB::EXP2_F80, 413 RTLIB::EXP2_F128, 414 RTLIB::EXP2_PPCF128)); 415 } 416 417 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) { 418 return SoftenFloatRes_Unary( 419 N, 420 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64, 421 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128)); 422 } 423 424 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { 425 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 426 RTLIB::FLOOR_F32, 427 RTLIB::FLOOR_F64, 428 RTLIB::FLOOR_F80, 429 RTLIB::FLOOR_F128, 430 RTLIB::FLOOR_PPCF128)); 431 } 432 433 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { 434 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 435 RTLIB::LOG_F32, 436 RTLIB::LOG_F64, 437 RTLIB::LOG_F80, 438 RTLIB::LOG_F128, 439 RTLIB::LOG_PPCF128)); 440 } 441 442 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { 443 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 444 RTLIB::LOG2_F32, 445 RTLIB::LOG2_F64, 446 RTLIB::LOG2_F80, 447 RTLIB::LOG2_F128, 448 RTLIB::LOG2_PPCF128)); 449 } 450 451 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { 452 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 453 RTLIB::LOG10_F32, 454 RTLIB::LOG10_F64, 455 RTLIB::LOG10_F80, 456 RTLIB::LOG10_F128, 457 RTLIB::LOG10_PPCF128)); 458 } 459 460 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { 461 bool IsStrict = N->isStrictFPOpcode(); 462 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 463 unsigned Offset = IsStrict ? 1 : 0; 464 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 465 GetSoftenedFloat(N->getOperand(1 + Offset)), 466 GetSoftenedFloat(N->getOperand(2 + Offset)) }; 467 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 468 TargetLowering::MakeLibCallOptions CallOptions; 469 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(), 470 N->getOperand(1 + Offset).getValueType(), 471 N->getOperand(2 + Offset).getValueType() }; 472 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 473 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, 474 GetFPLibCall(N->getValueType(0), 475 RTLIB::FMA_F32, 476 RTLIB::FMA_F64, 477 RTLIB::FMA_F80, 478 RTLIB::FMA_F128, 479 RTLIB::FMA_PPCF128), 480 NVT, Ops, CallOptions, SDLoc(N), Chain); 481 if (IsStrict) 482 ReplaceValueWith(SDValue(N, 1), Tmp.second); 483 return Tmp.first; 484 } 485 486 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { 487 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 488 RTLIB::MUL_F32, 489 RTLIB::MUL_F64, 490 RTLIB::MUL_F80, 491 RTLIB::MUL_F128, 492 RTLIB::MUL_PPCF128)); 493 } 494 495 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { 496 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 497 RTLIB::NEARBYINT_F32, 498 RTLIB::NEARBYINT_F64, 499 RTLIB::NEARBYINT_F80, 500 RTLIB::NEARBYINT_F128, 501 RTLIB::NEARBYINT_PPCF128)); 502 } 503 504 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { 505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 506 SDLoc dl(N); 507 508 // Expand Y = FNEG(X) -> Y = X ^ sign mask 509 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); 510 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), 511 DAG.getConstant(SignMask, dl, NVT)); 512 } 513 514 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { 515 bool IsStrict = N->isStrictFPOpcode(); 516 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 517 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 518 519 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 520 521 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { 522 Op = GetPromotedFloat(Op); 523 // If the promotion did the FP_EXTEND to the destination type for us, 524 // there's nothing left to do here. 525 if (Op.getValueType() == N->getValueType(0)) { 526 if (IsStrict) 527 ReplaceValueWith(SDValue(N, 1), Chain); 528 return BitConvertToInteger(Op); 529 } 530 } 531 532 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16 533 // -> f32, so proceed in two stages. Also, it's entirely possible for both 534 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather 535 // than FP16_TO_FP. 536 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) && 537 N->getValueType(0) != MVT::f32) { 538 if (IsStrict) { 539 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 540 { MVT::f32, MVT::Other }, { Chain, Op }); 541 Chain = Op.getValue(1); 542 } else { 543 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); 544 } 545 } 546 547 if (Op.getValueType() == MVT::bf16) { 548 // FIXME: Need ReplaceValueWith on chain in strict case 549 return SoftenFloatRes_BF16_TO_FP(N); 550 } 551 552 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); 553 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 554 TargetLowering::MakeLibCallOptions CallOptions; 555 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 556 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 557 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 558 CallOptions, SDLoc(N), 559 Chain); 560 if (IsStrict) 561 ReplaceValueWith(SDValue(N, 1), Tmp.second); 562 return Tmp.first; 563 } 564 565 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 566 // nodes? 567 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { 568 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 569 SDValue Op = N->getOperand(0); 570 TargetLowering::MakeLibCallOptions CallOptions; 571 EVT OpsVT[1] = { N->getOperand(0).getValueType() }; 572 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 573 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op, 574 CallOptions, SDLoc(N)).first; 575 if (N->getValueType(0) == MVT::f32) 576 return Res32; 577 578 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 579 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0)); 580 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 581 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first; 582 } 583 584 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 585 // nodes? 586 SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) { 587 assert(N->getValueType(0) == MVT::f32 && 588 "Can only soften BF16_TO_FP with f32 result"); 589 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 590 SDValue Op = N->getOperand(0); 591 SDLoc DL(N); 592 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT, 593 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op)); 594 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op, 595 DAG.getShiftAmountConstant(16, NVT, DL)); 596 return Res; 597 } 598 599 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { 600 bool IsStrict = N->isStrictFPOpcode(); 601 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 602 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 603 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 604 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); 605 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 606 TargetLowering::MakeLibCallOptions CallOptions; 607 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 608 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 609 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 610 CallOptions, SDLoc(N), 611 Chain); 612 if (IsStrict) 613 ReplaceValueWith(SDValue(N, 1), Tmp.second); 614 return Tmp.first; 615 } 616 617 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { 618 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 619 RTLIB::POW_F32, 620 RTLIB::POW_F64, 621 RTLIB::POW_F80, 622 RTLIB::POW_F128, 623 RTLIB::POW_PPCF128)); 624 } 625 626 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) { 627 bool IsStrict = N->isStrictFPOpcode(); 628 unsigned Offset = IsStrict ? 1 : 0; 629 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 || 630 N->getOperand(1 + Offset).getValueType() == MVT::i32) && 631 "Unsupported power type!"); 632 bool IsPowI = 633 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; 634 635 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0)) 636 : RTLIB::getLDEXP(N->getValueType(0)); 637 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 638 if (!TLI.getLibcallName(LC)) { 639 // Some targets don't have a powi libcall; use pow instead. 640 // FIXME: Implement this if some target needs it. 641 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow"); 642 return DAG.getUNDEF(N->getValueType(0)); 643 } 644 645 if (DAG.getLibInfo().getIntSize() != 646 N->getOperand(1 + Offset).getValueType().getSizeInBits()) { 647 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI 648 // would use the wrong type for the argument. 649 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); 650 return DAG.getUNDEF(N->getValueType(0)); 651 } 652 653 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 654 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 655 N->getOperand(1 + Offset) }; 656 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 657 TargetLowering::MakeLibCallOptions CallOptions; 658 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 659 N->getOperand(1 + Offset).getValueType() }; 660 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 661 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 662 CallOptions, SDLoc(N), 663 Chain); 664 if (IsStrict) 665 ReplaceValueWith(SDValue(N, 1), Tmp.second); 666 return Tmp.first; 667 } 668 669 SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) { 670 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp"); 671 EVT VT0 = N->getValueType(0); 672 EVT VT1 = N->getValueType(1); 673 RTLIB::Libcall LC = RTLIB::getFREXP(VT0); 674 675 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) { 676 // If the exponent does not match with sizeof(int) a libcall would use the 677 // wrong type for the argument. 678 // TODO: Should be able to handle mismatches. 679 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)"); 680 return DAG.getUNDEF(N->getValueType(0)); 681 } 682 683 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0); 684 SDValue StackSlot = DAG.CreateStackTemporary(VT1); 685 686 SDLoc DL(N); 687 688 TargetLowering::MakeLibCallOptions CallOptions; 689 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot}; 690 EVT OpsVT[2] = {VT0, StackSlot.getValueType()}; 691 692 // TODO: setTypeListBeforeSoften can't properly express multiple return types, 693 // but we only really need to handle the 0th one for softening anyway. 694 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true); 695 696 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL, 697 /*Chain=*/SDValue()); 698 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex(); 699 auto PtrInfo = 700 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 701 702 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo); 703 704 ReplaceValueWith(SDValue(N, 1), LoadExp); 705 return ReturnVal; 706 } 707 708 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { 709 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 710 RTLIB::REM_F32, 711 RTLIB::REM_F64, 712 RTLIB::REM_F80, 713 RTLIB::REM_F128, 714 RTLIB::REM_PPCF128)); 715 } 716 717 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { 718 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 719 RTLIB::RINT_F32, 720 RTLIB::RINT_F64, 721 RTLIB::RINT_F80, 722 RTLIB::RINT_F128, 723 RTLIB::RINT_PPCF128)); 724 } 725 726 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { 727 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 728 RTLIB::ROUND_F32, 729 RTLIB::ROUND_F64, 730 RTLIB::ROUND_F80, 731 RTLIB::ROUND_F128, 732 RTLIB::ROUND_PPCF128)); 733 } 734 735 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { 736 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 737 RTLIB::ROUNDEVEN_F32, 738 RTLIB::ROUNDEVEN_F64, 739 RTLIB::ROUNDEVEN_F80, 740 RTLIB::ROUNDEVEN_F128, 741 RTLIB::ROUNDEVEN_PPCF128)); 742 } 743 744 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { 745 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 746 RTLIB::SIN_F32, 747 RTLIB::SIN_F64, 748 RTLIB::SIN_F80, 749 RTLIB::SIN_F128, 750 RTLIB::SIN_PPCF128)); 751 } 752 753 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { 754 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 755 RTLIB::SQRT_F32, 756 RTLIB::SQRT_F64, 757 RTLIB::SQRT_F80, 758 RTLIB::SQRT_F128, 759 RTLIB::SQRT_PPCF128)); 760 } 761 762 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { 763 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 764 RTLIB::SUB_F32, 765 RTLIB::SUB_F64, 766 RTLIB::SUB_F80, 767 RTLIB::SUB_F128, 768 RTLIB::SUB_PPCF128)); 769 } 770 771 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { 772 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 773 RTLIB::TRUNC_F32, 774 RTLIB::TRUNC_F64, 775 RTLIB::TRUNC_F80, 776 RTLIB::TRUNC_F128, 777 RTLIB::TRUNC_PPCF128)); 778 } 779 780 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { 781 LoadSDNode *L = cast<LoadSDNode>(N); 782 EVT VT = N->getValueType(0); 783 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 784 SDLoc dl(N); 785 786 auto MMOFlags = 787 L->getMemOperand()->getFlags() & 788 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 789 SDValue NewL; 790 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 791 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, 792 L->getChain(), L->getBasePtr(), L->getOffset(), 793 L->getPointerInfo(), NVT, L->getOriginalAlign(), 794 MMOFlags, L->getAAInfo()); 795 // Legalized the chain result - switch anything that used the old chain to 796 // use the new one. 797 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 798 return NewL; 799 } 800 801 // Do a non-extending load followed by FP_EXTEND. 802 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(), 803 dl, L->getChain(), L->getBasePtr(), L->getOffset(), 804 L->getPointerInfo(), L->getMemoryVT(), 805 L->getOriginalAlign(), MMOFlags, L->getAAInfo()); 806 // Legalized the chain result - switch anything that used the old chain to 807 // use the new one. 808 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 809 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL); 810 return BitConvertToInteger(ExtendNode); 811 } 812 813 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { 814 SDValue LHS = GetSoftenedFloat(N->getOperand(1)); 815 SDValue RHS = GetSoftenedFloat(N->getOperand(2)); 816 return DAG.getSelect(SDLoc(N), 817 LHS.getValueType(), N->getOperand(0), LHS, RHS); 818 } 819 820 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { 821 SDValue LHS = GetSoftenedFloat(N->getOperand(2)); 822 SDValue RHS = GetSoftenedFloat(N->getOperand(3)); 823 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 824 LHS.getValueType(), N->getOperand(0), 825 N->getOperand(1), LHS, RHS, N->getOperand(4)); 826 } 827 828 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { 829 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 830 N->getValueType(0))); 831 } 832 833 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { 834 SDValue Chain = N->getOperand(0); // Get the chain. 835 SDValue Ptr = N->getOperand(1); // Get the pointer. 836 EVT VT = N->getValueType(0); 837 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 838 SDLoc dl(N); 839 840 SDValue NewVAARG; 841 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), 842 N->getConstantOperandVal(3)); 843 844 // Legalized the chain result - switch anything that used the old chain to 845 // use the new one. 846 if (N != NewVAARG.getValue(1).getNode()) 847 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1)); 848 return NewVAARG; 849 } 850 851 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { 852 bool IsStrict = N->isStrictFPOpcode(); 853 bool Signed = N->getOpcode() == ISD::SINT_TO_FP || 854 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 855 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 856 EVT RVT = N->getValueType(0); 857 EVT NVT = EVT(); 858 SDLoc dl(N); 859 860 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to 861 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly 862 // match. Look for an appropriate libcall. 863 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 864 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 865 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { 866 NVT = (MVT::SimpleValueType)t; 867 // The source needs to big enough to hold the operand. 868 if (NVT.bitsGE(SVT)) 869 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT); 870 } 871 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 872 873 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 874 // Sign/zero extend the argument if the libcall takes a larger type. 875 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 876 NVT, N->getOperand(IsStrict ? 1 : 0)); 877 TargetLowering::MakeLibCallOptions CallOptions; 878 CallOptions.setSExt(Signed); 879 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 880 std::pair<SDValue, SDValue> Tmp = 881 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), 882 Op, CallOptions, dl, Chain); 883 884 if (IsStrict) 885 ReplaceValueWith(SDValue(N, 1), Tmp.second); 886 return Tmp.first; 887 } 888 889 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) { 890 // Expand and soften recursively. 891 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 892 return SDValue(); 893 } 894 895 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) { 896 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 897 return SDValue(); 898 } 899 900 //===----------------------------------------------------------------------===// 901 // Convert Float Operand to Integer 902 //===----------------------------------------------------------------------===// 903 904 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { 905 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG)); 906 SDValue Res = SDValue(); 907 908 switch (N->getOpcode()) { 909 default: 910 #ifndef NDEBUG 911 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": "; 912 N->dump(&DAG); dbgs() << "\n"; 913 #endif 914 report_fatal_error("Do not know how to soften this operator's operand!"); 915 916 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; 917 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; 918 case ISD::STRICT_FP_TO_FP16: 919 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes 920 case ISD::FP_TO_BF16: 921 case ISD::STRICT_FP_ROUND: 922 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; 923 case ISD::STRICT_FP_TO_SINT: 924 case ISD::STRICT_FP_TO_UINT: 925 case ISD::FP_TO_SINT: 926 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; 927 case ISD::FP_TO_SINT_SAT: 928 case ISD::FP_TO_UINT_SAT: 929 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; 930 case ISD::STRICT_LROUND: 931 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; 932 case ISD::STRICT_LLROUND: 933 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; 934 case ISD::STRICT_LRINT: 935 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; 936 case ISD::STRICT_LLRINT: 937 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; 938 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; 939 case ISD::STRICT_FSETCC: 940 case ISD::STRICT_FSETCCS: 941 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; 942 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; 943 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; 944 } 945 946 // If the result is null, the sub-method took care of registering results etc. 947 if (!Res.getNode()) return false; 948 949 // If the result is N, the sub-method updated N in place. Tell the legalizer 950 // core about this to re-analyze. 951 if (Res.getNode() == N) 952 return true; 953 954 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 955 "Invalid operand softening"); 956 957 ReplaceValueWith(SDValue(N, 0), Res); 958 return false; 959 } 960 961 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { 962 SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); 963 964 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 965 } 966 967 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { 968 // We actually deal with the partially-softened FP_TO_FP16 node too, which 969 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. 970 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || 971 N->getOpcode() == ISD::STRICT_FP_TO_FP16 || 972 N->getOpcode() == ISD::FP_TO_BF16 || 973 N->getOpcode() == ISD::STRICT_FP_ROUND); 974 975 bool IsStrict = N->isStrictFPOpcode(); 976 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 977 EVT SVT = Op.getValueType(); 978 EVT RVT = N->getValueType(0); 979 EVT FloatRVT = RVT; 980 if (N->getOpcode() == ISD::FP_TO_FP16 || 981 N->getOpcode() == ISD::STRICT_FP_TO_FP16) 982 FloatRVT = MVT::f16; 983 else if (N->getOpcode() == ISD::FP_TO_BF16) 984 FloatRVT = MVT::bf16; 985 986 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT); 987 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 988 989 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 990 Op = GetSoftenedFloat(Op); 991 TargetLowering::MakeLibCallOptions CallOptions; 992 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 993 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 994 CallOptions, SDLoc(N), 995 Chain); 996 if (IsStrict) { 997 ReplaceValueWith(SDValue(N, 1), Tmp.second); 998 ReplaceValueWith(SDValue(N, 0), Tmp.first); 999 return SDValue(); 1000 } 1001 return Tmp.first; 1002 } 1003 1004 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 1005 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1006 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1007 1008 EVT VT = NewLHS.getValueType(); 1009 NewLHS = GetSoftenedFloat(NewLHS); 1010 NewRHS = GetSoftenedFloat(NewRHS); 1011 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1012 N->getOperand(2), N->getOperand(3)); 1013 1014 // If softenSetCCOperands returned a scalar, we need to compare the result 1015 // against zero to select between true and false values. 1016 if (!NewRHS.getNode()) { 1017 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1018 CCCode = ISD::SETNE; 1019 } 1020 1021 // Update N to have the operands specified. 1022 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1023 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1024 N->getOperand(4)), 1025 0); 1026 } 1027 1028 // Even if the result type is legal, no libcall may exactly match. (e.g. We 1029 // don't have FP-i8 conversions) This helper method looks for an appropriate 1030 // promoted libcall. 1031 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, 1032 bool Signed) { 1033 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1034 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 1035 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 1036 ++IntVT) { 1037 Promoted = (MVT::SimpleValueType)IntVT; 1038 // The type needs to big enough to hold the result. 1039 if (Promoted.bitsGE(RetVT)) 1040 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted) 1041 : RTLIB::getFPTOUINT(SrcVT, Promoted); 1042 } 1043 return LC; 1044 } 1045 1046 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { 1047 bool IsStrict = N->isStrictFPOpcode(); 1048 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 1049 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 1050 1051 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1052 EVT SVT = Op.getValueType(); 1053 EVT RVT = N->getValueType(0); 1054 EVT NVT = EVT(); 1055 SDLoc dl(N); 1056 1057 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to 1058 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly 1059 // match, eg. we don't have fp -> i8 conversions. 1060 // Look for an appropriate libcall. 1061 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed); 1062 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 1063 "Unsupported FP_TO_XINT!"); 1064 1065 Op = GetSoftenedFloat(Op); 1066 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1067 TargetLowering::MakeLibCallOptions CallOptions; 1068 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 1069 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1070 CallOptions, dl, Chain); 1071 1072 // Truncate the result if the libcall returns a larger type. 1073 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first); 1074 1075 if (!IsStrict) 1076 return Res; 1077 1078 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1079 ReplaceValueWith(SDValue(N, 0), Res); 1080 return SDValue(); 1081 } 1082 1083 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { 1084 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 1085 return Res; 1086 } 1087 1088 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { 1089 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1090 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1091 1092 EVT VT = NewLHS.getValueType(); 1093 NewLHS = GetSoftenedFloat(NewLHS); 1094 NewRHS = GetSoftenedFloat(NewRHS); 1095 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1096 N->getOperand(0), N->getOperand(1)); 1097 1098 // If softenSetCCOperands returned a scalar, we need to compare the result 1099 // against zero to select between true and false values. 1100 if (!NewRHS.getNode()) { 1101 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1102 CCCode = ISD::SETNE; 1103 } 1104 1105 // Update N to have the operands specified. 1106 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1107 N->getOperand(2), N->getOperand(3), 1108 DAG.getCondCode(CCCode)), 1109 0); 1110 } 1111 1112 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { 1113 bool IsStrict = N->isStrictFPOpcode(); 1114 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); 1115 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1); 1116 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1117 ISD::CondCode CCCode = 1118 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1119 1120 EVT VT = Op0.getValueType(); 1121 SDValue NewLHS = GetSoftenedFloat(Op0); 1122 SDValue NewRHS = GetSoftenedFloat(Op1); 1123 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1, 1124 Chain, N->getOpcode() == ISD::STRICT_FSETCCS); 1125 1126 // Update N to have the operands specified. 1127 if (NewRHS.getNode()) { 1128 if (IsStrict) 1129 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS, 1130 NewRHS, DAG.getCondCode(CCCode)); 1131 else 1132 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1133 DAG.getCondCode(CCCode)), 0); 1134 } 1135 1136 // Otherwise, softenSetCCOperands returned a scalar, use it. 1137 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && 1138 "Unexpected setcc expansion!"); 1139 1140 if (IsStrict) { 1141 ReplaceValueWith(SDValue(N, 0), NewLHS); 1142 ReplaceValueWith(SDValue(N, 1), Chain); 1143 return SDValue(); 1144 } 1145 return NewLHS; 1146 } 1147 1148 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { 1149 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1150 assert(OpNo == 1 && "Can only soften the stored value!"); 1151 StoreSDNode *ST = cast<StoreSDNode>(N); 1152 SDValue Val = ST->getValue(); 1153 SDLoc dl(N); 1154 1155 if (ST->isTruncatingStore()) 1156 // Do an FP_ROUND followed by a non-truncating store. 1157 Val = BitConvertToInteger( 1158 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val, 1159 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); 1160 else 1161 Val = GetSoftenedFloat(Val); 1162 1163 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(), 1164 ST->getMemOperand()); 1165 } 1166 1167 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { 1168 SDValue LHS = N->getOperand(0); 1169 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 1170 SDLoc dl(N); 1171 1172 EVT LVT = LHS.getValueType(); 1173 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits()); 1174 EVT RVT = RHS.getValueType(); 1175 1176 unsigned LSize = LVT.getSizeInBits(); 1177 unsigned RSize = RVT.getSizeInBits(); 1178 1179 // Shift right or sign-extend it if the two operands have different types. 1180 int SizeDiff = RSize - LSize; 1181 if (SizeDiff > 0) { 1182 RHS = 1183 DAG.getNode(ISD::SRL, dl, RVT, RHS, 1184 DAG.getConstant(SizeDiff, dl, 1185 TLI.getShiftAmountTy(RHS.getValueType(), 1186 DAG.getDataLayout()))); 1187 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS); 1188 } else if (SizeDiff < 0) { 1189 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS); 1190 RHS = 1191 DAG.getNode(ISD::SHL, dl, ILVT, RHS, 1192 DAG.getConstant(-SizeDiff, dl, 1193 TLI.getShiftAmountTy(RHS.getValueType(), 1194 DAG.getDataLayout()))); 1195 } 1196 1197 RHS = DAG.getBitcast(LVT, RHS); 1198 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS); 1199 } 1200 1201 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { 1202 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1203 bool IsStrict = N->isStrictFPOpcode(); 1204 unsigned Offset = IsStrict ? 1 : 0; 1205 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 1206 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1207 TargetLowering::MakeLibCallOptions CallOptions; 1208 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 1209 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 1210 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1211 CallOptions, SDLoc(N), 1212 Chain); 1213 if (IsStrict) { 1214 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1215 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1216 return SDValue(); 1217 } 1218 1219 return Tmp.first; 1220 } 1221 1222 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { 1223 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1224 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1225 RTLIB::LROUND_F32, 1226 RTLIB::LROUND_F64, 1227 RTLIB::LROUND_F80, 1228 RTLIB::LROUND_F128, 1229 RTLIB::LROUND_PPCF128)); 1230 } 1231 1232 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { 1233 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1234 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1235 RTLIB::LLROUND_F32, 1236 RTLIB::LLROUND_F64, 1237 RTLIB::LLROUND_F80, 1238 RTLIB::LLROUND_F128, 1239 RTLIB::LLROUND_PPCF128)); 1240 } 1241 1242 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { 1243 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1244 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1245 RTLIB::LRINT_F32, 1246 RTLIB::LRINT_F64, 1247 RTLIB::LRINT_F80, 1248 RTLIB::LRINT_F128, 1249 RTLIB::LRINT_PPCF128)); 1250 } 1251 1252 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { 1253 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1254 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1255 RTLIB::LLRINT_F32, 1256 RTLIB::LLRINT_F64, 1257 RTLIB::LLRINT_F80, 1258 RTLIB::LLRINT_F128, 1259 RTLIB::LLRINT_PPCF128)); 1260 } 1261 1262 //===----------------------------------------------------------------------===// 1263 // Float Result Expansion 1264 //===----------------------------------------------------------------------===// 1265 1266 /// ExpandFloatResult - This method is called when the specified result of the 1267 /// specified node is found to need expansion. At this point, the node may also 1268 /// have invalid operands or may have other results that need promotion, we just 1269 /// know that (at least) one result needs expansion. 1270 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { 1271 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG)); 1272 SDValue Lo, Hi; 1273 Lo = Hi = SDValue(); 1274 1275 // See if the target wants to custom expand this node. 1276 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1277 return; 1278 1279 switch (N->getOpcode()) { 1280 default: 1281 #ifndef NDEBUG 1282 dbgs() << "ExpandFloatResult #" << ResNo << ": "; 1283 N->dump(&DAG); dbgs() << "\n"; 1284 #endif 1285 report_fatal_error("Do not know how to expand the result of this " 1286 "operator!"); 1287 1288 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1289 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 1290 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1291 1292 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1293 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1294 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1295 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1296 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1297 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1298 1299 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; 1300 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; 1301 case ISD::STRICT_FMINNUM: 1302 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; 1303 case ISD::STRICT_FMAXNUM: 1304 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; 1305 case ISD::STRICT_FADD: 1306 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; 1307 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; 1308 case ISD::STRICT_FCEIL: 1309 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; 1310 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; 1311 case ISD::STRICT_FCOS: 1312 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; 1313 case ISD::STRICT_FDIV: 1314 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; 1315 case ISD::STRICT_FEXP: 1316 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; 1317 case ISD::STRICT_FEXP2: 1318 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; 1319 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break; 1320 case ISD::STRICT_FFLOOR: 1321 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; 1322 case ISD::STRICT_FLOG: 1323 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; 1324 case ISD::STRICT_FLOG2: 1325 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; 1326 case ISD::STRICT_FLOG10: 1327 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; 1328 case ISD::STRICT_FMA: 1329 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; 1330 case ISD::STRICT_FMUL: 1331 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; 1332 case ISD::STRICT_FNEARBYINT: 1333 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; 1334 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; 1335 case ISD::STRICT_FP_EXTEND: 1336 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; 1337 case ISD::STRICT_FPOW: 1338 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; 1339 case ISD::STRICT_FPOWI: 1340 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; 1341 case ISD::FLDEXP: 1342 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break; 1343 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; 1344 case ISD::STRICT_FRINT: 1345 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; 1346 case ISD::STRICT_FROUND: 1347 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; 1348 case ISD::STRICT_FROUNDEVEN: 1349 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; 1350 case ISD::STRICT_FSIN: 1351 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; 1352 case ISD::STRICT_FSQRT: 1353 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; 1354 case ISD::STRICT_FSUB: 1355 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; 1356 case ISD::STRICT_FTRUNC: 1357 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; 1358 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; 1359 case ISD::STRICT_SINT_TO_FP: 1360 case ISD::STRICT_UINT_TO_FP: 1361 case ISD::SINT_TO_FP: 1362 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; 1363 case ISD::STRICT_FREM: 1364 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; 1365 } 1366 1367 // If Lo/Hi is null, the sub-method took care of registering results etc. 1368 if (Lo.getNode()) 1369 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi); 1370 } 1371 1372 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, 1373 SDValue &Hi) { 1374 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1375 assert(NVT.getSizeInBits() == 64 && 1376 "Do not know how to expand this float constant!"); 1377 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); 1378 SDLoc dl(N); 1379 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1380 APInt(64, C.getRawData()[1])), 1381 dl, NVT); 1382 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1383 APInt(64, C.getRawData()[0])), 1384 dl, NVT); 1385 } 1386 1387 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, 1388 SDValue &Lo, SDValue &Hi) { 1389 bool IsStrict = N->isStrictFPOpcode(); 1390 unsigned Offset = IsStrict ? 1 : 0; 1391 SDValue Op = N->getOperand(0 + Offset); 1392 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1393 TargetLowering::MakeLibCallOptions CallOptions; 1394 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1395 Op, CallOptions, SDLoc(N), 1396 Chain); 1397 if (IsStrict) 1398 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1399 GetPairElements(Tmp.first, Lo, Hi); 1400 } 1401 1402 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, 1403 SDValue &Lo, SDValue &Hi) { 1404 bool IsStrict = N->isStrictFPOpcode(); 1405 unsigned Offset = IsStrict ? 1 : 0; 1406 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) }; 1407 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1408 TargetLowering::MakeLibCallOptions CallOptions; 1409 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1410 Ops, CallOptions, SDLoc(N), 1411 Chain); 1412 if (IsStrict) 1413 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1414 GetPairElements(Tmp.first, Lo, Hi); 1415 } 1416 1417 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, 1418 SDValue &Hi) { 1419 assert(N->getValueType(0) == MVT::ppcf128 && 1420 "Logic only correct for ppcf128!"); 1421 SDLoc dl(N); 1422 SDValue Tmp; 1423 GetExpandedFloat(N->getOperand(0), Lo, Tmp); 1424 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp); 1425 // Lo = Hi==fabs(Hi) ? Lo : -Lo; 1426 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo, 1427 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo), 1428 ISD::SETEQ); 1429 } 1430 1431 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, 1432 SDValue &Hi) { 1433 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1434 RTLIB::FMIN_F32, RTLIB::FMIN_F64, 1435 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 1436 RTLIB::FMIN_PPCF128), Lo, Hi); 1437 } 1438 1439 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, 1440 SDValue &Hi) { 1441 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1442 RTLIB::FMAX_F32, RTLIB::FMAX_F64, 1443 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 1444 RTLIB::FMAX_PPCF128), Lo, Hi); 1445 } 1446 1447 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, 1448 SDValue &Hi) { 1449 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1450 RTLIB::ADD_F32, RTLIB::ADD_F64, 1451 RTLIB::ADD_F80, RTLIB::ADD_F128, 1452 RTLIB::ADD_PPCF128), Lo, Hi); 1453 } 1454 1455 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, 1456 SDValue &Hi) { 1457 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32, 1458 RTLIB::CBRT_F64, RTLIB::CBRT_F80, 1459 RTLIB::CBRT_F128, 1460 RTLIB::CBRT_PPCF128), Lo, Hi); 1461 } 1462 1463 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, 1464 SDValue &Lo, SDValue &Hi) { 1465 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1466 RTLIB::CEIL_F32, RTLIB::CEIL_F64, 1467 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 1468 RTLIB::CEIL_PPCF128), Lo, Hi); 1469 } 1470 1471 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, 1472 SDValue &Lo, SDValue &Hi) { 1473 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1474 RTLIB::COPYSIGN_F32, 1475 RTLIB::COPYSIGN_F64, 1476 RTLIB::COPYSIGN_F80, 1477 RTLIB::COPYSIGN_F128, 1478 RTLIB::COPYSIGN_PPCF128), Lo, Hi); 1479 } 1480 1481 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, 1482 SDValue &Lo, SDValue &Hi) { 1483 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1484 RTLIB::COS_F32, RTLIB::COS_F64, 1485 RTLIB::COS_F80, RTLIB::COS_F128, 1486 RTLIB::COS_PPCF128), Lo, Hi); 1487 } 1488 1489 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, 1490 SDValue &Hi) { 1491 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1492 RTLIB::DIV_F32, 1493 RTLIB::DIV_F64, 1494 RTLIB::DIV_F80, 1495 RTLIB::DIV_F128, 1496 RTLIB::DIV_PPCF128), Lo, Hi); 1497 } 1498 1499 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, 1500 SDValue &Lo, SDValue &Hi) { 1501 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1502 RTLIB::EXP_F32, RTLIB::EXP_F64, 1503 RTLIB::EXP_F80, RTLIB::EXP_F128, 1504 RTLIB::EXP_PPCF128), Lo, Hi); 1505 } 1506 1507 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, 1508 SDValue &Lo, SDValue &Hi) { 1509 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1510 RTLIB::EXP2_F32, RTLIB::EXP2_F64, 1511 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 1512 RTLIB::EXP2_PPCF128), Lo, Hi); 1513 } 1514 1515 void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo, 1516 SDValue &Hi) { 1517 ExpandFloatRes_Unary(N, 1518 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, 1519 RTLIB::EXP10_F64, RTLIB::EXP10_F80, 1520 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128), 1521 Lo, Hi); 1522 } 1523 1524 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, 1525 SDValue &Lo, SDValue &Hi) { 1526 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1527 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 1528 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 1529 RTLIB::FLOOR_PPCF128), Lo, Hi); 1530 } 1531 1532 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, 1533 SDValue &Lo, SDValue &Hi) { 1534 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1535 RTLIB::LOG_F32, RTLIB::LOG_F64, 1536 RTLIB::LOG_F80, RTLIB::LOG_F128, 1537 RTLIB::LOG_PPCF128), Lo, Hi); 1538 } 1539 1540 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, 1541 SDValue &Lo, SDValue &Hi) { 1542 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1543 RTLIB::LOG2_F32, RTLIB::LOG2_F64, 1544 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 1545 RTLIB::LOG2_PPCF128), Lo, Hi); 1546 } 1547 1548 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, 1549 SDValue &Lo, SDValue &Hi) { 1550 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1551 RTLIB::LOG10_F32, RTLIB::LOG10_F64, 1552 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 1553 RTLIB::LOG10_PPCF128), Lo, Hi); 1554 } 1555 1556 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, 1557 SDValue &Hi) { 1558 bool IsStrict = N->isStrictFPOpcode(); 1559 unsigned Offset = IsStrict ? 1 : 0; 1560 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset), 1561 N->getOperand(2 + Offset) }; 1562 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1563 TargetLowering::MakeLibCallOptions CallOptions; 1564 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), 1565 RTLIB::FMA_F32, 1566 RTLIB::FMA_F64, 1567 RTLIB::FMA_F80, 1568 RTLIB::FMA_F128, 1569 RTLIB::FMA_PPCF128), 1570 N->getValueType(0), Ops, CallOptions, 1571 SDLoc(N), Chain); 1572 if (IsStrict) 1573 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1574 GetPairElements(Tmp.first, Lo, Hi); 1575 } 1576 1577 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, 1578 SDValue &Hi) { 1579 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1580 RTLIB::MUL_F32, 1581 RTLIB::MUL_F64, 1582 RTLIB::MUL_F80, 1583 RTLIB::MUL_F128, 1584 RTLIB::MUL_PPCF128), Lo, Hi); 1585 } 1586 1587 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, 1588 SDValue &Lo, SDValue &Hi) { 1589 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1590 RTLIB::NEARBYINT_F32, 1591 RTLIB::NEARBYINT_F64, 1592 RTLIB::NEARBYINT_F80, 1593 RTLIB::NEARBYINT_F128, 1594 RTLIB::NEARBYINT_PPCF128), Lo, Hi); 1595 } 1596 1597 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, 1598 SDValue &Hi) { 1599 SDLoc dl(N); 1600 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1601 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo); 1602 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); 1603 } 1604 1605 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, 1606 SDValue &Hi) { 1607 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1608 SDLoc dl(N); 1609 bool IsStrict = N->isStrictFPOpcode(); 1610 1611 SDValue Chain; 1612 if (IsStrict) { 1613 // If the expanded type is the same as the input type, just bypass the node. 1614 if (NVT == N->getOperand(1).getValueType()) { 1615 Hi = N->getOperand(1); 1616 Chain = N->getOperand(0); 1617 } else { 1618 // Other we need to extend. 1619 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, 1620 { N->getOperand(0), N->getOperand(1) }); 1621 Chain = Hi.getValue(1); 1622 } 1623 } else { 1624 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0)); 1625 } 1626 1627 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1628 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1629 1630 if (IsStrict) 1631 ReplaceValueWith(SDValue(N, 1), Chain); 1632 } 1633 1634 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, 1635 SDValue &Lo, SDValue &Hi) { 1636 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1637 RTLIB::POW_F32, RTLIB::POW_F64, 1638 RTLIB::POW_F80, RTLIB::POW_F128, 1639 RTLIB::POW_PPCF128), Lo, Hi); 1640 } 1641 1642 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, 1643 SDValue &Lo, SDValue &Hi) { 1644 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi); 1645 } 1646 1647 void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo, 1648 SDValue &Hi) { 1649 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi); 1650 } 1651 1652 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, 1653 SDValue &Lo, SDValue &Hi) { 1654 assert(N->getValueType(0) == MVT::ppcf128 && 1655 "Logic only correct for ppcf128!"); 1656 1657 SDLoc dl(N); 1658 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1659 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo); 1660 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi); 1661 } 1662 1663 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, 1664 SDValue &Lo, SDValue &Hi) { 1665 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1666 RTLIB::REM_F32, RTLIB::REM_F64, 1667 RTLIB::REM_F80, RTLIB::REM_F128, 1668 RTLIB::REM_PPCF128), Lo, Hi); 1669 } 1670 1671 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, 1672 SDValue &Lo, SDValue &Hi) { 1673 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1674 RTLIB::RINT_F32, RTLIB::RINT_F64, 1675 RTLIB::RINT_F80, RTLIB::RINT_F128, 1676 RTLIB::RINT_PPCF128), Lo, Hi); 1677 } 1678 1679 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, 1680 SDValue &Lo, SDValue &Hi) { 1681 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1682 RTLIB::ROUND_F32, 1683 RTLIB::ROUND_F64, 1684 RTLIB::ROUND_F80, 1685 RTLIB::ROUND_F128, 1686 RTLIB::ROUND_PPCF128), Lo, Hi); 1687 } 1688 1689 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, 1690 SDValue &Lo, SDValue &Hi) { 1691 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1692 RTLIB::ROUNDEVEN_F32, 1693 RTLIB::ROUNDEVEN_F64, 1694 RTLIB::ROUNDEVEN_F80, 1695 RTLIB::ROUNDEVEN_F128, 1696 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); 1697 } 1698 1699 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, 1700 SDValue &Lo, SDValue &Hi) { 1701 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1702 RTLIB::SIN_F32, RTLIB::SIN_F64, 1703 RTLIB::SIN_F80, RTLIB::SIN_F128, 1704 RTLIB::SIN_PPCF128), Lo, Hi); 1705 } 1706 1707 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, 1708 SDValue &Lo, SDValue &Hi) { 1709 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1710 RTLIB::SQRT_F32, RTLIB::SQRT_F64, 1711 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 1712 RTLIB::SQRT_PPCF128), Lo, Hi); 1713 } 1714 1715 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, 1716 SDValue &Hi) { 1717 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1718 RTLIB::SUB_F32, 1719 RTLIB::SUB_F64, 1720 RTLIB::SUB_F80, 1721 RTLIB::SUB_F128, 1722 RTLIB::SUB_PPCF128), Lo, Hi); 1723 } 1724 1725 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, 1726 SDValue &Lo, SDValue &Hi) { 1727 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1728 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 1729 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 1730 RTLIB::TRUNC_PPCF128), Lo, Hi); 1731 } 1732 1733 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, 1734 SDValue &Hi) { 1735 if (ISD::isNormalLoad(N)) { 1736 ExpandRes_NormalLoad(N, Lo, Hi); 1737 return; 1738 } 1739 1740 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1741 LoadSDNode *LD = cast<LoadSDNode>(N); 1742 SDValue Chain = LD->getChain(); 1743 SDValue Ptr = LD->getBasePtr(); 1744 SDLoc dl(N); 1745 1746 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1747 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1748 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1749 1750 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr, 1751 LD->getMemoryVT(), LD->getMemOperand()); 1752 1753 // Remember the chain. 1754 Chain = Hi.getValue(1); 1755 1756 // The low part is zero. 1757 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1758 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1759 1760 // Modified the chain - switch anything that used the old chain to use the 1761 // new one. 1762 ReplaceValueWith(SDValue(LD, 1), Chain); 1763 } 1764 1765 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, 1766 SDValue &Hi) { 1767 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!"); 1768 EVT VT = N->getValueType(0); 1769 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1770 bool Strict = N->isStrictFPOpcode(); 1771 SDValue Src = N->getOperand(Strict ? 1 : 0); 1772 EVT SrcVT = Src.getValueType(); 1773 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP || 1774 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 1775 SDLoc dl(N); 1776 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode(); 1777 1778 // TODO: Any other flags to propagate? 1779 SDNodeFlags Flags; 1780 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept()); 1781 1782 // First do an SINT_TO_FP, whether the original was signed or unsigned. 1783 // When promoting partial word types to i32 we must honor the signedness, 1784 // though. 1785 if (SrcVT.bitsLE(MVT::i32)) { 1786 // The integer can be represented exactly in an f64. 1787 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1788 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1789 if (Strict) { 1790 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), 1791 {Chain, Src}, Flags); 1792 Chain = Hi.getValue(1); 1793 } else 1794 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src); 1795 } else { 1796 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1797 if (SrcVT.bitsLE(MVT::i64)) { 1798 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1799 MVT::i64, Src); 1800 LC = RTLIB::SINTTOFP_I64_PPCF128; 1801 } else if (SrcVT.bitsLE(MVT::i128)) { 1802 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); 1803 LC = RTLIB::SINTTOFP_I128_PPCF128; 1804 } 1805 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 1806 1807 TargetLowering::MakeLibCallOptions CallOptions; 1808 CallOptions.setSExt(true); 1809 std::pair<SDValue, SDValue> Tmp = 1810 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); 1811 if (Strict) 1812 Chain = Tmp.second; 1813 GetPairElements(Tmp.first, Lo, Hi); 1814 } 1815 1816 // No need to complement for unsigned 32-bit integers 1817 if (isSigned || SrcVT.bitsLE(MVT::i32)) { 1818 if (Strict) 1819 ReplaceValueWith(SDValue(N, 1), Chain); 1820 1821 return; 1822 } 1823 1824 // Unsigned - fix up the SINT_TO_FP value just calculated. 1825 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully 1826 // keep semantics correctness if the integer is not exactly representable 1827 // here. See ExpandLegalINT_TO_FP. 1828 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 1829 SrcVT = Src.getValueType(); 1830 1831 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. 1832 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 1833 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 1834 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; 1835 ArrayRef<uint64_t> Parts; 1836 1837 switch (SrcVT.getSimpleVT().SimpleTy) { 1838 default: 1839 llvm_unreachable("Unsupported UINT_TO_FP!"); 1840 case MVT::i32: 1841 Parts = TwoE32; 1842 break; 1843 case MVT::i64: 1844 Parts = TwoE64; 1845 break; 1846 case MVT::i128: 1847 Parts = TwoE128; 1848 break; 1849 } 1850 1851 // TODO: Are there other fast-math-flags to propagate to this FADD? 1852 SDValue NewLo = DAG.getConstantFP( 1853 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128); 1854 if (Strict) { 1855 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other), 1856 {Chain, Hi, NewLo}, Flags); 1857 Chain = Lo.getValue(1); 1858 ReplaceValueWith(SDValue(N, 1), Chain); 1859 } else 1860 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo); 1861 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT), 1862 Lo, Hi, ISD::SETLT); 1863 GetPairElements(Lo, Lo, Hi); 1864 } 1865 1866 1867 //===----------------------------------------------------------------------===// 1868 // Float Operand Expansion 1869 //===----------------------------------------------------------------------===// 1870 1871 /// ExpandFloatOperand - This method is called when the specified operand of the 1872 /// specified node is found to need expansion. At this point, all of the result 1873 /// types of the node are known to be legal, but other operands of the node may 1874 /// need promotion or expansion as well as the specified one. 1875 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { 1876 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG)); 1877 SDValue Res = SDValue(); 1878 1879 // See if the target wants to custom expand this node. 1880 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1881 return false; 1882 1883 switch (N->getOpcode()) { 1884 default: 1885 #ifndef NDEBUG 1886 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": "; 1887 N->dump(&DAG); dbgs() << "\n"; 1888 #endif 1889 report_fatal_error("Do not know how to expand this operator's operand!"); 1890 1891 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 1892 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1893 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1894 1895 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; 1896 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; 1897 case ISD::STRICT_FP_ROUND: 1898 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; 1899 case ISD::STRICT_FP_TO_SINT: 1900 case ISD::STRICT_FP_TO_UINT: 1901 case ISD::FP_TO_SINT: 1902 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break; 1903 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; 1904 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; 1905 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; 1906 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; 1907 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; 1908 case ISD::STRICT_FSETCC: 1909 case ISD::STRICT_FSETCCS: 1910 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; 1911 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N), 1912 OpNo); break; 1913 } 1914 1915 // If the result is null, the sub-method took care of registering results etc. 1916 if (!Res.getNode()) return false; 1917 1918 // If the result is N, the sub-method updated N in place. Tell the legalizer 1919 // core about this. 1920 if (Res.getNode() == N) 1921 return true; 1922 1923 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1924 "Invalid operand expansion"); 1925 1926 ReplaceValueWith(SDValue(N, 0), Res); 1927 return false; 1928 } 1929 1930 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code 1931 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1932 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, 1933 SDValue &NewRHS, 1934 ISD::CondCode &CCCode, 1935 const SDLoc &dl, SDValue &Chain, 1936 bool IsSignaling) { 1937 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 1938 GetExpandedFloat(NewLHS, LHSLo, LHSHi); 1939 GetExpandedFloat(NewRHS, RHSLo, RHSHi); 1940 1941 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); 1942 1943 // FIXME: This generated code sucks. We want to generate 1944 // FCMPU crN, hi1, hi2 1945 // BNE crN, L: 1946 // FCMPU crN, lo1, lo2 1947 // The following can be improved, but not that much. 1948 SDValue Tmp1, Tmp2, Tmp3, OutputChain; 1949 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1950 RHSHi, ISD::SETOEQ, Chain, IsSignaling); 1951 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1952 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 1953 RHSLo, CCCode, OutputChain, IsSignaling); 1954 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1955 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1956 Tmp1 = 1957 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi, 1958 ISD::SETUNE, OutputChain, IsSignaling); 1959 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1960 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1961 RHSHi, CCCode, OutputChain, IsSignaling); 1962 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1963 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1964 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 1965 NewRHS = SDValue(); // LHS is the result, not a compare. 1966 Chain = OutputChain; 1967 } 1968 1969 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { 1970 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1971 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1972 SDValue Chain; 1973 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 1974 1975 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1976 // against zero to select between true and false values. 1977 if (!NewRHS.getNode()) { 1978 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1979 CCCode = ISD::SETNE; 1980 } 1981 1982 // Update N to have the operands specified. 1983 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1984 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1985 N->getOperand(4)), 0); 1986 } 1987 1988 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { 1989 assert(N->getOperand(1).getValueType() == MVT::ppcf128 && 1990 "Logic only correct for ppcf128!"); 1991 SDValue Lo, Hi; 1992 GetExpandedFloat(N->getOperand(1), Lo, Hi); 1993 // The ppcf128 value is providing only the sign; take it from the 1994 // higher-order double (which must have the larger magnitude). 1995 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), 1996 N->getValueType(0), N->getOperand(0), Hi); 1997 } 1998 1999 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { 2000 bool IsStrict = N->isStrictFPOpcode(); 2001 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && 2002 "Logic only correct for ppcf128!"); 2003 SDValue Lo, Hi; 2004 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi); 2005 2006 if (!IsStrict) 2007 // Round it the rest of the way (e.g. to f32) if needed. 2008 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 2009 N->getValueType(0), Hi, N->getOperand(1)); 2010 2011 // Eliminate the node if the input float type is the same as the output float 2012 // type. 2013 if (Hi.getValueType() == N->getValueType(0)) { 2014 // Connect the output chain to the input chain, unlinking the node. 2015 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 2016 ReplaceValueWith(SDValue(N, 0), Hi); 2017 return SDValue(); 2018 } 2019 2020 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 2021 {N->getValueType(0), MVT::Other}, 2022 {N->getOperand(0), Hi, N->getOperand(2)}); 2023 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1)); 2024 ReplaceValueWith(SDValue(N, 0), Expansion); 2025 return SDValue(); 2026 } 2027 2028 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) { 2029 EVT RVT = N->getValueType(0); 2030 SDLoc dl(N); 2031 2032 bool IsStrict = N->isStrictFPOpcode(); 2033 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 2034 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 2035 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 2036 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2037 2038 EVT NVT; 2039 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed); 2040 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 2041 "Unsupported FP_TO_XINT!"); 2042 TargetLowering::MakeLibCallOptions CallOptions; 2043 std::pair<SDValue, SDValue> Tmp = 2044 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); 2045 if (!IsStrict) 2046 return Tmp.first; 2047 2048 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2049 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2050 return SDValue(); 2051 } 2052 2053 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { 2054 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2055 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2056 SDValue Chain; 2057 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 2058 2059 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2060 // against zero to select between true and false values. 2061 if (!NewRHS.getNode()) { 2062 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 2063 CCCode = ISD::SETNE; 2064 } 2065 2066 // Update N to have the operands specified. 2067 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2068 N->getOperand(2), N->getOperand(3), 2069 DAG.getCondCode(CCCode)), 0); 2070 } 2071 2072 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { 2073 bool IsStrict = N->isStrictFPOpcode(); 2074 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0); 2075 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1); 2076 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2077 ISD::CondCode CCCode = 2078 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 2079 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain, 2080 N->getOpcode() == ISD::STRICT_FSETCCS); 2081 2082 // FloatExpandSetCCOperands always returned a scalar. 2083 assert(!NewRHS.getNode() && "Expect to return scalar"); 2084 assert(NewLHS.getValueType() == N->getValueType(0) && 2085 "Unexpected setcc expansion!"); 2086 if (Chain) { 2087 ReplaceValueWith(SDValue(N, 0), NewLHS); 2088 ReplaceValueWith(SDValue(N, 1), Chain); 2089 return SDValue(); 2090 } 2091 return NewLHS; 2092 } 2093 2094 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { 2095 if (ISD::isNormalStore(N)) 2096 return ExpandOp_NormalStore(N, OpNo); 2097 2098 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2099 assert(OpNo == 1 && "Can only expand the stored value so far"); 2100 StoreSDNode *ST = cast<StoreSDNode>(N); 2101 2102 SDValue Chain = ST->getChain(); 2103 SDValue Ptr = ST->getBasePtr(); 2104 2105 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 2106 ST->getValue().getValueType()); 2107 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2108 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 2109 (void)NVT; 2110 2111 SDValue Lo, Hi; 2112 GetExpandedOp(ST->getValue(), Lo, Hi); 2113 2114 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr, 2115 ST->getMemoryVT(), ST->getMemOperand()); 2116 } 2117 2118 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { 2119 EVT RVT = N->getValueType(0); 2120 EVT RetVT = N->getOperand(0).getValueType(); 2121 TargetLowering::MakeLibCallOptions CallOptions; 2122 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2123 RTLIB::LROUND_F32, 2124 RTLIB::LROUND_F64, 2125 RTLIB::LROUND_F80, 2126 RTLIB::LROUND_F128, 2127 RTLIB::LROUND_PPCF128), 2128 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2129 } 2130 2131 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { 2132 EVT RVT = N->getValueType(0); 2133 EVT RetVT = N->getOperand(0).getValueType(); 2134 TargetLowering::MakeLibCallOptions CallOptions; 2135 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2136 RTLIB::LLROUND_F32, 2137 RTLIB::LLROUND_F64, 2138 RTLIB::LLROUND_F80, 2139 RTLIB::LLROUND_F128, 2140 RTLIB::LLROUND_PPCF128), 2141 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2142 } 2143 2144 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { 2145 EVT RVT = N->getValueType(0); 2146 EVT RetVT = N->getOperand(0).getValueType(); 2147 TargetLowering::MakeLibCallOptions CallOptions; 2148 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2149 RTLIB::LRINT_F32, 2150 RTLIB::LRINT_F64, 2151 RTLIB::LRINT_F80, 2152 RTLIB::LRINT_F128, 2153 RTLIB::LRINT_PPCF128), 2154 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2155 } 2156 2157 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { 2158 EVT RVT = N->getValueType(0); 2159 EVT RetVT = N->getOperand(0).getValueType(); 2160 TargetLowering::MakeLibCallOptions CallOptions; 2161 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2162 RTLIB::LLRINT_F32, 2163 RTLIB::LLRINT_F64, 2164 RTLIB::LLRINT_F80, 2165 RTLIB::LLRINT_F128, 2166 RTLIB::LLRINT_PPCF128), 2167 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2168 } 2169 2170 //===----------------------------------------------------------------------===// 2171 // Float Operand Promotion 2172 //===----------------------------------------------------------------------===// 2173 // 2174 2175 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { 2176 if (OpVT == MVT::f16) { 2177 return ISD::FP16_TO_FP; 2178 } else if (RetVT == MVT::f16) { 2179 return ISD::FP_TO_FP16; 2180 } else if (OpVT == MVT::bf16) { 2181 return ISD::BF16_TO_FP; 2182 } else if (RetVT == MVT::bf16) { 2183 return ISD::FP_TO_BF16; 2184 } 2185 2186 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2187 } 2188 2189 static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) { 2190 if (OpVT == MVT::f16) 2191 return ISD::STRICT_FP16_TO_FP; 2192 2193 if (RetVT == MVT::f16) 2194 return ISD::STRICT_FP_TO_FP16; 2195 2196 if (OpVT == MVT::bf16) { 2197 // TODO: return ISD::STRICT_BF16_TO_FP; 2198 } 2199 2200 if (RetVT == MVT::bf16) { 2201 // TODO: return ISD::STRICT_FP_TO_BF16; 2202 } 2203 2204 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2205 } 2206 2207 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { 2208 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG)); 2209 SDValue R = SDValue(); 2210 2211 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2212 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2213 return false; 2214 } 2215 2216 // Nodes that use a promotion-requiring floating point operand, but doesn't 2217 // produce a promotion-requiring floating point result, need to be legalized 2218 // to use the promoted float operand. Nodes that produce at least one 2219 // promotion-requiring floating point result have their operands legalized as 2220 // a part of PromoteFloatResult. 2221 // clang-format off 2222 switch (N->getOpcode()) { 2223 default: 2224 #ifndef NDEBUG 2225 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; 2226 N->dump(&DAG); dbgs() << "\n"; 2227 #endif 2228 report_fatal_error("Do not know how to promote this operator's operand!"); 2229 2230 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; 2231 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; 2232 case ISD::FP_TO_SINT: 2233 case ISD::FP_TO_UINT: 2234 case ISD::LRINT: 2235 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break; 2236 case ISD::FP_TO_SINT_SAT: 2237 case ISD::FP_TO_UINT_SAT: 2238 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; 2239 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; 2240 case ISD::STRICT_FP_EXTEND: 2241 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo); 2242 break; 2243 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; 2244 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; 2245 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; 2246 } 2247 // clang-format on 2248 2249 if (R.getNode()) 2250 ReplaceValueWith(SDValue(N, 0), R); 2251 return false; 2252 } 2253 2254 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { 2255 SDValue Op = N->getOperand(0); 2256 EVT OpVT = Op->getValueType(0); 2257 2258 SDValue Promoted = GetPromotedFloat(N->getOperand(0)); 2259 EVT PromotedVT = Promoted->getValueType(0); 2260 2261 // Convert the promoted float value to the desired IVT. 2262 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); 2263 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), 2264 IVT, Promoted); 2265 // The final result type might not be an scalar so we need a bitcast. The 2266 // bitcast will be further legalized if needed. 2267 return DAG.getBitcast(N->getValueType(0), Convert); 2268 } 2269 2270 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by 2271 // PromoteFloatRes_FCOPYSIGN. 2272 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { 2273 assert (OpNo == 1 && "Only Operand 1 must need promotion here"); 2274 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2275 2276 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2277 N->getOperand(0), Op1); 2278 } 2279 2280 // Convert the promoted float value to the desired integer type 2281 SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) { 2282 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2283 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); 2284 } 2285 2286 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, 2287 unsigned OpNo) { 2288 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2289 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, 2290 N->getOperand(1)); 2291 } 2292 2293 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { 2294 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2295 EVT VT = N->getValueType(0); 2296 2297 // Desired VT is same as promoted type. Use promoted float directly. 2298 if (VT == Op->getValueType(0)) 2299 return Op; 2300 2301 // Else, extend the promoted float value to the desired VT. 2302 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); 2303 } 2304 2305 SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, 2306 unsigned OpNo) { 2307 assert(OpNo == 1 && "Promoting unpromotable operand"); 2308 2309 SDValue Op = GetPromotedFloat(N->getOperand(1)); 2310 EVT VT = N->getValueType(0); 2311 2312 // Desired VT is same as promoted type. Use promoted float directly. 2313 if (VT == Op->getValueType(0)) { 2314 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 2315 return Op; 2316 } 2317 2318 // Else, extend the promoted float value to the desired VT. 2319 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(), 2320 N->getOperand(0), Op); 2321 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2322 return Res; 2323 } 2324 2325 // Promote the float operands used for comparison. The true- and false- 2326 // operands have the same type as the result and are promoted, if needed, by 2327 // PromoteFloatRes_SELECT_CC 2328 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2329 SDValue LHS = GetPromotedFloat(N->getOperand(0)); 2330 SDValue RHS = GetPromotedFloat(N->getOperand(1)); 2331 2332 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 2333 LHS, RHS, N->getOperand(2), N->getOperand(3), 2334 N->getOperand(4)); 2335 } 2336 2337 // Construct a SETCC that compares the promoted values and sets the conditional 2338 // code. 2339 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { 2340 EVT VT = N->getValueType(0); 2341 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2342 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2343 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2344 2345 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); 2346 2347 } 2348 2349 // Lower the promoted Float down to the integer value of same size and construct 2350 // a STORE of the integer value. 2351 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { 2352 StoreSDNode *ST = cast<StoreSDNode>(N); 2353 SDValue Val = ST->getValue(); 2354 SDLoc DL(N); 2355 2356 SDValue Promoted = GetPromotedFloat(Val); 2357 EVT VT = ST->getOperand(1).getValueType(); 2358 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2359 2360 SDValue NewVal; 2361 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, 2362 IVT, Promoted); 2363 2364 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), 2365 ST->getMemOperand()); 2366 } 2367 2368 //===----------------------------------------------------------------------===// 2369 // Float Result Promotion 2370 //===----------------------------------------------------------------------===// 2371 2372 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { 2373 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG)); 2374 SDValue R = SDValue(); 2375 2376 // See if the target wants to custom expand this node. 2377 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2378 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2379 return; 2380 } 2381 2382 switch (N->getOpcode()) { 2383 // These opcodes cannot appear if promotion of FP16 is done in the backend 2384 // instead of Clang 2385 case ISD::FP16_TO_FP: 2386 case ISD::FP_TO_FP16: 2387 default: 2388 #ifndef NDEBUG 2389 dbgs() << "PromoteFloatResult #" << ResNo << ": "; 2390 N->dump(&DAG); dbgs() << "\n"; 2391 #endif 2392 report_fatal_error("Do not know how to promote this operator's result!"); 2393 2394 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; 2395 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; 2396 case ISD::EXTRACT_VECTOR_ELT: 2397 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; 2398 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; 2399 2400 // Unary FP Operations 2401 case ISD::FABS: 2402 case ISD::FCBRT: 2403 case ISD::FCEIL: 2404 case ISD::FCOS: 2405 case ISD::FEXP: 2406 case ISD::FEXP2: 2407 case ISD::FEXP10: 2408 case ISD::FFLOOR: 2409 case ISD::FLOG: 2410 case ISD::FLOG2: 2411 case ISD::FLOG10: 2412 case ISD::FNEARBYINT: 2413 case ISD::FNEG: 2414 case ISD::FRINT: 2415 case ISD::FROUND: 2416 case ISD::FROUNDEVEN: 2417 case ISD::FSIN: 2418 case ISD::FSQRT: 2419 case ISD::FTRUNC: 2420 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; 2421 2422 // Binary FP Operations 2423 case ISD::FADD: 2424 case ISD::FDIV: 2425 case ISD::FMAXIMUM: 2426 case ISD::FMINIMUM: 2427 case ISD::FMAXNUM: 2428 case ISD::FMINNUM: 2429 case ISD::FMUL: 2430 case ISD::FPOW: 2431 case ISD::FREM: 2432 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; 2433 2434 case ISD::FMA: // FMA is same as FMAD 2435 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; 2436 2437 case ISD::FPOWI: 2438 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break; 2439 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break; 2440 2441 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; 2442 case ISD::STRICT_FP_ROUND: 2443 R = PromoteFloatRes_STRICT_FP_ROUND(N); 2444 break; 2445 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; 2446 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; 2447 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; 2448 2449 case ISD::SINT_TO_FP: 2450 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; 2451 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; 2452 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2453 case ISD::VECREDUCE_FADD: 2454 case ISD::VECREDUCE_FMUL: 2455 case ISD::VECREDUCE_FMIN: 2456 case ISD::VECREDUCE_FMAX: 2457 case ISD::VECREDUCE_FMAXIMUM: 2458 case ISD::VECREDUCE_FMINIMUM: 2459 R = PromoteFloatRes_VECREDUCE(N); 2460 break; 2461 case ISD::VECREDUCE_SEQ_FADD: 2462 case ISD::VECREDUCE_SEQ_FMUL: 2463 R = PromoteFloatRes_VECREDUCE_SEQ(N); 2464 break; 2465 } 2466 2467 if (R.getNode()) 2468 SetPromotedFloat(SDValue(N, ResNo), R); 2469 } 2470 2471 // Bitcast from i16 to f16: convert the i16 to a f32 value instead. 2472 // At this point, it is not possible to determine if the bitcast value is 2473 // eventually stored to memory or promoted to f32 or promoted to a floating 2474 // point at a higher precision. Some of these cases are handled by FP_EXTEND, 2475 // STORE promotion handlers. 2476 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { 2477 EVT VT = N->getValueType(0); 2478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2479 // Input type isn't guaranteed to be a scalar int so bitcast if not. The 2480 // bitcast will be legalized further if necessary. 2481 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), 2482 N->getOperand(0).getValueType().getSizeInBits()); 2483 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); 2484 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); 2485 } 2486 2487 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { 2488 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); 2489 EVT VT = N->getValueType(0); 2490 SDLoc DL(N); 2491 2492 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2493 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2494 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, 2495 IVT); 2496 2497 // Convert the Constant to the desired FP type 2498 // FIXME We might be able to do the conversion during compilation and get rid 2499 // of it from the object code 2500 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2501 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); 2502 } 2503 2504 // If the Index operand is a constant, try to redirect the extract operation to 2505 // the correct legalized vector. If not, bit-convert the input vector to 2506 // equivalent integer vector. Extract the element as an (bit-cast) integer 2507 // value and convert it to the promoted type. 2508 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2509 SDLoc DL(N); 2510 2511 // If the index is constant, try to extract the value from the legalized 2512 // vector type. 2513 if (isa<ConstantSDNode>(N->getOperand(1))) { 2514 SDValue Vec = N->getOperand(0); 2515 SDValue Idx = N->getOperand(1); 2516 EVT VecVT = Vec->getValueType(0); 2517 EVT EltVT = VecVT.getVectorElementType(); 2518 2519 uint64_t IdxVal = Idx->getAsZExtVal(); 2520 2521 switch (getTypeAction(VecVT)) { 2522 default: break; 2523 case TargetLowering::TypeScalarizeVector: { 2524 SDValue Res = GetScalarizedVector(N->getOperand(0)); 2525 ReplaceValueWith(SDValue(N, 0), Res); 2526 return SDValue(); 2527 } 2528 case TargetLowering::TypeWidenVector: { 2529 Vec = GetWidenedVector(Vec); 2530 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); 2531 ReplaceValueWith(SDValue(N, 0), Res); 2532 return SDValue(); 2533 } 2534 case TargetLowering::TypeSplitVector: { 2535 SDValue Lo, Hi; 2536 GetSplitVector(Vec, Lo, Hi); 2537 2538 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2539 SDValue Res; 2540 if (IdxVal < LoElts) 2541 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); 2542 else 2543 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, 2544 DAG.getConstant(IdxVal - LoElts, DL, 2545 Idx.getValueType())); 2546 ReplaceValueWith(SDValue(N, 0), Res); 2547 return SDValue(); 2548 } 2549 2550 } 2551 } 2552 2553 // Bit-convert the input vector to the equivalent integer vector 2554 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2555 EVT IVT = NewOp.getValueType().getVectorElementType(); 2556 2557 // Extract the element as an (bit-cast) integer value 2558 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, 2559 NewOp, N->getOperand(1)); 2560 2561 // Convert the element to the desired FP type 2562 EVT VT = N->getValueType(0); 2563 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2564 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); 2565 } 2566 2567 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result 2568 // needs promotion, so does the argument X. Note that Y, if needed, will be 2569 // handled during operand promotion. 2570 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { 2571 EVT VT = N->getValueType(0); 2572 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2573 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2574 2575 SDValue Op1 = N->getOperand(1); 2576 2577 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2578 } 2579 2580 // Unary operation where the result and the operand have PromoteFloat type 2581 // action. Construct a new SDNode with the promoted float value of the old 2582 // operand. 2583 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { 2584 EVT VT = N->getValueType(0); 2585 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2586 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2587 2588 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); 2589 } 2590 2591 // Binary operations where the result and both operands have PromoteFloat type 2592 // action. Construct a new SDNode with the promoted float values of the old 2593 // operands. 2594 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { 2595 EVT VT = N->getValueType(0); 2596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2597 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2598 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2599 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); 2600 } 2601 2602 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { 2603 EVT VT = N->getValueType(0); 2604 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2605 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2606 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2607 SDValue Op2 = GetPromotedFloat(N->getOperand(2)); 2608 2609 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); 2610 } 2611 2612 // Promote the Float (first) operand and retain the Integer (second) operand 2613 SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) { 2614 EVT VT = N->getValueType(0); 2615 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2616 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2617 SDValue Op1 = N->getOperand(1); 2618 2619 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2620 } 2621 2622 SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) { 2623 EVT VT = N->getValueType(0); 2624 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2625 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2626 SDValue Res = 2627 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op); 2628 2629 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2630 return Res; 2631 } 2632 2633 // Explicit operation to reduce precision. Reduce the value to half precision 2634 // and promote it back to the legal type. 2635 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { 2636 SDLoc DL(N); 2637 2638 SDValue Op = N->getOperand(0); 2639 EVT VT = N->getValueType(0); 2640 EVT OpVT = Op->getValueType(0); 2641 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2642 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2643 2644 // Round promoted float to desired precision 2645 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); 2646 // Promote it back to the legal output type 2647 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); 2648 } 2649 2650 // Explicit operation to reduce precision. Reduce the value to half precision 2651 // and promote it back to the legal type. 2652 SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) { 2653 SDLoc DL(N); 2654 2655 SDValue Chain = N->getOperand(0); 2656 SDValue Op = N->getOperand(1); 2657 EVT VT = N->getValueType(0); 2658 EVT OpVT = Op->getValueType(0); 2659 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2660 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2661 2662 // Round promoted float to desired precision 2663 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL, 2664 DAG.getVTList(IVT, MVT::Other), Chain, Op); 2665 // Promote it back to the legal output type 2666 SDValue Res = 2667 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL, 2668 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round); 2669 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2670 return Res; 2671 } 2672 2673 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { 2674 LoadSDNode *L = cast<LoadSDNode>(N); 2675 EVT VT = N->getValueType(0); 2676 2677 // Load the value as an integer value with the same number of bits. 2678 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2679 SDValue newL = DAG.getLoad( 2680 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), 2681 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, 2682 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); 2683 // Legalize the chain result by replacing uses of the old value chain with the 2684 // new one 2685 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 2686 2687 // Convert the integer value to the desired FP type 2688 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2689 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); 2690 } 2691 2692 // Construct a new SELECT node with the promoted true- and false- values. 2693 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { 2694 SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); 2695 SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); 2696 2697 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), 2698 N->getOperand(0), TrueVal, FalseVal); 2699 } 2700 2701 // Construct a new SELECT_CC node with the promoted true- and false- values. 2702 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. 2703 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { 2704 SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); 2705 SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); 2706 2707 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2708 TrueVal.getNode()->getValueType(0), N->getOperand(0), 2709 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); 2710 } 2711 2712 // Construct a SDNode that transforms the SINT or UINT operand to the promoted 2713 // float type. 2714 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { 2715 SDLoc DL(N); 2716 EVT VT = N->getValueType(0); 2717 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2718 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); 2719 // Round the value to the desired precision (that of the source type). 2720 return DAG.getNode( 2721 ISD::FP_EXTEND, DL, NVT, 2722 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, 2723 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true))); 2724 } 2725 2726 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { 2727 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 2728 N->getValueType(0))); 2729 } 2730 2731 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { 2732 // Expand and promote recursively. 2733 // TODO: This is non-optimal, but dealing with the concurrently happening 2734 // vector-legalization is non-trivial. We could do something similar to 2735 // PromoteFloatRes_EXTRACT_VECTOR_ELT here. 2736 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2737 return SDValue(); 2738 } 2739 2740 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { 2741 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2742 return SDValue(); 2743 } 2744 2745 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { 2746 EVT VT = N->getValueType(0); 2747 2748 AtomicSDNode *AM = cast<AtomicSDNode>(N); 2749 SDLoc SL(N); 2750 2751 SDValue CastVal = BitConvertToInteger(AM->getVal()); 2752 EVT CastVT = CastVal.getValueType(); 2753 2754 SDValue NewAtomic 2755 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, 2756 DAG.getVTList(CastVT, MVT::Other), 2757 { AM->getChain(), AM->getBasePtr(), CastVal }, 2758 AM->getMemOperand()); 2759 2760 SDValue Result = NewAtomic; 2761 2762 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { 2763 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2764 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, 2765 NewAtomic); 2766 } 2767 2768 // Legalize the chain result by replacing uses of the old value chain with the 2769 // new one 2770 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); 2771 2772 return Result; 2773 2774 } 2775 2776 //===----------------------------------------------------------------------===// 2777 // Half Result Soft Promotion 2778 //===----------------------------------------------------------------------===// 2779 2780 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { 2781 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": "; 2782 N->dump(&DAG)); 2783 SDValue R = SDValue(); 2784 2785 // See if the target wants to custom expand this node. 2786 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2787 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2788 return; 2789 } 2790 2791 switch (N->getOpcode()) { 2792 default: 2793 #ifndef NDEBUG 2794 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": "; 2795 N->dump(&DAG); dbgs() << "\n"; 2796 #endif 2797 report_fatal_error("Do not know how to soft promote this operator's " 2798 "result!"); 2799 2800 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; 2801 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; 2802 case ISD::EXTRACT_VECTOR_ELT: 2803 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; 2804 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; 2805 case ISD::STRICT_FP_ROUND: 2806 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; 2807 2808 // Unary FP Operations 2809 case ISD::FABS: 2810 case ISD::FCBRT: 2811 case ISD::FCEIL: 2812 case ISD::FCOS: 2813 case ISD::FEXP: 2814 case ISD::FEXP2: 2815 case ISD::FEXP10: 2816 case ISD::FFLOOR: 2817 case ISD::FLOG: 2818 case ISD::FLOG2: 2819 case ISD::FLOG10: 2820 case ISD::FNEARBYINT: 2821 case ISD::FNEG: 2822 case ISD::FREEZE: 2823 case ISD::FRINT: 2824 case ISD::FROUND: 2825 case ISD::FROUNDEVEN: 2826 case ISD::FSIN: 2827 case ISD::FSQRT: 2828 case ISD::FTRUNC: 2829 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; 2830 2831 // Binary FP Operations 2832 case ISD::FADD: 2833 case ISD::FDIV: 2834 case ISD::FMAXIMUM: 2835 case ISD::FMINIMUM: 2836 case ISD::FMAXNUM: 2837 case ISD::FMINNUM: 2838 case ISD::FMUL: 2839 case ISD::FPOW: 2840 case ISD::FREM: 2841 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; 2842 2843 case ISD::FMA: // FMA is same as FMAD 2844 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; 2845 2846 case ISD::FPOWI: 2847 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break; 2848 2849 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break; 2850 2851 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; 2852 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; 2853 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; 2854 case ISD::SINT_TO_FP: 2855 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; 2856 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; 2857 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2858 case ISD::VECREDUCE_FADD: 2859 case ISD::VECREDUCE_FMUL: 2860 case ISD::VECREDUCE_FMIN: 2861 case ISD::VECREDUCE_FMAX: 2862 case ISD::VECREDUCE_FMAXIMUM: 2863 case ISD::VECREDUCE_FMINIMUM: 2864 R = SoftPromoteHalfRes_VECREDUCE(N); 2865 break; 2866 case ISD::VECREDUCE_SEQ_FADD: 2867 case ISD::VECREDUCE_SEQ_FMUL: 2868 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N); 2869 break; 2870 } 2871 2872 if (R.getNode()) 2873 SetSoftPromotedHalf(SDValue(N, ResNo), R); 2874 } 2875 2876 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { 2877 return BitConvertToInteger(N->getOperand(0)); 2878 } 2879 2880 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { 2881 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 2882 2883 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2884 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 2885 MVT::i16); 2886 } 2887 2888 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2889 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2890 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2891 NewOp.getValueType().getVectorElementType(), NewOp, 2892 N->getOperand(1)); 2893 } 2894 2895 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { 2896 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0)); 2897 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 2898 SDLoc dl(N); 2899 2900 EVT LVT = LHS.getValueType(); 2901 EVT RVT = RHS.getValueType(); 2902 2903 unsigned LSize = LVT.getSizeInBits(); 2904 unsigned RSize = RVT.getSizeInBits(); 2905 2906 // First get the sign bit of second operand. 2907 SDValue SignBit = DAG.getNode( 2908 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 2909 DAG.getConstant(RSize - 1, dl, 2910 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 2911 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 2912 2913 // Shift right or sign-extend it if the two operands have different types. 2914 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 2915 if (SizeDiff > 0) { 2916 SignBit = 2917 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 2918 DAG.getConstant(SizeDiff, dl, 2919 TLI.getShiftAmountTy(SignBit.getValueType(), 2920 DAG.getDataLayout()))); 2921 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 2922 } else if (SizeDiff < 0) { 2923 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 2924 SignBit = 2925 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 2926 DAG.getConstant(-SizeDiff, dl, 2927 TLI.getShiftAmountTy(SignBit.getValueType(), 2928 DAG.getDataLayout()))); 2929 } 2930 2931 // Clear the sign bit of the first operand. 2932 SDValue Mask = DAG.getNode( 2933 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 2934 DAG.getConstant(LSize - 1, dl, 2935 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 2936 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 2937 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 2938 2939 // Or the value with the sign bit. 2940 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 2941 } 2942 2943 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { 2944 EVT OVT = N->getValueType(0); 2945 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2946 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2947 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2948 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2949 SDLoc dl(N); 2950 2951 // Promote to the larger FP type. 2952 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 2953 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 2954 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 2955 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2); 2956 2957 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2); 2958 2959 // Convert back to FP16 as an integer. 2960 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2961 } 2962 2963 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) { 2964 EVT OVT = N->getValueType(0); 2965 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2966 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2967 SDValue Op1 = N->getOperand(1); 2968 SDLoc dl(N); 2969 2970 // Promote to the larger FP type. 2971 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0); 2972 2973 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2974 2975 // Convert back to FP16 as an integer. 2976 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2977 } 2978 2979 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) { 2980 EVT OVT = N->getValueType(0); 2981 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2982 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 2983 SDLoc dl(N); 2984 2985 // Promote to the larger FP type. 2986 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 2987 2988 SDValue Res = DAG.getNode(N->getOpcode(), dl, 2989 DAG.getVTList(NVT, N->getValueType(1)), Op); 2990 2991 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2992 2993 // Convert back to FP16 as an integer. 2994 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2995 } 2996 2997 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { 2998 EVT RVT = N->getValueType(0); 2999 EVT SVT = N->getOperand(0).getValueType(); 3000 3001 if (N->isStrictFPOpcode()) { 3002 assert(RVT == MVT::f16); 3003 SDValue Res = 3004 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other}, 3005 {N->getOperand(0), N->getOperand(1)}); 3006 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3007 return Res; 3008 } 3009 3010 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16, 3011 N->getOperand(0)); 3012 } 3013 3014 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { 3015 LoadSDNode *L = cast<LoadSDNode>(N); 3016 3017 // Load the value as an integer value with the same number of bits. 3018 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!"); 3019 SDValue NewL = 3020 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, 3021 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), 3022 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), 3023 L->getMemOperand()->getFlags(), L->getAAInfo()); 3024 // Legalize the chain result by replacing uses of the old value chain with the 3025 // new one 3026 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 3027 return NewL; 3028 } 3029 3030 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { 3031 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 3032 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 3033 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1, 3034 Op2); 3035 } 3036 3037 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { 3038 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 3039 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3)); 3040 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(), 3041 N->getOperand(0), N->getOperand(1), Op2, Op3, 3042 N->getOperand(4)); 3043 } 3044 3045 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { 3046 EVT OVT = N->getValueType(0); 3047 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3048 SDLoc dl(N); 3049 3050 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 3051 3052 // Round the value to the softened type. 3053 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3054 } 3055 3056 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { 3057 return DAG.getUNDEF(MVT::i16); 3058 } 3059 3060 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { 3061 EVT OVT = N->getValueType(0); 3062 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3063 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 3064 SDLoc dl(N); 3065 3066 // Promote to the larger FP type. 3067 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 3068 3069 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op); 3070 3071 // Convert back to FP16 as an integer. 3072 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3073 } 3074 3075 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { 3076 EVT OVT = N->getValueType(0); 3077 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3078 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3079 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 3080 SDLoc dl(N); 3081 3082 // Promote to the larger FP type. 3083 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 3084 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3085 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3086 3087 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 3088 3089 // Convert back to FP16 as an integer. 3090 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3091 } 3092 3093 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) { 3094 // Expand and soften recursively. 3095 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 3096 return SDValue(); 3097 } 3098 3099 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) { 3100 // Expand and soften. 3101 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 3102 return SDValue(); 3103 } 3104 3105 //===----------------------------------------------------------------------===// 3106 // Half Operand Soft Promotion 3107 //===----------------------------------------------------------------------===// 3108 3109 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { 3110 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": "; 3111 N->dump(&DAG)); 3112 SDValue Res = SDValue(); 3113 3114 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 3115 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 3116 return false; 3117 } 3118 3119 // Nodes that use a promotion-requiring floating point operand, but doesn't 3120 // produce a soft promotion-requiring floating point result, need to be 3121 // legalized to use the soft promoted float operand. Nodes that produce at 3122 // least one soft promotion-requiring floating point result have their 3123 // operands legalized as a part of PromoteFloatResult. 3124 switch (N->getOpcode()) { 3125 default: 3126 #ifndef NDEBUG 3127 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": "; 3128 N->dump(&DAG); dbgs() << "\n"; 3129 #endif 3130 report_fatal_error("Do not know how to soft promote this operator's " 3131 "operand!"); 3132 3133 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; 3134 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; 3135 case ISD::FP_TO_SINT: 3136 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; 3137 case ISD::FP_TO_SINT_SAT: 3138 case ISD::FP_TO_UINT_SAT: 3139 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; 3140 case ISD::STRICT_FP_EXTEND: 3141 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; 3142 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; 3143 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; 3144 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; 3145 case ISD::STACKMAP: 3146 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); 3147 break; 3148 case ISD::PATCHPOINT: 3149 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo); 3150 break; 3151 } 3152 3153 if (!Res.getNode()) 3154 return false; 3155 3156 assert(Res.getNode() != N && "Expected a new node!"); 3157 3158 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 3159 "Invalid operand expansion"); 3160 3161 ReplaceValueWith(SDValue(N, 0), Res); 3162 return false; 3163 } 3164 3165 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { 3166 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3167 3168 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 3169 } 3170 3171 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, 3172 unsigned OpNo) { 3173 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 3174 SDValue Op1 = N->getOperand(1); 3175 EVT RVT = Op1.getValueType(); 3176 SDLoc dl(N); 3177 3178 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType()); 3179 3180 Op1 = GetSoftPromotedHalf(Op1); 3181 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1); 3182 3183 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0), 3184 Op1); 3185 } 3186 3187 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { 3188 EVT RVT = N->getValueType(0); 3189 bool IsStrict = N->isStrictFPOpcode(); 3190 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3191 EVT SVT = Op.getValueType(); 3192 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0)); 3193 3194 if (IsStrict) { 3195 assert(SVT == MVT::f16); 3196 SDValue Res = 3197 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N), 3198 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op}); 3199 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3200 ReplaceValueWith(SDValue(N, 0), Res); 3201 return SDValue(); 3202 } 3203 3204 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op); 3205 } 3206 3207 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { 3208 EVT RVT = N->getValueType(0); 3209 SDValue Op = N->getOperand(0); 3210 EVT SVT = Op.getValueType(); 3211 SDLoc dl(N); 3212 3213 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3214 3215 Op = GetSoftPromotedHalf(Op); 3216 3217 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3218 3219 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res); 3220 } 3221 3222 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { 3223 EVT RVT = N->getValueType(0); 3224 SDValue Op = N->getOperand(0); 3225 EVT SVT = Op.getValueType(); 3226 SDLoc dl(N); 3227 3228 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3229 3230 Op = GetSoftPromotedHalf(Op); 3231 3232 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3233 3234 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res, 3235 N->getOperand(1)); 3236 } 3237 3238 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, 3239 unsigned OpNo) { 3240 assert(OpNo == 0 && "Can only soften the comparison values"); 3241 SDValue Op0 = N->getOperand(0); 3242 SDValue Op1 = N->getOperand(1); 3243 SDLoc dl(N); 3244 3245 EVT SVT = Op0.getValueType(); 3246 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT); 3247 3248 Op0 = GetSoftPromotedHalf(Op0); 3249 Op1 = GetSoftPromotedHalf(Op1); 3250 3251 // Promote to the larger FP type. 3252 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3253 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3254 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3255 3256 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1, 3257 N->getOperand(2), N->getOperand(3), N->getOperand(4)); 3258 } 3259 3260 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { 3261 SDValue Op0 = N->getOperand(0); 3262 SDValue Op1 = N->getOperand(1); 3263 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3264 SDLoc dl(N); 3265 3266 EVT SVT = Op0.getValueType(); 3267 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 3268 3269 Op0 = GetSoftPromotedHalf(Op0); 3270 Op1 = GetSoftPromotedHalf(Op1); 3271 3272 // Promote to the larger FP type. 3273 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3274 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3275 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3276 3277 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode); 3278 } 3279 3280 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { 3281 assert(OpNo == 1 && "Can only soften the stored value!"); 3282 StoreSDNode *ST = cast<StoreSDNode>(N); 3283 SDValue Val = ST->getValue(); 3284 SDLoc dl(N); 3285 3286 assert(!ST->isTruncatingStore() && "Unexpected truncating store."); 3287 SDValue Promoted = GetSoftPromotedHalf(Val); 3288 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), 3289 ST->getMemOperand()); 3290 } 3291 3292 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { 3293 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 3294 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3295 SDValue Op = N->getOperand(OpNo); 3296 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3297 SDValue NewNode = 3298 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3299 3300 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3301 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3302 3303 return SDValue(); // Signal that we replaced the node ourselves. 3304 } 3305 3306 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N, 3307 unsigned OpNo) { 3308 assert(OpNo >= 7); 3309 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3310 SDValue Op = N->getOperand(OpNo); 3311 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3312 SDValue NewNode = 3313 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3314 3315 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3316 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3317 3318 return SDValue(); // Signal that we replaced the node ourselves. 3319 } 3320