1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements float type expansion and softening for LegalizeTypes. 10 // Softening is the act of turning a computation in an illegal floating point 11 // type into a computation in an integer type of the same size; also known as 12 // "soft float". For example, turning f32 arithmetic into operations using i32. 13 // The resulting integer value is the same as what you would get by performing 14 // the floating point operation and bitcasting the result to the integer type. 15 // Expansion is the act of changing a computation in an illegal type to be a 16 // computation in two identical registers of a smaller type. For example, 17 // implementing ppcf128 arithmetic in two f64 registers. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/Analysis/TargetLibraryInfo.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 /// GetFPLibCall - Return the right libcall for the given floating point type. 30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be 31 /// refactored away (see RTLIB::getPOWI for an example). 32 static RTLIB::Libcall GetFPLibCall(EVT VT, 33 RTLIB::Libcall Call_F32, 34 RTLIB::Libcall Call_F64, 35 RTLIB::Libcall Call_F80, 36 RTLIB::Libcall Call_F128, 37 RTLIB::Libcall Call_PPCF128) { 38 return 39 VT == MVT::f32 ? Call_F32 : 40 VT == MVT::f64 ? Call_F64 : 41 VT == MVT::f80 ? Call_F80 : 42 VT == MVT::f128 ? Call_F128 : 43 VT == MVT::ppcf128 ? Call_PPCF128 : 44 RTLIB::UNKNOWN_LIBCALL; 45 } 46 47 //===----------------------------------------------------------------------===// 48 // Convert Float Results to Integer 49 //===----------------------------------------------------------------------===// 50 51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { 52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG); 53 dbgs() << "\n"); 54 SDValue R = SDValue(); 55 56 switch (N->getOpcode()) { 57 default: 58 #ifndef NDEBUG 59 dbgs() << "SoftenFloatResult #" << ResNo << ": "; 60 N->dump(&DAG); dbgs() << "\n"; 61 #endif 62 report_fatal_error("Do not know how to soften the result of this " 63 "operator!"); 64 65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break; 66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; 67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; 68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; 69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; 70 case ISD::EXTRACT_VECTOR_ELT: 71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; 72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break; 73 case ISD::STRICT_FMINNUM: 74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; 75 case ISD::STRICT_FMAXNUM: 76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; 77 case ISD::STRICT_FADD: 78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break; 79 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; 80 case ISD::STRICT_FCEIL: 81 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; 82 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; 83 case ISD::STRICT_FCOS: 84 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; 85 case ISD::STRICT_FDIV: 86 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; 87 case ISD::STRICT_FEXP: 88 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; 89 case ISD::STRICT_FEXP2: 90 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; 91 case ISD::STRICT_FFLOOR: 92 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; 93 case ISD::STRICT_FLOG: 94 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; 95 case ISD::STRICT_FLOG2: 96 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; 97 case ISD::STRICT_FLOG10: 98 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; 99 case ISD::STRICT_FMA: 100 case ISD::FMA: R = SoftenFloatRes_FMA(N); break; 101 case ISD::STRICT_FMUL: 102 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; 103 case ISD::STRICT_FNEARBYINT: 104 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; 105 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; 106 case ISD::STRICT_FP_EXTEND: 107 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; 108 case ISD::STRICT_FP_ROUND: 109 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; 110 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; 111 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break; 112 case ISD::STRICT_FPOW: 113 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; 114 case ISD::STRICT_FPOWI: 115 case ISD::FPOWI: 116 case ISD::FLDEXP: 117 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break; 118 case ISD::FFREXP: 119 R = SoftenFloatRes_FFREXP(N); 120 break; 121 case ISD::STRICT_FREM: 122 case ISD::FREM: R = SoftenFloatRes_FREM(N); break; 123 case ISD::STRICT_FRINT: 124 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; 125 case ISD::STRICT_FROUND: 126 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; 127 case ISD::STRICT_FROUNDEVEN: 128 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; 129 case ISD::STRICT_FSIN: 130 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; 131 case ISD::STRICT_FSQRT: 132 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; 133 case ISD::STRICT_FSUB: 134 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; 135 case ISD::STRICT_FTRUNC: 136 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; 137 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; 138 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 139 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; 140 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; 141 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; 142 case ISD::STRICT_SINT_TO_FP: 143 case ISD::STRICT_UINT_TO_FP: 144 case ISD::SINT_TO_FP: 145 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; 146 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; 147 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; 148 case ISD::VECREDUCE_FADD: 149 case ISD::VECREDUCE_FMUL: 150 case ISD::VECREDUCE_FMIN: 151 case ISD::VECREDUCE_FMAX: 152 case ISD::VECREDUCE_FMAXIMUM: 153 case ISD::VECREDUCE_FMINIMUM: 154 R = SoftenFloatRes_VECREDUCE(N); 155 break; 156 case ISD::VECREDUCE_SEQ_FADD: 157 case ISD::VECREDUCE_SEQ_FMUL: 158 R = SoftenFloatRes_VECREDUCE_SEQ(N); 159 break; 160 } 161 162 // If R is null, the sub-method took care of registering the result. 163 if (R.getNode()) { 164 assert(R.getNode() != N); 165 SetSoftenedFloat(SDValue(N, ResNo), R); 166 } 167 } 168 169 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { 170 bool IsStrict = N->isStrictFPOpcode(); 171 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 172 unsigned Offset = IsStrict ? 1 : 0; 173 assert(N->getNumOperands() == (1 + Offset) && 174 "Unexpected number of operands!"); 175 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 176 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 177 TargetLowering::MakeLibCallOptions CallOptions; 178 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 179 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 180 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 181 CallOptions, SDLoc(N), 182 Chain); 183 if (IsStrict) 184 ReplaceValueWith(SDValue(N, 1), Tmp.second); 185 return Tmp.first; 186 } 187 188 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { 189 bool IsStrict = N->isStrictFPOpcode(); 190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 191 unsigned Offset = IsStrict ? 1 : 0; 192 assert(N->getNumOperands() == (2 + Offset) && 193 "Unexpected number of operands!"); 194 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 195 GetSoftenedFloat(N->getOperand(1 + Offset)) }; 196 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 197 TargetLowering::MakeLibCallOptions CallOptions; 198 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 199 N->getOperand(1 + Offset).getValueType() }; 200 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 202 CallOptions, SDLoc(N), 203 Chain); 204 if (IsStrict) 205 ReplaceValueWith(SDValue(N, 1), Tmp.second); 206 return Tmp.first; 207 } 208 209 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { 210 return BitConvertToInteger(N->getOperand(0)); 211 } 212 213 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { 214 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 215 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty, 216 GetSoftenedFloat(N->getOperand(0))); 217 } 218 219 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) { 220 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 221 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty, 222 GetSoftenedFloat(N->getOperand(0))); 223 return NewFence; 224 } 225 226 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, 227 unsigned ResNo) { 228 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 229 return BitConvertToInteger(Op); 230 } 231 232 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { 233 // Convert the inputs to integers, and build a new pair out of them. 234 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), 235 TLI.getTypeToTransformTo(*DAG.getContext(), 236 N->getValueType(0)), 237 BitConvertToInteger(N->getOperand(0)), 238 BitConvertToInteger(N->getOperand(1))); 239 } 240 241 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { 242 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 243 // In ppcf128, the high 64 bits are always first in memory regardless 244 // of Endianness. LLVM's APFloat representation is not Endian sensitive, 245 // and so always converts into a 128-bit APInt in a non-Endian-sensitive 246 // way. However, APInt's are serialized in an Endian-sensitive fashion, 247 // so on big-Endian targets, the two doubles are output in the wrong 248 // order. Fix this by manually flipping the order of the high 64 bits 249 // and the low 64 bits here. 250 if (DAG.getDataLayout().isBigEndian() && 251 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) { 252 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], 253 CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; 254 APInt Val(128, words); 255 return DAG.getConstant(Val, SDLoc(CN), 256 TLI.getTypeToTransformTo(*DAG.getContext(), 257 CN->getValueType(0))); 258 } else { 259 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 260 TLI.getTypeToTransformTo(*DAG.getContext(), 261 CN->getValueType(0))); 262 } 263 } 264 265 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) { 266 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 267 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 268 NewOp.getValueType().getVectorElementType(), 269 NewOp, N->getOperand(1)); 270 } 271 272 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { 273 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 274 unsigned Size = NVT.getSizeInBits(); 275 276 // Mask = ~(1 << (Size-1)) 277 APInt API = APInt::getAllOnes(Size); 278 API.clearBit(Size - 1); 279 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT); 280 SDValue Op = GetSoftenedFloat(N->getOperand(0)); 281 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask); 282 } 283 284 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { 285 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 286 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 287 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 288 RTLIB::FMIN_F32, 289 RTLIB::FMIN_F64, 290 RTLIB::FMIN_F80, 291 RTLIB::FMIN_F128, 292 RTLIB::FMIN_PPCF128)); 293 } 294 295 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { 296 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 297 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 299 RTLIB::FMAX_F32, 300 RTLIB::FMAX_F64, 301 RTLIB::FMAX_F80, 302 RTLIB::FMAX_F128, 303 RTLIB::FMAX_PPCF128)); 304 } 305 306 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { 307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 308 RTLIB::ADD_F32, 309 RTLIB::ADD_F64, 310 RTLIB::ADD_F80, 311 RTLIB::ADD_F128, 312 RTLIB::ADD_PPCF128)); 313 } 314 315 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { 316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 317 RTLIB::CBRT_F32, 318 RTLIB::CBRT_F64, 319 RTLIB::CBRT_F80, 320 RTLIB::CBRT_F128, 321 RTLIB::CBRT_PPCF128)); 322 } 323 324 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { 325 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 326 RTLIB::CEIL_F32, 327 RTLIB::CEIL_F64, 328 RTLIB::CEIL_F80, 329 RTLIB::CEIL_F128, 330 RTLIB::CEIL_PPCF128)); 331 } 332 333 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { 334 SDValue LHS = GetSoftenedFloat(N->getOperand(0)); 335 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 336 SDLoc dl(N); 337 338 EVT LVT = LHS.getValueType(); 339 EVT RVT = RHS.getValueType(); 340 341 unsigned LSize = LVT.getSizeInBits(); 342 unsigned RSize = RVT.getSizeInBits(); 343 344 // First get the sign bit of second operand. 345 SDValue SignBit = DAG.getNode( 346 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 347 DAG.getConstant(RSize - 1, dl, 348 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 349 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 350 351 // Shift right or sign-extend it if the two operands have different types. 352 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 353 if (SizeDiff > 0) { 354 SignBit = 355 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 356 DAG.getConstant(SizeDiff, dl, 357 TLI.getShiftAmountTy(SignBit.getValueType(), 358 DAG.getDataLayout()))); 359 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 360 } else if (SizeDiff < 0) { 361 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 362 SignBit = 363 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 364 DAG.getConstant(-SizeDiff, dl, 365 TLI.getShiftAmountTy(SignBit.getValueType(), 366 DAG.getDataLayout()))); 367 } 368 369 // Clear the sign bit of the first operand. 370 SDValue Mask = DAG.getNode( 371 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 372 DAG.getConstant(LSize - 1, dl, 373 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 374 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 375 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 376 377 // Or the value with the sign bit. 378 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 379 } 380 381 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { 382 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 383 RTLIB::COS_F32, 384 RTLIB::COS_F64, 385 RTLIB::COS_F80, 386 RTLIB::COS_F128, 387 RTLIB::COS_PPCF128)); 388 } 389 390 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { 391 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 392 RTLIB::DIV_F32, 393 RTLIB::DIV_F64, 394 RTLIB::DIV_F80, 395 RTLIB::DIV_F128, 396 RTLIB::DIV_PPCF128)); 397 } 398 399 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { 400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 401 RTLIB::EXP_F32, 402 RTLIB::EXP_F64, 403 RTLIB::EXP_F80, 404 RTLIB::EXP_F128, 405 RTLIB::EXP_PPCF128)); 406 } 407 408 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { 409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 410 RTLIB::EXP2_F32, 411 RTLIB::EXP2_F64, 412 RTLIB::EXP2_F80, 413 RTLIB::EXP2_F128, 414 RTLIB::EXP2_PPCF128)); 415 } 416 417 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { 418 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 419 RTLIB::FLOOR_F32, 420 RTLIB::FLOOR_F64, 421 RTLIB::FLOOR_F80, 422 RTLIB::FLOOR_F128, 423 RTLIB::FLOOR_PPCF128)); 424 } 425 426 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { 427 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 428 RTLIB::LOG_F32, 429 RTLIB::LOG_F64, 430 RTLIB::LOG_F80, 431 RTLIB::LOG_F128, 432 RTLIB::LOG_PPCF128)); 433 } 434 435 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { 436 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 437 RTLIB::LOG2_F32, 438 RTLIB::LOG2_F64, 439 RTLIB::LOG2_F80, 440 RTLIB::LOG2_F128, 441 RTLIB::LOG2_PPCF128)); 442 } 443 444 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { 445 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 446 RTLIB::LOG10_F32, 447 RTLIB::LOG10_F64, 448 RTLIB::LOG10_F80, 449 RTLIB::LOG10_F128, 450 RTLIB::LOG10_PPCF128)); 451 } 452 453 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { 454 bool IsStrict = N->isStrictFPOpcode(); 455 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 456 unsigned Offset = IsStrict ? 1 : 0; 457 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 458 GetSoftenedFloat(N->getOperand(1 + Offset)), 459 GetSoftenedFloat(N->getOperand(2 + Offset)) }; 460 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 461 TargetLowering::MakeLibCallOptions CallOptions; 462 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(), 463 N->getOperand(1 + Offset).getValueType(), 464 N->getOperand(2 + Offset).getValueType() }; 465 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 466 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, 467 GetFPLibCall(N->getValueType(0), 468 RTLIB::FMA_F32, 469 RTLIB::FMA_F64, 470 RTLIB::FMA_F80, 471 RTLIB::FMA_F128, 472 RTLIB::FMA_PPCF128), 473 NVT, Ops, CallOptions, SDLoc(N), Chain); 474 if (IsStrict) 475 ReplaceValueWith(SDValue(N, 1), Tmp.second); 476 return Tmp.first; 477 } 478 479 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { 480 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 481 RTLIB::MUL_F32, 482 RTLIB::MUL_F64, 483 RTLIB::MUL_F80, 484 RTLIB::MUL_F128, 485 RTLIB::MUL_PPCF128)); 486 } 487 488 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { 489 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 490 RTLIB::NEARBYINT_F32, 491 RTLIB::NEARBYINT_F64, 492 RTLIB::NEARBYINT_F80, 493 RTLIB::NEARBYINT_F128, 494 RTLIB::NEARBYINT_PPCF128)); 495 } 496 497 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { 498 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 499 SDLoc dl(N); 500 501 // Expand Y = FNEG(X) -> Y = X ^ sign mask 502 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); 503 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), 504 DAG.getConstant(SignMask, dl, NVT)); 505 } 506 507 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { 508 bool IsStrict = N->isStrictFPOpcode(); 509 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 510 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 511 512 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 513 514 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { 515 Op = GetPromotedFloat(Op); 516 // If the promotion did the FP_EXTEND to the destination type for us, 517 // there's nothing left to do here. 518 if (Op.getValueType() == N->getValueType(0)) 519 return BitConvertToInteger(Op); 520 } 521 522 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16 523 // -> f32, so proceed in two stages. Also, it's entirely possible for both 524 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather 525 // than FP16_TO_FP. 526 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) && 527 N->getValueType(0) != MVT::f32) { 528 if (IsStrict) { 529 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 530 { MVT::f32, MVT::Other }, { Chain, Op }); 531 Chain = Op.getValue(1); 532 } else { 533 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); 534 } 535 } 536 537 if (Op.getValueType() == MVT::bf16) 538 return SoftenFloatRes_BF16_TO_FP(N); 539 540 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); 541 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 542 TargetLowering::MakeLibCallOptions CallOptions; 543 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 544 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 545 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 546 CallOptions, SDLoc(N), 547 Chain); 548 if (IsStrict) 549 ReplaceValueWith(SDValue(N, 1), Tmp.second); 550 return Tmp.first; 551 } 552 553 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 554 // nodes? 555 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { 556 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 557 SDValue Op = N->getOperand(0); 558 TargetLowering::MakeLibCallOptions CallOptions; 559 EVT OpsVT[1] = { N->getOperand(0).getValueType() }; 560 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 561 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op, 562 CallOptions, SDLoc(N)).first; 563 if (N->getValueType(0) == MVT::f32) 564 return Res32; 565 566 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 567 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0)); 568 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 569 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first; 570 } 571 572 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 573 // nodes? 574 SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) { 575 assert(N->getValueType(0) == MVT::f32 && 576 "Can only soften BF16_TO_FP with f32 result"); 577 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 578 SDValue Op = N->getOperand(0); 579 SDLoc DL(N); 580 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT, 581 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op)); 582 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op, 583 DAG.getShiftAmountConstant(16, NVT, DL)); 584 return Res; 585 } 586 587 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { 588 bool IsStrict = N->isStrictFPOpcode(); 589 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 590 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 591 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 592 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); 593 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 594 TargetLowering::MakeLibCallOptions CallOptions; 595 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 596 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 597 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 598 CallOptions, SDLoc(N), 599 Chain); 600 if (IsStrict) 601 ReplaceValueWith(SDValue(N, 1), Tmp.second); 602 return Tmp.first; 603 } 604 605 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { 606 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 607 RTLIB::POW_F32, 608 RTLIB::POW_F64, 609 RTLIB::POW_F80, 610 RTLIB::POW_F128, 611 RTLIB::POW_PPCF128)); 612 } 613 614 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) { 615 bool IsStrict = N->isStrictFPOpcode(); 616 unsigned Offset = IsStrict ? 1 : 0; 617 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 || 618 N->getOperand(1 + Offset).getValueType() == MVT::i32) && 619 "Unsupported power type!"); 620 bool IsPowI = 621 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; 622 623 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0)) 624 : RTLIB::getLDEXP(N->getValueType(0)); 625 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 626 if (!TLI.getLibcallName(LC)) { 627 // Some targets don't have a powi libcall; use pow instead. 628 // FIXME: Implement this if some target needs it. 629 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow"); 630 return DAG.getUNDEF(N->getValueType(0)); 631 } 632 633 if (DAG.getLibInfo().getIntSize() != 634 N->getOperand(1 + Offset).getValueType().getSizeInBits()) { 635 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI 636 // would use the wrong type for the argument. 637 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); 638 return DAG.getUNDEF(N->getValueType(0)); 639 } 640 641 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 642 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 643 N->getOperand(1 + Offset) }; 644 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 645 TargetLowering::MakeLibCallOptions CallOptions; 646 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 647 N->getOperand(1 + Offset).getValueType() }; 648 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 649 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 650 CallOptions, SDLoc(N), 651 Chain); 652 if (IsStrict) 653 ReplaceValueWith(SDValue(N, 1), Tmp.second); 654 return Tmp.first; 655 } 656 657 SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) { 658 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp"); 659 EVT VT0 = N->getValueType(0); 660 EVT VT1 = N->getValueType(1); 661 RTLIB::Libcall LC = RTLIB::getFREXP(VT0); 662 663 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) { 664 // If the exponent does not match with sizeof(int) a libcall would use the 665 // wrong type for the argument. 666 // TODO: Should be able to handle mismatches. 667 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)"); 668 return DAG.getUNDEF(N->getValueType(0)); 669 } 670 671 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0); 672 SDValue StackSlot = DAG.CreateStackTemporary(VT1); 673 674 SDLoc DL(N); 675 676 TargetLowering::MakeLibCallOptions CallOptions; 677 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot}; 678 EVT OpsVT[2] = {VT0, StackSlot.getValueType()}; 679 680 // TODO: setTypeListBeforeSoften can't properly express multiple return types, 681 // but we only really need to handle the 0th one for softening anyway. 682 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true); 683 684 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL, 685 /*Chain=*/SDValue()); 686 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex(); 687 auto PtrInfo = 688 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 689 690 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo); 691 692 ReplaceValueWith(SDValue(N, 1), LoadExp); 693 return ReturnVal; 694 } 695 696 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { 697 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 698 RTLIB::REM_F32, 699 RTLIB::REM_F64, 700 RTLIB::REM_F80, 701 RTLIB::REM_F128, 702 RTLIB::REM_PPCF128)); 703 } 704 705 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { 706 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 707 RTLIB::RINT_F32, 708 RTLIB::RINT_F64, 709 RTLIB::RINT_F80, 710 RTLIB::RINT_F128, 711 RTLIB::RINT_PPCF128)); 712 } 713 714 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { 715 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 716 RTLIB::ROUND_F32, 717 RTLIB::ROUND_F64, 718 RTLIB::ROUND_F80, 719 RTLIB::ROUND_F128, 720 RTLIB::ROUND_PPCF128)); 721 } 722 723 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { 724 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 725 RTLIB::ROUNDEVEN_F32, 726 RTLIB::ROUNDEVEN_F64, 727 RTLIB::ROUNDEVEN_F80, 728 RTLIB::ROUNDEVEN_F128, 729 RTLIB::ROUNDEVEN_PPCF128)); 730 } 731 732 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { 733 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 734 RTLIB::SIN_F32, 735 RTLIB::SIN_F64, 736 RTLIB::SIN_F80, 737 RTLIB::SIN_F128, 738 RTLIB::SIN_PPCF128)); 739 } 740 741 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { 742 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 743 RTLIB::SQRT_F32, 744 RTLIB::SQRT_F64, 745 RTLIB::SQRT_F80, 746 RTLIB::SQRT_F128, 747 RTLIB::SQRT_PPCF128)); 748 } 749 750 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { 751 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 752 RTLIB::SUB_F32, 753 RTLIB::SUB_F64, 754 RTLIB::SUB_F80, 755 RTLIB::SUB_F128, 756 RTLIB::SUB_PPCF128)); 757 } 758 759 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { 760 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 761 RTLIB::TRUNC_F32, 762 RTLIB::TRUNC_F64, 763 RTLIB::TRUNC_F80, 764 RTLIB::TRUNC_F128, 765 RTLIB::TRUNC_PPCF128)); 766 } 767 768 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { 769 LoadSDNode *L = cast<LoadSDNode>(N); 770 EVT VT = N->getValueType(0); 771 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 772 SDLoc dl(N); 773 774 auto MMOFlags = 775 L->getMemOperand()->getFlags() & 776 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 777 SDValue NewL; 778 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 779 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, 780 L->getChain(), L->getBasePtr(), L->getOffset(), 781 L->getPointerInfo(), NVT, L->getOriginalAlign(), 782 MMOFlags, L->getAAInfo()); 783 // Legalized the chain result - switch anything that used the old chain to 784 // use the new one. 785 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 786 return NewL; 787 } 788 789 // Do a non-extending load followed by FP_EXTEND. 790 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(), 791 dl, L->getChain(), L->getBasePtr(), L->getOffset(), 792 L->getPointerInfo(), L->getMemoryVT(), 793 L->getOriginalAlign(), MMOFlags, L->getAAInfo()); 794 // Legalized the chain result - switch anything that used the old chain to 795 // use the new one. 796 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 797 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL); 798 return BitConvertToInteger(ExtendNode); 799 } 800 801 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { 802 SDValue LHS = GetSoftenedFloat(N->getOperand(1)); 803 SDValue RHS = GetSoftenedFloat(N->getOperand(2)); 804 return DAG.getSelect(SDLoc(N), 805 LHS.getValueType(), N->getOperand(0), LHS, RHS); 806 } 807 808 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { 809 SDValue LHS = GetSoftenedFloat(N->getOperand(2)); 810 SDValue RHS = GetSoftenedFloat(N->getOperand(3)); 811 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 812 LHS.getValueType(), N->getOperand(0), 813 N->getOperand(1), LHS, RHS, N->getOperand(4)); 814 } 815 816 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { 817 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 818 N->getValueType(0))); 819 } 820 821 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { 822 SDValue Chain = N->getOperand(0); // Get the chain. 823 SDValue Ptr = N->getOperand(1); // Get the pointer. 824 EVT VT = N->getValueType(0); 825 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 826 SDLoc dl(N); 827 828 SDValue NewVAARG; 829 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), 830 N->getConstantOperandVal(3)); 831 832 // Legalized the chain result - switch anything that used the old chain to 833 // use the new one. 834 if (N != NewVAARG.getValue(1).getNode()) 835 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1)); 836 return NewVAARG; 837 } 838 839 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { 840 bool IsStrict = N->isStrictFPOpcode(); 841 bool Signed = N->getOpcode() == ISD::SINT_TO_FP || 842 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 843 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 844 EVT RVT = N->getValueType(0); 845 EVT NVT = EVT(); 846 SDLoc dl(N); 847 848 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to 849 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly 850 // match. Look for an appropriate libcall. 851 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 852 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 853 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { 854 NVT = (MVT::SimpleValueType)t; 855 // The source needs to big enough to hold the operand. 856 if (NVT.bitsGE(SVT)) 857 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT); 858 } 859 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 860 861 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 862 // Sign/zero extend the argument if the libcall takes a larger type. 863 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 864 NVT, N->getOperand(IsStrict ? 1 : 0)); 865 TargetLowering::MakeLibCallOptions CallOptions; 866 CallOptions.setSExt(Signed); 867 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 868 std::pair<SDValue, SDValue> Tmp = 869 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), 870 Op, CallOptions, dl, Chain); 871 872 if (IsStrict) 873 ReplaceValueWith(SDValue(N, 1), Tmp.second); 874 return Tmp.first; 875 } 876 877 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) { 878 // Expand and soften recursively. 879 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 880 return SDValue(); 881 } 882 883 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) { 884 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 885 return SDValue(); 886 } 887 888 //===----------------------------------------------------------------------===// 889 // Convert Float Operand to Integer 890 //===----------------------------------------------------------------------===// 891 892 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { 893 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG); 894 dbgs() << "\n"); 895 SDValue Res = SDValue(); 896 897 switch (N->getOpcode()) { 898 default: 899 #ifndef NDEBUG 900 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": "; 901 N->dump(&DAG); dbgs() << "\n"; 902 #endif 903 report_fatal_error("Do not know how to soften this operator's operand!"); 904 905 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; 906 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; 907 case ISD::STRICT_FP_TO_FP16: 908 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes 909 case ISD::FP_TO_BF16: 910 case ISD::STRICT_FP_ROUND: 911 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; 912 case ISD::STRICT_FP_TO_SINT: 913 case ISD::STRICT_FP_TO_UINT: 914 case ISD::FP_TO_SINT: 915 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; 916 case ISD::FP_TO_SINT_SAT: 917 case ISD::FP_TO_UINT_SAT: 918 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; 919 case ISD::STRICT_LROUND: 920 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; 921 case ISD::STRICT_LLROUND: 922 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; 923 case ISD::STRICT_LRINT: 924 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; 925 case ISD::STRICT_LLRINT: 926 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; 927 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; 928 case ISD::STRICT_FSETCC: 929 case ISD::STRICT_FSETCCS: 930 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; 931 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; 932 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; 933 } 934 935 // If the result is null, the sub-method took care of registering results etc. 936 if (!Res.getNode()) return false; 937 938 // If the result is N, the sub-method updated N in place. Tell the legalizer 939 // core about this to re-analyze. 940 if (Res.getNode() == N) 941 return true; 942 943 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 944 "Invalid operand softening"); 945 946 ReplaceValueWith(SDValue(N, 0), Res); 947 return false; 948 } 949 950 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { 951 SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); 952 953 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 954 } 955 956 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { 957 // We actually deal with the partially-softened FP_TO_FP16 node too, which 958 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. 959 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || 960 N->getOpcode() == ISD::STRICT_FP_TO_FP16 || 961 N->getOpcode() == ISD::FP_TO_BF16 || 962 N->getOpcode() == ISD::STRICT_FP_ROUND); 963 964 bool IsStrict = N->isStrictFPOpcode(); 965 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 966 EVT SVT = Op.getValueType(); 967 EVT RVT = N->getValueType(0); 968 EVT FloatRVT = RVT; 969 if (N->getOpcode() == ISD::FP_TO_FP16 || 970 N->getOpcode() == ISD::STRICT_FP_TO_FP16) 971 FloatRVT = MVT::f16; 972 else if (N->getOpcode() == ISD::FP_TO_BF16) 973 FloatRVT = MVT::bf16; 974 975 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT); 976 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 977 978 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 979 Op = GetSoftenedFloat(Op); 980 TargetLowering::MakeLibCallOptions CallOptions; 981 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 982 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 983 CallOptions, SDLoc(N), 984 Chain); 985 if (IsStrict) { 986 ReplaceValueWith(SDValue(N, 1), Tmp.second); 987 ReplaceValueWith(SDValue(N, 0), Tmp.first); 988 return SDValue(); 989 } 990 return Tmp.first; 991 } 992 993 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 994 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 995 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 996 997 EVT VT = NewLHS.getValueType(); 998 NewLHS = GetSoftenedFloat(NewLHS); 999 NewRHS = GetSoftenedFloat(NewRHS); 1000 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1001 N->getOperand(2), N->getOperand(3)); 1002 1003 // If softenSetCCOperands returned a scalar, we need to compare the result 1004 // against zero to select between true and false values. 1005 if (!NewRHS.getNode()) { 1006 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1007 CCCode = ISD::SETNE; 1008 } 1009 1010 // Update N to have the operands specified. 1011 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1012 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1013 N->getOperand(4)), 1014 0); 1015 } 1016 1017 // Even if the result type is legal, no libcall may exactly match. (e.g. We 1018 // don't have FP-i8 conversions) This helper method looks for an appropriate 1019 // promoted libcall. 1020 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, 1021 bool Signed) { 1022 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1023 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 1024 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 1025 ++IntVT) { 1026 Promoted = (MVT::SimpleValueType)IntVT; 1027 // The type needs to big enough to hold the result. 1028 if (Promoted.bitsGE(RetVT)) 1029 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted) 1030 : RTLIB::getFPTOUINT(SrcVT, Promoted); 1031 } 1032 return LC; 1033 } 1034 1035 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { 1036 bool IsStrict = N->isStrictFPOpcode(); 1037 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 1038 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 1039 1040 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1041 EVT SVT = Op.getValueType(); 1042 EVT RVT = N->getValueType(0); 1043 EVT NVT = EVT(); 1044 SDLoc dl(N); 1045 1046 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to 1047 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly 1048 // match, eg. we don't have fp -> i8 conversions. 1049 // Look for an appropriate libcall. 1050 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed); 1051 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 1052 "Unsupported FP_TO_XINT!"); 1053 1054 Op = GetSoftenedFloat(Op); 1055 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1056 TargetLowering::MakeLibCallOptions CallOptions; 1057 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 1058 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1059 CallOptions, dl, Chain); 1060 1061 // Truncate the result if the libcall returns a larger type. 1062 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first); 1063 1064 if (!IsStrict) 1065 return Res; 1066 1067 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1068 ReplaceValueWith(SDValue(N, 0), Res); 1069 return SDValue(); 1070 } 1071 1072 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { 1073 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 1074 return Res; 1075 } 1076 1077 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { 1078 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1079 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1080 1081 EVT VT = NewLHS.getValueType(); 1082 NewLHS = GetSoftenedFloat(NewLHS); 1083 NewRHS = GetSoftenedFloat(NewRHS); 1084 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1085 N->getOperand(0), N->getOperand(1)); 1086 1087 // If softenSetCCOperands returned a scalar, we need to compare the result 1088 // against zero to select between true and false values. 1089 if (!NewRHS.getNode()) { 1090 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1091 CCCode = ISD::SETNE; 1092 } 1093 1094 // Update N to have the operands specified. 1095 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1096 N->getOperand(2), N->getOperand(3), 1097 DAG.getCondCode(CCCode)), 1098 0); 1099 } 1100 1101 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { 1102 bool IsStrict = N->isStrictFPOpcode(); 1103 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); 1104 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1); 1105 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1106 ISD::CondCode CCCode = 1107 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1108 1109 EVT VT = Op0.getValueType(); 1110 SDValue NewLHS = GetSoftenedFloat(Op0); 1111 SDValue NewRHS = GetSoftenedFloat(Op1); 1112 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1, 1113 Chain, N->getOpcode() == ISD::STRICT_FSETCCS); 1114 1115 // Update N to have the operands specified. 1116 if (NewRHS.getNode()) { 1117 if (IsStrict) 1118 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS, 1119 NewRHS, DAG.getCondCode(CCCode)); 1120 else 1121 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1122 DAG.getCondCode(CCCode)), 0); 1123 } 1124 1125 // Otherwise, softenSetCCOperands returned a scalar, use it. 1126 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && 1127 "Unexpected setcc expansion!"); 1128 1129 if (IsStrict) { 1130 ReplaceValueWith(SDValue(N, 0), NewLHS); 1131 ReplaceValueWith(SDValue(N, 1), Chain); 1132 return SDValue(); 1133 } 1134 return NewLHS; 1135 } 1136 1137 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { 1138 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1139 assert(OpNo == 1 && "Can only soften the stored value!"); 1140 StoreSDNode *ST = cast<StoreSDNode>(N); 1141 SDValue Val = ST->getValue(); 1142 SDLoc dl(N); 1143 1144 if (ST->isTruncatingStore()) 1145 // Do an FP_ROUND followed by a non-truncating store. 1146 Val = BitConvertToInteger( 1147 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val, 1148 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); 1149 else 1150 Val = GetSoftenedFloat(Val); 1151 1152 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(), 1153 ST->getMemOperand()); 1154 } 1155 1156 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { 1157 SDValue LHS = N->getOperand(0); 1158 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 1159 SDLoc dl(N); 1160 1161 EVT LVT = LHS.getValueType(); 1162 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits()); 1163 EVT RVT = RHS.getValueType(); 1164 1165 unsigned LSize = LVT.getSizeInBits(); 1166 unsigned RSize = RVT.getSizeInBits(); 1167 1168 // Shift right or sign-extend it if the two operands have different types. 1169 int SizeDiff = RSize - LSize; 1170 if (SizeDiff > 0) { 1171 RHS = 1172 DAG.getNode(ISD::SRL, dl, RVT, RHS, 1173 DAG.getConstant(SizeDiff, dl, 1174 TLI.getShiftAmountTy(RHS.getValueType(), 1175 DAG.getDataLayout()))); 1176 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS); 1177 } else if (SizeDiff < 0) { 1178 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS); 1179 RHS = 1180 DAG.getNode(ISD::SHL, dl, ILVT, RHS, 1181 DAG.getConstant(-SizeDiff, dl, 1182 TLI.getShiftAmountTy(RHS.getValueType(), 1183 DAG.getDataLayout()))); 1184 } 1185 1186 RHS = DAG.getBitcast(LVT, RHS); 1187 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS); 1188 } 1189 1190 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { 1191 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1192 bool IsStrict = N->isStrictFPOpcode(); 1193 unsigned Offset = IsStrict ? 1 : 0; 1194 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 1195 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1196 TargetLowering::MakeLibCallOptions CallOptions; 1197 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 1198 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 1199 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1200 CallOptions, SDLoc(N), 1201 Chain); 1202 if (IsStrict) { 1203 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1204 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1205 return SDValue(); 1206 } 1207 1208 return Tmp.first; 1209 } 1210 1211 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { 1212 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1213 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1214 RTLIB::LROUND_F32, 1215 RTLIB::LROUND_F64, 1216 RTLIB::LROUND_F80, 1217 RTLIB::LROUND_F128, 1218 RTLIB::LROUND_PPCF128)); 1219 } 1220 1221 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { 1222 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1223 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1224 RTLIB::LLROUND_F32, 1225 RTLIB::LLROUND_F64, 1226 RTLIB::LLROUND_F80, 1227 RTLIB::LLROUND_F128, 1228 RTLIB::LLROUND_PPCF128)); 1229 } 1230 1231 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { 1232 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1233 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1234 RTLIB::LRINT_F32, 1235 RTLIB::LRINT_F64, 1236 RTLIB::LRINT_F80, 1237 RTLIB::LRINT_F128, 1238 RTLIB::LRINT_PPCF128)); 1239 } 1240 1241 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { 1242 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1243 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1244 RTLIB::LLRINT_F32, 1245 RTLIB::LLRINT_F64, 1246 RTLIB::LLRINT_F80, 1247 RTLIB::LLRINT_F128, 1248 RTLIB::LLRINT_PPCF128)); 1249 } 1250 1251 //===----------------------------------------------------------------------===// 1252 // Float Result Expansion 1253 //===----------------------------------------------------------------------===// 1254 1255 /// ExpandFloatResult - This method is called when the specified result of the 1256 /// specified node is found to need expansion. At this point, the node may also 1257 /// have invalid operands or may have other results that need promotion, we just 1258 /// know that (at least) one result needs expansion. 1259 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { 1260 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n"); 1261 SDValue Lo, Hi; 1262 Lo = Hi = SDValue(); 1263 1264 // See if the target wants to custom expand this node. 1265 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1266 return; 1267 1268 switch (N->getOpcode()) { 1269 default: 1270 #ifndef NDEBUG 1271 dbgs() << "ExpandFloatResult #" << ResNo << ": "; 1272 N->dump(&DAG); dbgs() << "\n"; 1273 #endif 1274 report_fatal_error("Do not know how to expand the result of this " 1275 "operator!"); 1276 1277 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1278 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 1279 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1280 1281 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1282 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1283 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1284 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1285 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1286 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1287 1288 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; 1289 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; 1290 case ISD::STRICT_FMINNUM: 1291 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; 1292 case ISD::STRICT_FMAXNUM: 1293 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; 1294 case ISD::STRICT_FADD: 1295 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; 1296 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; 1297 case ISD::STRICT_FCEIL: 1298 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; 1299 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; 1300 case ISD::STRICT_FCOS: 1301 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; 1302 case ISD::STRICT_FDIV: 1303 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; 1304 case ISD::STRICT_FEXP: 1305 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; 1306 case ISD::STRICT_FEXP2: 1307 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; 1308 case ISD::STRICT_FFLOOR: 1309 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; 1310 case ISD::STRICT_FLOG: 1311 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; 1312 case ISD::STRICT_FLOG2: 1313 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; 1314 case ISD::STRICT_FLOG10: 1315 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; 1316 case ISD::STRICT_FMA: 1317 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; 1318 case ISD::STRICT_FMUL: 1319 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; 1320 case ISD::STRICT_FNEARBYINT: 1321 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; 1322 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; 1323 case ISD::STRICT_FP_EXTEND: 1324 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; 1325 case ISD::STRICT_FPOW: 1326 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; 1327 case ISD::STRICT_FPOWI: 1328 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; 1329 case ISD::FLDEXP: 1330 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break; 1331 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; 1332 case ISD::STRICT_FRINT: 1333 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; 1334 case ISD::STRICT_FROUND: 1335 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; 1336 case ISD::STRICT_FROUNDEVEN: 1337 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; 1338 case ISD::STRICT_FSIN: 1339 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; 1340 case ISD::STRICT_FSQRT: 1341 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; 1342 case ISD::STRICT_FSUB: 1343 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; 1344 case ISD::STRICT_FTRUNC: 1345 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; 1346 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; 1347 case ISD::STRICT_SINT_TO_FP: 1348 case ISD::STRICT_UINT_TO_FP: 1349 case ISD::SINT_TO_FP: 1350 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; 1351 case ISD::STRICT_FREM: 1352 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; 1353 } 1354 1355 // If Lo/Hi is null, the sub-method took care of registering results etc. 1356 if (Lo.getNode()) 1357 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi); 1358 } 1359 1360 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, 1361 SDValue &Hi) { 1362 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1363 assert(NVT.getSizeInBits() == 64 && 1364 "Do not know how to expand this float constant!"); 1365 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); 1366 SDLoc dl(N); 1367 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1368 APInt(64, C.getRawData()[1])), 1369 dl, NVT); 1370 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1371 APInt(64, C.getRawData()[0])), 1372 dl, NVT); 1373 } 1374 1375 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, 1376 SDValue &Lo, SDValue &Hi) { 1377 bool IsStrict = N->isStrictFPOpcode(); 1378 unsigned Offset = IsStrict ? 1 : 0; 1379 SDValue Op = N->getOperand(0 + Offset); 1380 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1381 TargetLowering::MakeLibCallOptions CallOptions; 1382 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1383 Op, CallOptions, SDLoc(N), 1384 Chain); 1385 if (IsStrict) 1386 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1387 GetPairElements(Tmp.first, Lo, Hi); 1388 } 1389 1390 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, 1391 SDValue &Lo, SDValue &Hi) { 1392 bool IsStrict = N->isStrictFPOpcode(); 1393 unsigned Offset = IsStrict ? 1 : 0; 1394 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) }; 1395 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1396 TargetLowering::MakeLibCallOptions CallOptions; 1397 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1398 Ops, CallOptions, SDLoc(N), 1399 Chain); 1400 if (IsStrict) 1401 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1402 GetPairElements(Tmp.first, Lo, Hi); 1403 } 1404 1405 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, 1406 SDValue &Hi) { 1407 assert(N->getValueType(0) == MVT::ppcf128 && 1408 "Logic only correct for ppcf128!"); 1409 SDLoc dl(N); 1410 SDValue Tmp; 1411 GetExpandedFloat(N->getOperand(0), Lo, Tmp); 1412 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp); 1413 // Lo = Hi==fabs(Hi) ? Lo : -Lo; 1414 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo, 1415 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo), 1416 ISD::SETEQ); 1417 } 1418 1419 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, 1420 SDValue &Hi) { 1421 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1422 RTLIB::FMIN_F32, RTLIB::FMIN_F64, 1423 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 1424 RTLIB::FMIN_PPCF128), Lo, Hi); 1425 } 1426 1427 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, 1428 SDValue &Hi) { 1429 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1430 RTLIB::FMAX_F32, RTLIB::FMAX_F64, 1431 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 1432 RTLIB::FMAX_PPCF128), Lo, Hi); 1433 } 1434 1435 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, 1436 SDValue &Hi) { 1437 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1438 RTLIB::ADD_F32, RTLIB::ADD_F64, 1439 RTLIB::ADD_F80, RTLIB::ADD_F128, 1440 RTLIB::ADD_PPCF128), Lo, Hi); 1441 } 1442 1443 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, 1444 SDValue &Hi) { 1445 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32, 1446 RTLIB::CBRT_F64, RTLIB::CBRT_F80, 1447 RTLIB::CBRT_F128, 1448 RTLIB::CBRT_PPCF128), Lo, Hi); 1449 } 1450 1451 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, 1452 SDValue &Lo, SDValue &Hi) { 1453 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1454 RTLIB::CEIL_F32, RTLIB::CEIL_F64, 1455 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 1456 RTLIB::CEIL_PPCF128), Lo, Hi); 1457 } 1458 1459 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, 1460 SDValue &Lo, SDValue &Hi) { 1461 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1462 RTLIB::COPYSIGN_F32, 1463 RTLIB::COPYSIGN_F64, 1464 RTLIB::COPYSIGN_F80, 1465 RTLIB::COPYSIGN_F128, 1466 RTLIB::COPYSIGN_PPCF128), Lo, Hi); 1467 } 1468 1469 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, 1470 SDValue &Lo, SDValue &Hi) { 1471 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1472 RTLIB::COS_F32, RTLIB::COS_F64, 1473 RTLIB::COS_F80, RTLIB::COS_F128, 1474 RTLIB::COS_PPCF128), Lo, Hi); 1475 } 1476 1477 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, 1478 SDValue &Hi) { 1479 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1480 RTLIB::DIV_F32, 1481 RTLIB::DIV_F64, 1482 RTLIB::DIV_F80, 1483 RTLIB::DIV_F128, 1484 RTLIB::DIV_PPCF128), Lo, Hi); 1485 } 1486 1487 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, 1488 SDValue &Lo, SDValue &Hi) { 1489 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1490 RTLIB::EXP_F32, RTLIB::EXP_F64, 1491 RTLIB::EXP_F80, RTLIB::EXP_F128, 1492 RTLIB::EXP_PPCF128), Lo, Hi); 1493 } 1494 1495 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, 1496 SDValue &Lo, SDValue &Hi) { 1497 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1498 RTLIB::EXP2_F32, RTLIB::EXP2_F64, 1499 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 1500 RTLIB::EXP2_PPCF128), Lo, Hi); 1501 } 1502 1503 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, 1504 SDValue &Lo, SDValue &Hi) { 1505 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1506 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 1507 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 1508 RTLIB::FLOOR_PPCF128), Lo, Hi); 1509 } 1510 1511 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, 1512 SDValue &Lo, SDValue &Hi) { 1513 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1514 RTLIB::LOG_F32, RTLIB::LOG_F64, 1515 RTLIB::LOG_F80, RTLIB::LOG_F128, 1516 RTLIB::LOG_PPCF128), Lo, Hi); 1517 } 1518 1519 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, 1520 SDValue &Lo, SDValue &Hi) { 1521 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1522 RTLIB::LOG2_F32, RTLIB::LOG2_F64, 1523 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 1524 RTLIB::LOG2_PPCF128), Lo, Hi); 1525 } 1526 1527 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, 1528 SDValue &Lo, SDValue &Hi) { 1529 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1530 RTLIB::LOG10_F32, RTLIB::LOG10_F64, 1531 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 1532 RTLIB::LOG10_PPCF128), Lo, Hi); 1533 } 1534 1535 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, 1536 SDValue &Hi) { 1537 bool IsStrict = N->isStrictFPOpcode(); 1538 unsigned Offset = IsStrict ? 1 : 0; 1539 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset), 1540 N->getOperand(2 + Offset) }; 1541 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1542 TargetLowering::MakeLibCallOptions CallOptions; 1543 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), 1544 RTLIB::FMA_F32, 1545 RTLIB::FMA_F64, 1546 RTLIB::FMA_F80, 1547 RTLIB::FMA_F128, 1548 RTLIB::FMA_PPCF128), 1549 N->getValueType(0), Ops, CallOptions, 1550 SDLoc(N), Chain); 1551 if (IsStrict) 1552 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1553 GetPairElements(Tmp.first, Lo, Hi); 1554 } 1555 1556 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, 1557 SDValue &Hi) { 1558 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1559 RTLIB::MUL_F32, 1560 RTLIB::MUL_F64, 1561 RTLIB::MUL_F80, 1562 RTLIB::MUL_F128, 1563 RTLIB::MUL_PPCF128), Lo, Hi); 1564 } 1565 1566 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, 1567 SDValue &Lo, SDValue &Hi) { 1568 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1569 RTLIB::NEARBYINT_F32, 1570 RTLIB::NEARBYINT_F64, 1571 RTLIB::NEARBYINT_F80, 1572 RTLIB::NEARBYINT_F128, 1573 RTLIB::NEARBYINT_PPCF128), Lo, Hi); 1574 } 1575 1576 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, 1577 SDValue &Hi) { 1578 SDLoc dl(N); 1579 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1580 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo); 1581 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); 1582 } 1583 1584 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, 1585 SDValue &Hi) { 1586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1587 SDLoc dl(N); 1588 bool IsStrict = N->isStrictFPOpcode(); 1589 1590 SDValue Chain; 1591 if (IsStrict) { 1592 // If the expanded type is the same as the input type, just bypass the node. 1593 if (NVT == N->getOperand(1).getValueType()) { 1594 Hi = N->getOperand(1); 1595 Chain = N->getOperand(0); 1596 } else { 1597 // Other we need to extend. 1598 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, 1599 { N->getOperand(0), N->getOperand(1) }); 1600 Chain = Hi.getValue(1); 1601 } 1602 } else { 1603 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0)); 1604 } 1605 1606 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1607 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1608 1609 if (IsStrict) 1610 ReplaceValueWith(SDValue(N, 1), Chain); 1611 } 1612 1613 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, 1614 SDValue &Lo, SDValue &Hi) { 1615 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1616 RTLIB::POW_F32, RTLIB::POW_F64, 1617 RTLIB::POW_F80, RTLIB::POW_F128, 1618 RTLIB::POW_PPCF128), Lo, Hi); 1619 } 1620 1621 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, 1622 SDValue &Lo, SDValue &Hi) { 1623 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi); 1624 } 1625 1626 void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo, 1627 SDValue &Hi) { 1628 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi); 1629 } 1630 1631 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, 1632 SDValue &Lo, SDValue &Hi) { 1633 assert(N->getValueType(0) == MVT::ppcf128 && 1634 "Logic only correct for ppcf128!"); 1635 1636 SDLoc dl(N); 1637 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1638 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo); 1639 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi); 1640 } 1641 1642 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, 1643 SDValue &Lo, SDValue &Hi) { 1644 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1645 RTLIB::REM_F32, RTLIB::REM_F64, 1646 RTLIB::REM_F80, RTLIB::REM_F128, 1647 RTLIB::REM_PPCF128), Lo, Hi); 1648 } 1649 1650 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, 1651 SDValue &Lo, SDValue &Hi) { 1652 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1653 RTLIB::RINT_F32, RTLIB::RINT_F64, 1654 RTLIB::RINT_F80, RTLIB::RINT_F128, 1655 RTLIB::RINT_PPCF128), Lo, Hi); 1656 } 1657 1658 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, 1659 SDValue &Lo, SDValue &Hi) { 1660 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1661 RTLIB::ROUND_F32, 1662 RTLIB::ROUND_F64, 1663 RTLIB::ROUND_F80, 1664 RTLIB::ROUND_F128, 1665 RTLIB::ROUND_PPCF128), Lo, Hi); 1666 } 1667 1668 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, 1669 SDValue &Lo, SDValue &Hi) { 1670 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1671 RTLIB::ROUNDEVEN_F32, 1672 RTLIB::ROUNDEVEN_F64, 1673 RTLIB::ROUNDEVEN_F80, 1674 RTLIB::ROUNDEVEN_F128, 1675 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); 1676 } 1677 1678 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, 1679 SDValue &Lo, SDValue &Hi) { 1680 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1681 RTLIB::SIN_F32, RTLIB::SIN_F64, 1682 RTLIB::SIN_F80, RTLIB::SIN_F128, 1683 RTLIB::SIN_PPCF128), Lo, Hi); 1684 } 1685 1686 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, 1687 SDValue &Lo, SDValue &Hi) { 1688 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1689 RTLIB::SQRT_F32, RTLIB::SQRT_F64, 1690 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 1691 RTLIB::SQRT_PPCF128), Lo, Hi); 1692 } 1693 1694 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, 1695 SDValue &Hi) { 1696 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1697 RTLIB::SUB_F32, 1698 RTLIB::SUB_F64, 1699 RTLIB::SUB_F80, 1700 RTLIB::SUB_F128, 1701 RTLIB::SUB_PPCF128), Lo, Hi); 1702 } 1703 1704 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, 1705 SDValue &Lo, SDValue &Hi) { 1706 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1707 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 1708 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 1709 RTLIB::TRUNC_PPCF128), Lo, Hi); 1710 } 1711 1712 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, 1713 SDValue &Hi) { 1714 if (ISD::isNormalLoad(N)) { 1715 ExpandRes_NormalLoad(N, Lo, Hi); 1716 return; 1717 } 1718 1719 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1720 LoadSDNode *LD = cast<LoadSDNode>(N); 1721 SDValue Chain = LD->getChain(); 1722 SDValue Ptr = LD->getBasePtr(); 1723 SDLoc dl(N); 1724 1725 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1726 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1727 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1728 1729 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr, 1730 LD->getMemoryVT(), LD->getMemOperand()); 1731 1732 // Remember the chain. 1733 Chain = Hi.getValue(1); 1734 1735 // The low part is zero. 1736 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1737 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1738 1739 // Modified the chain - switch anything that used the old chain to use the 1740 // new one. 1741 ReplaceValueWith(SDValue(LD, 1), Chain); 1742 } 1743 1744 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, 1745 SDValue &Hi) { 1746 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!"); 1747 EVT VT = N->getValueType(0); 1748 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1749 bool Strict = N->isStrictFPOpcode(); 1750 SDValue Src = N->getOperand(Strict ? 1 : 0); 1751 EVT SrcVT = Src.getValueType(); 1752 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP || 1753 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 1754 SDLoc dl(N); 1755 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode(); 1756 1757 // TODO: Any other flags to propagate? 1758 SDNodeFlags Flags; 1759 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept()); 1760 1761 // First do an SINT_TO_FP, whether the original was signed or unsigned. 1762 // When promoting partial word types to i32 we must honor the signedness, 1763 // though. 1764 if (SrcVT.bitsLE(MVT::i32)) { 1765 // The integer can be represented exactly in an f64. 1766 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1767 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1768 if (Strict) { 1769 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), 1770 {Chain, Src}, Flags); 1771 Chain = Hi.getValue(1); 1772 } else 1773 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src); 1774 } else { 1775 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1776 if (SrcVT.bitsLE(MVT::i64)) { 1777 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1778 MVT::i64, Src); 1779 LC = RTLIB::SINTTOFP_I64_PPCF128; 1780 } else if (SrcVT.bitsLE(MVT::i128)) { 1781 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); 1782 LC = RTLIB::SINTTOFP_I128_PPCF128; 1783 } 1784 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 1785 1786 TargetLowering::MakeLibCallOptions CallOptions; 1787 CallOptions.setSExt(true); 1788 std::pair<SDValue, SDValue> Tmp = 1789 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); 1790 if (Strict) 1791 Chain = Tmp.second; 1792 GetPairElements(Tmp.first, Lo, Hi); 1793 } 1794 1795 // No need to complement for unsigned 32-bit integers 1796 if (isSigned || SrcVT.bitsLE(MVT::i32)) { 1797 if (Strict) 1798 ReplaceValueWith(SDValue(N, 1), Chain); 1799 1800 return; 1801 } 1802 1803 // Unsigned - fix up the SINT_TO_FP value just calculated. 1804 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully 1805 // keep semantics correctness if the integer is not exactly representable 1806 // here. See ExpandLegalINT_TO_FP. 1807 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 1808 SrcVT = Src.getValueType(); 1809 1810 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. 1811 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 1812 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 1813 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; 1814 ArrayRef<uint64_t> Parts; 1815 1816 switch (SrcVT.getSimpleVT().SimpleTy) { 1817 default: 1818 llvm_unreachable("Unsupported UINT_TO_FP!"); 1819 case MVT::i32: 1820 Parts = TwoE32; 1821 break; 1822 case MVT::i64: 1823 Parts = TwoE64; 1824 break; 1825 case MVT::i128: 1826 Parts = TwoE128; 1827 break; 1828 } 1829 1830 // TODO: Are there other fast-math-flags to propagate to this FADD? 1831 SDValue NewLo = DAG.getConstantFP( 1832 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128); 1833 if (Strict) { 1834 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other), 1835 {Chain, Hi, NewLo}, Flags); 1836 Chain = Lo.getValue(1); 1837 ReplaceValueWith(SDValue(N, 1), Chain); 1838 } else 1839 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo); 1840 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT), 1841 Lo, Hi, ISD::SETLT); 1842 GetPairElements(Lo, Lo, Hi); 1843 } 1844 1845 1846 //===----------------------------------------------------------------------===// 1847 // Float Operand Expansion 1848 //===----------------------------------------------------------------------===// 1849 1850 /// ExpandFloatOperand - This method is called when the specified operand of the 1851 /// specified node is found to need expansion. At this point, all of the result 1852 /// types of the node are known to be legal, but other operands of the node may 1853 /// need promotion or expansion as well as the specified one. 1854 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { 1855 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n"); 1856 SDValue Res = SDValue(); 1857 1858 // See if the target wants to custom expand this node. 1859 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1860 return false; 1861 1862 switch (N->getOpcode()) { 1863 default: 1864 #ifndef NDEBUG 1865 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": "; 1866 N->dump(&DAG); dbgs() << "\n"; 1867 #endif 1868 report_fatal_error("Do not know how to expand this operator's operand!"); 1869 1870 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 1871 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1872 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1873 1874 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; 1875 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; 1876 case ISD::STRICT_FP_ROUND: 1877 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; 1878 case ISD::STRICT_FP_TO_SINT: 1879 case ISD::STRICT_FP_TO_UINT: 1880 case ISD::FP_TO_SINT: 1881 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break; 1882 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; 1883 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; 1884 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; 1885 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; 1886 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; 1887 case ISD::STRICT_FSETCC: 1888 case ISD::STRICT_FSETCCS: 1889 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; 1890 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N), 1891 OpNo); break; 1892 } 1893 1894 // If the result is null, the sub-method took care of registering results etc. 1895 if (!Res.getNode()) return false; 1896 1897 // If the result is N, the sub-method updated N in place. Tell the legalizer 1898 // core about this. 1899 if (Res.getNode() == N) 1900 return true; 1901 1902 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1903 "Invalid operand expansion"); 1904 1905 ReplaceValueWith(SDValue(N, 0), Res); 1906 return false; 1907 } 1908 1909 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code 1910 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1911 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, 1912 SDValue &NewRHS, 1913 ISD::CondCode &CCCode, 1914 const SDLoc &dl, SDValue &Chain, 1915 bool IsSignaling) { 1916 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 1917 GetExpandedFloat(NewLHS, LHSLo, LHSHi); 1918 GetExpandedFloat(NewRHS, RHSLo, RHSHi); 1919 1920 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); 1921 1922 // FIXME: This generated code sucks. We want to generate 1923 // FCMPU crN, hi1, hi2 1924 // BNE crN, L: 1925 // FCMPU crN, lo1, lo2 1926 // The following can be improved, but not that much. 1927 SDValue Tmp1, Tmp2, Tmp3, OutputChain; 1928 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1929 RHSHi, ISD::SETOEQ, Chain, IsSignaling); 1930 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1931 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 1932 RHSLo, CCCode, OutputChain, IsSignaling); 1933 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1934 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1935 Tmp1 = 1936 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi, 1937 ISD::SETUNE, OutputChain, IsSignaling); 1938 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1939 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1940 RHSHi, CCCode, OutputChain, IsSignaling); 1941 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1942 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1943 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 1944 NewRHS = SDValue(); // LHS is the result, not a compare. 1945 Chain = OutputChain; 1946 } 1947 1948 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { 1949 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1950 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1951 SDValue Chain; 1952 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 1953 1954 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1955 // against zero to select between true and false values. 1956 if (!NewRHS.getNode()) { 1957 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1958 CCCode = ISD::SETNE; 1959 } 1960 1961 // Update N to have the operands specified. 1962 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1963 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1964 N->getOperand(4)), 0); 1965 } 1966 1967 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { 1968 assert(N->getOperand(1).getValueType() == MVT::ppcf128 && 1969 "Logic only correct for ppcf128!"); 1970 SDValue Lo, Hi; 1971 GetExpandedFloat(N->getOperand(1), Lo, Hi); 1972 // The ppcf128 value is providing only the sign; take it from the 1973 // higher-order double (which must have the larger magnitude). 1974 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), 1975 N->getValueType(0), N->getOperand(0), Hi); 1976 } 1977 1978 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { 1979 bool IsStrict = N->isStrictFPOpcode(); 1980 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && 1981 "Logic only correct for ppcf128!"); 1982 SDValue Lo, Hi; 1983 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi); 1984 1985 if (!IsStrict) 1986 // Round it the rest of the way (e.g. to f32) if needed. 1987 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 1988 N->getValueType(0), Hi, N->getOperand(1)); 1989 1990 // Eliminate the node if the input float type is the same as the output float 1991 // type. 1992 if (Hi.getValueType() == N->getValueType(0)) { 1993 // Connect the output chain to the input chain, unlinking the node. 1994 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 1995 ReplaceValueWith(SDValue(N, 0), Hi); 1996 return SDValue(); 1997 } 1998 1999 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 2000 {N->getValueType(0), MVT::Other}, 2001 {N->getOperand(0), Hi, N->getOperand(2)}); 2002 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1)); 2003 ReplaceValueWith(SDValue(N, 0), Expansion); 2004 return SDValue(); 2005 } 2006 2007 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) { 2008 EVT RVT = N->getValueType(0); 2009 SDLoc dl(N); 2010 2011 bool IsStrict = N->isStrictFPOpcode(); 2012 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 2013 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 2014 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 2015 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2016 2017 EVT NVT; 2018 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed); 2019 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 2020 "Unsupported FP_TO_XINT!"); 2021 TargetLowering::MakeLibCallOptions CallOptions; 2022 std::pair<SDValue, SDValue> Tmp = 2023 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); 2024 if (!IsStrict) 2025 return Tmp.first; 2026 2027 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2028 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2029 return SDValue(); 2030 } 2031 2032 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { 2033 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2034 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2035 SDValue Chain; 2036 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 2037 2038 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2039 // against zero to select between true and false values. 2040 if (!NewRHS.getNode()) { 2041 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 2042 CCCode = ISD::SETNE; 2043 } 2044 2045 // Update N to have the operands specified. 2046 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2047 N->getOperand(2), N->getOperand(3), 2048 DAG.getCondCode(CCCode)), 0); 2049 } 2050 2051 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { 2052 bool IsStrict = N->isStrictFPOpcode(); 2053 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0); 2054 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1); 2055 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2056 ISD::CondCode CCCode = 2057 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 2058 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain, 2059 N->getOpcode() == ISD::STRICT_FSETCCS); 2060 2061 // FloatExpandSetCCOperands always returned a scalar. 2062 assert(!NewRHS.getNode() && "Expect to return scalar"); 2063 assert(NewLHS.getValueType() == N->getValueType(0) && 2064 "Unexpected setcc expansion!"); 2065 if (Chain) { 2066 ReplaceValueWith(SDValue(N, 0), NewLHS); 2067 ReplaceValueWith(SDValue(N, 1), Chain); 2068 return SDValue(); 2069 } 2070 return NewLHS; 2071 } 2072 2073 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { 2074 if (ISD::isNormalStore(N)) 2075 return ExpandOp_NormalStore(N, OpNo); 2076 2077 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2078 assert(OpNo == 1 && "Can only expand the stored value so far"); 2079 StoreSDNode *ST = cast<StoreSDNode>(N); 2080 2081 SDValue Chain = ST->getChain(); 2082 SDValue Ptr = ST->getBasePtr(); 2083 2084 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 2085 ST->getValue().getValueType()); 2086 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2087 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 2088 (void)NVT; 2089 2090 SDValue Lo, Hi; 2091 GetExpandedOp(ST->getValue(), Lo, Hi); 2092 2093 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr, 2094 ST->getMemoryVT(), ST->getMemOperand()); 2095 } 2096 2097 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { 2098 EVT RVT = N->getValueType(0); 2099 EVT RetVT = N->getOperand(0).getValueType(); 2100 TargetLowering::MakeLibCallOptions CallOptions; 2101 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2102 RTLIB::LROUND_F32, 2103 RTLIB::LROUND_F64, 2104 RTLIB::LROUND_F80, 2105 RTLIB::LROUND_F128, 2106 RTLIB::LROUND_PPCF128), 2107 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2108 } 2109 2110 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { 2111 EVT RVT = N->getValueType(0); 2112 EVT RetVT = N->getOperand(0).getValueType(); 2113 TargetLowering::MakeLibCallOptions CallOptions; 2114 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2115 RTLIB::LLROUND_F32, 2116 RTLIB::LLROUND_F64, 2117 RTLIB::LLROUND_F80, 2118 RTLIB::LLROUND_F128, 2119 RTLIB::LLROUND_PPCF128), 2120 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2121 } 2122 2123 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { 2124 EVT RVT = N->getValueType(0); 2125 EVT RetVT = N->getOperand(0).getValueType(); 2126 TargetLowering::MakeLibCallOptions CallOptions; 2127 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2128 RTLIB::LRINT_F32, 2129 RTLIB::LRINT_F64, 2130 RTLIB::LRINT_F80, 2131 RTLIB::LRINT_F128, 2132 RTLIB::LRINT_PPCF128), 2133 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2134 } 2135 2136 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { 2137 EVT RVT = N->getValueType(0); 2138 EVT RetVT = N->getOperand(0).getValueType(); 2139 TargetLowering::MakeLibCallOptions CallOptions; 2140 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2141 RTLIB::LLRINT_F32, 2142 RTLIB::LLRINT_F64, 2143 RTLIB::LLRINT_F80, 2144 RTLIB::LLRINT_F128, 2145 RTLIB::LLRINT_PPCF128), 2146 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2147 } 2148 2149 //===----------------------------------------------------------------------===// 2150 // Float Operand Promotion 2151 //===----------------------------------------------------------------------===// 2152 // 2153 2154 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { 2155 if (OpVT == MVT::f16) { 2156 return ISD::FP16_TO_FP; 2157 } else if (RetVT == MVT::f16) { 2158 return ISD::FP_TO_FP16; 2159 } else if (OpVT == MVT::bf16) { 2160 return ISD::BF16_TO_FP; 2161 } else if (RetVT == MVT::bf16) { 2162 return ISD::FP_TO_BF16; 2163 } 2164 2165 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2166 } 2167 2168 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { 2169 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG); 2170 dbgs() << "\n"); 2171 SDValue R = SDValue(); 2172 2173 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2174 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2175 return false; 2176 } 2177 2178 // Nodes that use a promotion-requiring floating point operand, but doesn't 2179 // produce a promotion-requiring floating point result, need to be legalized 2180 // to use the promoted float operand. Nodes that produce at least one 2181 // promotion-requiring floating point result have their operands legalized as 2182 // a part of PromoteFloatResult. 2183 switch (N->getOpcode()) { 2184 default: 2185 #ifndef NDEBUG 2186 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; 2187 N->dump(&DAG); dbgs() << "\n"; 2188 #endif 2189 report_fatal_error("Do not know how to promote this operator's operand!"); 2190 2191 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; 2192 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; 2193 case ISD::FP_TO_SINT: 2194 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break; 2195 case ISD::FP_TO_SINT_SAT: 2196 case ISD::FP_TO_UINT_SAT: 2197 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; 2198 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; 2199 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; 2200 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; 2201 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; 2202 } 2203 2204 if (R.getNode()) 2205 ReplaceValueWith(SDValue(N, 0), R); 2206 return false; 2207 } 2208 2209 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { 2210 SDValue Op = N->getOperand(0); 2211 EVT OpVT = Op->getValueType(0); 2212 2213 SDValue Promoted = GetPromotedFloat(N->getOperand(0)); 2214 EVT PromotedVT = Promoted->getValueType(0); 2215 2216 // Convert the promoted float value to the desired IVT. 2217 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); 2218 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), 2219 IVT, Promoted); 2220 // The final result type might not be an scalar so we need a bitcast. The 2221 // bitcast will be further legalized if needed. 2222 return DAG.getBitcast(N->getValueType(0), Convert); 2223 } 2224 2225 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by 2226 // PromoteFloatRes_FCOPYSIGN. 2227 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { 2228 assert (OpNo == 1 && "Only Operand 1 must need promotion here"); 2229 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2230 2231 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2232 N->getOperand(0), Op1); 2233 } 2234 2235 // Convert the promoted float value to the desired integer type 2236 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) { 2237 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2238 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); 2239 } 2240 2241 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, 2242 unsigned OpNo) { 2243 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2244 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, 2245 N->getOperand(1)); 2246 } 2247 2248 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { 2249 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2250 EVT VT = N->getValueType(0); 2251 2252 // Desired VT is same as promoted type. Use promoted float directly. 2253 if (VT == Op->getValueType(0)) 2254 return Op; 2255 2256 // Else, extend the promoted float value to the desired VT. 2257 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); 2258 } 2259 2260 // Promote the float operands used for comparison. The true- and false- 2261 // operands have the same type as the result and are promoted, if needed, by 2262 // PromoteFloatRes_SELECT_CC 2263 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2264 SDValue LHS = GetPromotedFloat(N->getOperand(0)); 2265 SDValue RHS = GetPromotedFloat(N->getOperand(1)); 2266 2267 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 2268 LHS, RHS, N->getOperand(2), N->getOperand(3), 2269 N->getOperand(4)); 2270 } 2271 2272 // Construct a SETCC that compares the promoted values and sets the conditional 2273 // code. 2274 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { 2275 EVT VT = N->getValueType(0); 2276 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2277 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2278 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2279 2280 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); 2281 2282 } 2283 2284 // Lower the promoted Float down to the integer value of same size and construct 2285 // a STORE of the integer value. 2286 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { 2287 StoreSDNode *ST = cast<StoreSDNode>(N); 2288 SDValue Val = ST->getValue(); 2289 SDLoc DL(N); 2290 2291 SDValue Promoted = GetPromotedFloat(Val); 2292 EVT VT = ST->getOperand(1).getValueType(); 2293 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2294 2295 SDValue NewVal; 2296 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, 2297 IVT, Promoted); 2298 2299 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), 2300 ST->getMemOperand()); 2301 } 2302 2303 //===----------------------------------------------------------------------===// 2304 // Float Result Promotion 2305 //===----------------------------------------------------------------------===// 2306 2307 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { 2308 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG); 2309 dbgs() << "\n"); 2310 SDValue R = SDValue(); 2311 2312 // See if the target wants to custom expand this node. 2313 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2314 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2315 return; 2316 } 2317 2318 switch (N->getOpcode()) { 2319 // These opcodes cannot appear if promotion of FP16 is done in the backend 2320 // instead of Clang 2321 case ISD::FP16_TO_FP: 2322 case ISD::FP_TO_FP16: 2323 default: 2324 #ifndef NDEBUG 2325 dbgs() << "PromoteFloatResult #" << ResNo << ": "; 2326 N->dump(&DAG); dbgs() << "\n"; 2327 #endif 2328 report_fatal_error("Do not know how to promote this operator's result!"); 2329 2330 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; 2331 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; 2332 case ISD::EXTRACT_VECTOR_ELT: 2333 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; 2334 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; 2335 2336 // Unary FP Operations 2337 case ISD::FABS: 2338 case ISD::FCBRT: 2339 case ISD::FCEIL: 2340 case ISD::FCOS: 2341 case ISD::FEXP: 2342 case ISD::FEXP2: 2343 case ISD::FFLOOR: 2344 case ISD::FLOG: 2345 case ISD::FLOG2: 2346 case ISD::FLOG10: 2347 case ISD::FNEARBYINT: 2348 case ISD::FNEG: 2349 case ISD::FRINT: 2350 case ISD::FROUND: 2351 case ISD::FROUNDEVEN: 2352 case ISD::FSIN: 2353 case ISD::FSQRT: 2354 case ISD::FTRUNC: 2355 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; 2356 2357 // Binary FP Operations 2358 case ISD::FADD: 2359 case ISD::FDIV: 2360 case ISD::FMAXIMUM: 2361 case ISD::FMINIMUM: 2362 case ISD::FMAXNUM: 2363 case ISD::FMINNUM: 2364 case ISD::FMUL: 2365 case ISD::FPOW: 2366 case ISD::FREM: 2367 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; 2368 2369 case ISD::FMA: // FMA is same as FMAD 2370 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; 2371 2372 case ISD::FPOWI: 2373 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break; 2374 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break; 2375 2376 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; 2377 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; 2378 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; 2379 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; 2380 2381 case ISD::SINT_TO_FP: 2382 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; 2383 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; 2384 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2385 case ISD::VECREDUCE_FADD: 2386 case ISD::VECREDUCE_FMUL: 2387 case ISD::VECREDUCE_FMIN: 2388 case ISD::VECREDUCE_FMAX: 2389 case ISD::VECREDUCE_FMAXIMUM: 2390 case ISD::VECREDUCE_FMINIMUM: 2391 R = PromoteFloatRes_VECREDUCE(N); 2392 break; 2393 case ISD::VECREDUCE_SEQ_FADD: 2394 case ISD::VECREDUCE_SEQ_FMUL: 2395 R = PromoteFloatRes_VECREDUCE_SEQ(N); 2396 break; 2397 } 2398 2399 if (R.getNode()) 2400 SetPromotedFloat(SDValue(N, ResNo), R); 2401 } 2402 2403 // Bitcast from i16 to f16: convert the i16 to a f32 value instead. 2404 // At this point, it is not possible to determine if the bitcast value is 2405 // eventually stored to memory or promoted to f32 or promoted to a floating 2406 // point at a higher precision. Some of these cases are handled by FP_EXTEND, 2407 // STORE promotion handlers. 2408 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { 2409 EVT VT = N->getValueType(0); 2410 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2411 // Input type isn't guaranteed to be a scalar int so bitcast if not. The 2412 // bitcast will be legalized further if necessary. 2413 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), 2414 N->getOperand(0).getValueType().getSizeInBits()); 2415 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); 2416 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); 2417 } 2418 2419 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { 2420 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); 2421 EVT VT = N->getValueType(0); 2422 SDLoc DL(N); 2423 2424 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2425 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2426 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, 2427 IVT); 2428 2429 // Convert the Constant to the desired FP type 2430 // FIXME We might be able to do the conversion during compilation and get rid 2431 // of it from the object code 2432 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2433 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); 2434 } 2435 2436 // If the Index operand is a constant, try to redirect the extract operation to 2437 // the correct legalized vector. If not, bit-convert the input vector to 2438 // equivalent integer vector. Extract the element as an (bit-cast) integer 2439 // value and convert it to the promoted type. 2440 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2441 SDLoc DL(N); 2442 2443 // If the index is constant, try to extract the value from the legalized 2444 // vector type. 2445 if (isa<ConstantSDNode>(N->getOperand(1))) { 2446 SDValue Vec = N->getOperand(0); 2447 SDValue Idx = N->getOperand(1); 2448 EVT VecVT = Vec->getValueType(0); 2449 EVT EltVT = VecVT.getVectorElementType(); 2450 2451 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2452 2453 switch (getTypeAction(VecVT)) { 2454 default: break; 2455 case TargetLowering::TypeScalarizeVector: { 2456 SDValue Res = GetScalarizedVector(N->getOperand(0)); 2457 ReplaceValueWith(SDValue(N, 0), Res); 2458 return SDValue(); 2459 } 2460 case TargetLowering::TypeWidenVector: { 2461 Vec = GetWidenedVector(Vec); 2462 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); 2463 ReplaceValueWith(SDValue(N, 0), Res); 2464 return SDValue(); 2465 } 2466 case TargetLowering::TypeSplitVector: { 2467 SDValue Lo, Hi; 2468 GetSplitVector(Vec, Lo, Hi); 2469 2470 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2471 SDValue Res; 2472 if (IdxVal < LoElts) 2473 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); 2474 else 2475 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, 2476 DAG.getConstant(IdxVal - LoElts, DL, 2477 Idx.getValueType())); 2478 ReplaceValueWith(SDValue(N, 0), Res); 2479 return SDValue(); 2480 } 2481 2482 } 2483 } 2484 2485 // Bit-convert the input vector to the equivalent integer vector 2486 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2487 EVT IVT = NewOp.getValueType().getVectorElementType(); 2488 2489 // Extract the element as an (bit-cast) integer value 2490 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, 2491 NewOp, N->getOperand(1)); 2492 2493 // Convert the element to the desired FP type 2494 EVT VT = N->getValueType(0); 2495 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2496 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); 2497 } 2498 2499 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result 2500 // needs promotion, so does the argument X. Note that Y, if needed, will be 2501 // handled during operand promotion. 2502 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { 2503 EVT VT = N->getValueType(0); 2504 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2505 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2506 2507 SDValue Op1 = N->getOperand(1); 2508 2509 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2510 } 2511 2512 // Unary operation where the result and the operand have PromoteFloat type 2513 // action. Construct a new SDNode with the promoted float value of the old 2514 // operand. 2515 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { 2516 EVT VT = N->getValueType(0); 2517 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2518 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2519 2520 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); 2521 } 2522 2523 // Binary operations where the result and both operands have PromoteFloat type 2524 // action. Construct a new SDNode with the promoted float values of the old 2525 // operands. 2526 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { 2527 EVT VT = N->getValueType(0); 2528 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2529 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2530 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2531 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); 2532 } 2533 2534 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { 2535 EVT VT = N->getValueType(0); 2536 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2537 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2538 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2539 SDValue Op2 = GetPromotedFloat(N->getOperand(2)); 2540 2541 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); 2542 } 2543 2544 // Promote the Float (first) operand and retain the Integer (second) operand 2545 SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) { 2546 EVT VT = N->getValueType(0); 2547 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2548 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2549 SDValue Op1 = N->getOperand(1); 2550 2551 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2552 } 2553 2554 SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) { 2555 EVT VT = N->getValueType(0); 2556 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2557 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2558 SDValue Res = 2559 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op); 2560 2561 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2562 return Res; 2563 } 2564 2565 // Explicit operation to reduce precision. Reduce the value to half precision 2566 // and promote it back to the legal type. 2567 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { 2568 SDLoc DL(N); 2569 2570 SDValue Op = N->getOperand(0); 2571 EVT VT = N->getValueType(0); 2572 EVT OpVT = Op->getValueType(0); 2573 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2574 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2575 2576 // Round promoted float to desired precision 2577 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); 2578 // Promote it back to the legal output type 2579 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); 2580 } 2581 2582 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { 2583 LoadSDNode *L = cast<LoadSDNode>(N); 2584 EVT VT = N->getValueType(0); 2585 2586 // Load the value as an integer value with the same number of bits. 2587 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2588 SDValue newL = DAG.getLoad( 2589 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), 2590 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, 2591 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); 2592 // Legalize the chain result by replacing uses of the old value chain with the 2593 // new one 2594 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 2595 2596 // Convert the integer value to the desired FP type 2597 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2598 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); 2599 } 2600 2601 // Construct a new SELECT node with the promoted true- and false- values. 2602 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { 2603 SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); 2604 SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); 2605 2606 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), 2607 N->getOperand(0), TrueVal, FalseVal); 2608 } 2609 2610 // Construct a new SELECT_CC node with the promoted true- and false- values. 2611 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. 2612 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { 2613 SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); 2614 SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); 2615 2616 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2617 TrueVal.getNode()->getValueType(0), N->getOperand(0), 2618 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); 2619 } 2620 2621 // Construct a SDNode that transforms the SINT or UINT operand to the promoted 2622 // float type. 2623 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { 2624 SDLoc DL(N); 2625 EVT VT = N->getValueType(0); 2626 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2627 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); 2628 // Round the value to the desired precision (that of the source type). 2629 return DAG.getNode( 2630 ISD::FP_EXTEND, DL, NVT, 2631 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, 2632 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true))); 2633 } 2634 2635 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { 2636 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 2637 N->getValueType(0))); 2638 } 2639 2640 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { 2641 // Expand and promote recursively. 2642 // TODO: This is non-optimal, but dealing with the concurrently happening 2643 // vector-legalization is non-trivial. We could do something similar to 2644 // PromoteFloatRes_EXTRACT_VECTOR_ELT here. 2645 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2646 return SDValue(); 2647 } 2648 2649 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { 2650 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2651 return SDValue(); 2652 } 2653 2654 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { 2655 EVT VT = N->getValueType(0); 2656 2657 AtomicSDNode *AM = cast<AtomicSDNode>(N); 2658 SDLoc SL(N); 2659 2660 SDValue CastVal = BitConvertToInteger(AM->getVal()); 2661 EVT CastVT = CastVal.getValueType(); 2662 2663 SDValue NewAtomic 2664 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, 2665 DAG.getVTList(CastVT, MVT::Other), 2666 { AM->getChain(), AM->getBasePtr(), CastVal }, 2667 AM->getMemOperand()); 2668 2669 SDValue Result = NewAtomic; 2670 2671 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { 2672 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2673 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, 2674 NewAtomic); 2675 } 2676 2677 // Legalize the chain result by replacing uses of the old value chain with the 2678 // new one 2679 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); 2680 2681 return Result; 2682 2683 } 2684 2685 //===----------------------------------------------------------------------===// 2686 // Half Result Soft Promotion 2687 //===----------------------------------------------------------------------===// 2688 2689 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { 2690 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": "; 2691 N->dump(&DAG); dbgs() << "\n"); 2692 SDValue R = SDValue(); 2693 2694 // See if the target wants to custom expand this node. 2695 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2696 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2697 return; 2698 } 2699 2700 switch (N->getOpcode()) { 2701 default: 2702 #ifndef NDEBUG 2703 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": "; 2704 N->dump(&DAG); dbgs() << "\n"; 2705 #endif 2706 report_fatal_error("Do not know how to soft promote this operator's " 2707 "result!"); 2708 2709 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; 2710 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; 2711 case ISD::EXTRACT_VECTOR_ELT: 2712 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; 2713 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; 2714 case ISD::STRICT_FP_ROUND: 2715 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; 2716 2717 // Unary FP Operations 2718 case ISD::FABS: 2719 case ISD::FCBRT: 2720 case ISD::FCEIL: 2721 case ISD::FCOS: 2722 case ISD::FEXP: 2723 case ISD::FEXP2: 2724 case ISD::FFLOOR: 2725 case ISD::FLOG: 2726 case ISD::FLOG2: 2727 case ISD::FLOG10: 2728 case ISD::FNEARBYINT: 2729 case ISD::FNEG: 2730 case ISD::FREEZE: 2731 case ISD::FRINT: 2732 case ISD::FROUND: 2733 case ISD::FROUNDEVEN: 2734 case ISD::FSIN: 2735 case ISD::FSQRT: 2736 case ISD::FTRUNC: 2737 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; 2738 2739 // Binary FP Operations 2740 case ISD::FADD: 2741 case ISD::FDIV: 2742 case ISD::FMAXIMUM: 2743 case ISD::FMINIMUM: 2744 case ISD::FMAXNUM: 2745 case ISD::FMINNUM: 2746 case ISD::FMUL: 2747 case ISD::FPOW: 2748 case ISD::FREM: 2749 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; 2750 2751 case ISD::FMA: // FMA is same as FMAD 2752 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; 2753 2754 case ISD::FPOWI: 2755 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break; 2756 2757 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; 2758 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; 2759 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; 2760 case ISD::SINT_TO_FP: 2761 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; 2762 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; 2763 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2764 case ISD::VECREDUCE_FADD: 2765 case ISD::VECREDUCE_FMUL: 2766 case ISD::VECREDUCE_FMIN: 2767 case ISD::VECREDUCE_FMAX: 2768 case ISD::VECREDUCE_FMAXIMUM: 2769 case ISD::VECREDUCE_FMINIMUM: 2770 R = SoftPromoteHalfRes_VECREDUCE(N); 2771 break; 2772 case ISD::VECREDUCE_SEQ_FADD: 2773 case ISD::VECREDUCE_SEQ_FMUL: 2774 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N); 2775 break; 2776 } 2777 2778 if (R.getNode()) 2779 SetSoftPromotedHalf(SDValue(N, ResNo), R); 2780 } 2781 2782 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { 2783 return BitConvertToInteger(N->getOperand(0)); 2784 } 2785 2786 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { 2787 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 2788 2789 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2790 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 2791 MVT::i16); 2792 } 2793 2794 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2795 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2796 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2797 NewOp.getValueType().getVectorElementType(), NewOp, 2798 N->getOperand(1)); 2799 } 2800 2801 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { 2802 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0)); 2803 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 2804 SDLoc dl(N); 2805 2806 EVT LVT = LHS.getValueType(); 2807 EVT RVT = RHS.getValueType(); 2808 2809 unsigned LSize = LVT.getSizeInBits(); 2810 unsigned RSize = RVT.getSizeInBits(); 2811 2812 // First get the sign bit of second operand. 2813 SDValue SignBit = DAG.getNode( 2814 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 2815 DAG.getConstant(RSize - 1, dl, 2816 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 2817 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 2818 2819 // Shift right or sign-extend it if the two operands have different types. 2820 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 2821 if (SizeDiff > 0) { 2822 SignBit = 2823 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 2824 DAG.getConstant(SizeDiff, dl, 2825 TLI.getShiftAmountTy(SignBit.getValueType(), 2826 DAG.getDataLayout()))); 2827 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 2828 } else if (SizeDiff < 0) { 2829 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 2830 SignBit = 2831 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 2832 DAG.getConstant(-SizeDiff, dl, 2833 TLI.getShiftAmountTy(SignBit.getValueType(), 2834 DAG.getDataLayout()))); 2835 } 2836 2837 // Clear the sign bit of the first operand. 2838 SDValue Mask = DAG.getNode( 2839 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 2840 DAG.getConstant(LSize - 1, dl, 2841 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 2842 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 2843 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 2844 2845 // Or the value with the sign bit. 2846 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 2847 } 2848 2849 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { 2850 EVT OVT = N->getValueType(0); 2851 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2852 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2853 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2854 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2855 SDLoc dl(N); 2856 2857 // Promote to the larger FP type. 2858 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 2859 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 2860 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 2861 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2); 2862 2863 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2); 2864 2865 // Convert back to FP16 as an integer. 2866 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2867 } 2868 2869 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) { 2870 EVT OVT = N->getValueType(0); 2871 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2872 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2873 SDValue Op1 = N->getOperand(1); 2874 SDLoc dl(N); 2875 2876 // Promote to the larger FP type. 2877 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0); 2878 2879 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2880 2881 // Convert back to FP16 as an integer. 2882 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2883 } 2884 2885 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { 2886 EVT RVT = N->getValueType(0); 2887 EVT SVT = N->getOperand(0).getValueType(); 2888 2889 if (N->isStrictFPOpcode()) { 2890 assert(RVT == MVT::f16); 2891 SDValue Res = 2892 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other}, 2893 {N->getOperand(0), N->getOperand(1)}); 2894 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2895 return Res; 2896 } 2897 2898 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16, 2899 N->getOperand(0)); 2900 } 2901 2902 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { 2903 LoadSDNode *L = cast<LoadSDNode>(N); 2904 2905 // Load the value as an integer value with the same number of bits. 2906 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!"); 2907 SDValue NewL = 2908 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, 2909 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), 2910 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), 2911 L->getMemOperand()->getFlags(), L->getAAInfo()); 2912 // Legalize the chain result by replacing uses of the old value chain with the 2913 // new one 2914 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 2915 return NewL; 2916 } 2917 2918 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { 2919 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2920 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2921 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1, 2922 Op2); 2923 } 2924 2925 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { 2926 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2927 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3)); 2928 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(), 2929 N->getOperand(0), N->getOperand(1), Op2, Op3, 2930 N->getOperand(4)); 2931 } 2932 2933 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { 2934 EVT OVT = N->getValueType(0); 2935 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2936 SDLoc dl(N); 2937 2938 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 2939 2940 // Round the value to the softened type. 2941 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2942 } 2943 2944 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { 2945 return DAG.getUNDEF(MVT::i16); 2946 } 2947 2948 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { 2949 EVT OVT = N->getValueType(0); 2950 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2951 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 2952 SDLoc dl(N); 2953 2954 // Promote to the larger FP type. 2955 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 2956 2957 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op); 2958 2959 // Convert back to FP16 as an integer. 2960 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2961 } 2962 2963 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { 2964 EVT OVT = N->getValueType(0); 2965 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2966 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2967 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2968 SDLoc dl(N); 2969 2970 // Promote to the larger FP type. 2971 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 2972 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 2973 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 2974 2975 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2976 2977 // Convert back to FP16 as an integer. 2978 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2979 } 2980 2981 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) { 2982 // Expand and soften recursively. 2983 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2984 return SDValue(); 2985 } 2986 2987 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) { 2988 // Expand and soften. 2989 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2990 return SDValue(); 2991 } 2992 2993 //===----------------------------------------------------------------------===// 2994 // Half Operand Soft Promotion 2995 //===----------------------------------------------------------------------===// 2996 2997 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { 2998 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": "; 2999 N->dump(&DAG); dbgs() << "\n"); 3000 SDValue Res = SDValue(); 3001 3002 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 3003 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 3004 return false; 3005 } 3006 3007 // Nodes that use a promotion-requiring floating point operand, but doesn't 3008 // produce a soft promotion-requiring floating point result, need to be 3009 // legalized to use the soft promoted float operand. Nodes that produce at 3010 // least one soft promotion-requiring floating point result have their 3011 // operands legalized as a part of PromoteFloatResult. 3012 switch (N->getOpcode()) { 3013 default: 3014 #ifndef NDEBUG 3015 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": "; 3016 N->dump(&DAG); dbgs() << "\n"; 3017 #endif 3018 report_fatal_error("Do not know how to soft promote this operator's " 3019 "operand!"); 3020 3021 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; 3022 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; 3023 case ISD::FP_TO_SINT: 3024 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; 3025 case ISD::FP_TO_SINT_SAT: 3026 case ISD::FP_TO_UINT_SAT: 3027 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; 3028 case ISD::STRICT_FP_EXTEND: 3029 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; 3030 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; 3031 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; 3032 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; 3033 case ISD::STACKMAP: 3034 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); 3035 break; 3036 case ISD::PATCHPOINT: 3037 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo); 3038 break; 3039 } 3040 3041 if (!Res.getNode()) 3042 return false; 3043 3044 assert(Res.getNode() != N && "Expected a new node!"); 3045 3046 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 3047 "Invalid operand expansion"); 3048 3049 ReplaceValueWith(SDValue(N, 0), Res); 3050 return false; 3051 } 3052 3053 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { 3054 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3055 3056 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 3057 } 3058 3059 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, 3060 unsigned OpNo) { 3061 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 3062 SDValue Op1 = N->getOperand(1); 3063 EVT RVT = Op1.getValueType(); 3064 SDLoc dl(N); 3065 3066 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType()); 3067 3068 Op1 = GetSoftPromotedHalf(Op1); 3069 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1); 3070 3071 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0), 3072 Op1); 3073 } 3074 3075 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { 3076 EVT RVT = N->getValueType(0); 3077 bool IsStrict = N->isStrictFPOpcode(); 3078 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3079 EVT SVT = Op.getValueType(); 3080 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0)); 3081 3082 if (IsStrict) { 3083 assert(SVT == MVT::f16); 3084 SDValue Res = 3085 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N), 3086 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op}); 3087 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3088 ReplaceValueWith(SDValue(N, 0), Res); 3089 return SDValue(); 3090 } 3091 3092 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op); 3093 } 3094 3095 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { 3096 EVT RVT = N->getValueType(0); 3097 SDValue Op = N->getOperand(0); 3098 EVT SVT = Op.getValueType(); 3099 SDLoc dl(N); 3100 3101 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3102 3103 Op = GetSoftPromotedHalf(Op); 3104 3105 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3106 3107 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res); 3108 } 3109 3110 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { 3111 EVT RVT = N->getValueType(0); 3112 SDValue Op = N->getOperand(0); 3113 EVT SVT = Op.getValueType(); 3114 SDLoc dl(N); 3115 3116 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3117 3118 Op = GetSoftPromotedHalf(Op); 3119 3120 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3121 3122 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res, 3123 N->getOperand(1)); 3124 } 3125 3126 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, 3127 unsigned OpNo) { 3128 assert(OpNo == 0 && "Can only soften the comparison values"); 3129 SDValue Op0 = N->getOperand(0); 3130 SDValue Op1 = N->getOperand(1); 3131 SDLoc dl(N); 3132 3133 EVT SVT = Op0.getValueType(); 3134 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT); 3135 3136 Op0 = GetSoftPromotedHalf(Op0); 3137 Op1 = GetSoftPromotedHalf(Op1); 3138 3139 // Promote to the larger FP type. 3140 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3141 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3142 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3143 3144 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1, 3145 N->getOperand(2), N->getOperand(3), N->getOperand(4)); 3146 } 3147 3148 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { 3149 SDValue Op0 = N->getOperand(0); 3150 SDValue Op1 = N->getOperand(1); 3151 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3152 SDLoc dl(N); 3153 3154 EVT SVT = Op0.getValueType(); 3155 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 3156 3157 Op0 = GetSoftPromotedHalf(Op0); 3158 Op1 = GetSoftPromotedHalf(Op1); 3159 3160 // Promote to the larger FP type. 3161 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3162 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3163 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3164 3165 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode); 3166 } 3167 3168 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { 3169 assert(OpNo == 1 && "Can only soften the stored value!"); 3170 StoreSDNode *ST = cast<StoreSDNode>(N); 3171 SDValue Val = ST->getValue(); 3172 SDLoc dl(N); 3173 3174 assert(!ST->isTruncatingStore() && "Unexpected truncating store."); 3175 SDValue Promoted = GetSoftPromotedHalf(Val); 3176 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), 3177 ST->getMemOperand()); 3178 } 3179 3180 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { 3181 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 3182 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3183 SDValue Op = N->getOperand(OpNo); 3184 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3185 SDValue NewNode = 3186 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3187 3188 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3189 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3190 3191 return SDValue(); // Signal that we replaced the node ourselves. 3192 } 3193 3194 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N, 3195 unsigned OpNo) { 3196 assert(OpNo >= 7); 3197 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3198 SDValue Op = N->getOperand(OpNo); 3199 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3200 SDValue NewNode = 3201 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3202 3203 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3204 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3205 3206 return SDValue(); // Signal that we replaced the node ourselves. 3207 } 3208