1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements float type expansion and softening for LegalizeTypes. 10 // Softening is the act of turning a computation in an illegal floating point 11 // type into a computation in an integer type of the same size; also known as 12 // "soft float". For example, turning f32 arithmetic into operations using i32. 13 // The resulting integer value is the same as what you would get by performing 14 // the floating point operation and bitcasting the result to the integer type. 15 // Expansion is the act of changing a computation in an illegal type to be a 16 // computation in two identical registers of a smaller type. For example, 17 // implementing ppcf128 arithmetic in two f64 registers. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/Analysis/TargetLibraryInfo.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 /// GetFPLibCall - Return the right libcall for the given floating point type. 30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be 31 /// refactored away (see RTLIB::getPOWI for an example). 32 static RTLIB::Libcall GetFPLibCall(EVT VT, 33 RTLIB::Libcall Call_F32, 34 RTLIB::Libcall Call_F64, 35 RTLIB::Libcall Call_F80, 36 RTLIB::Libcall Call_F128, 37 RTLIB::Libcall Call_PPCF128) { 38 return 39 VT == MVT::f32 ? Call_F32 : 40 VT == MVT::f64 ? Call_F64 : 41 VT == MVT::f80 ? Call_F80 : 42 VT == MVT::f128 ? Call_F128 : 43 VT == MVT::ppcf128 ? Call_PPCF128 : 44 RTLIB::UNKNOWN_LIBCALL; 45 } 46 47 //===----------------------------------------------------------------------===// 48 // Convert Float Results to Integer 49 //===----------------------------------------------------------------------===// 50 51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { 52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG); 53 dbgs() << "\n"); 54 SDValue R = SDValue(); 55 56 switch (N->getOpcode()) { 57 default: 58 #ifndef NDEBUG 59 dbgs() << "SoftenFloatResult #" << ResNo << ": "; 60 N->dump(&DAG); dbgs() << "\n"; 61 #endif 62 llvm_unreachable("Do not know how to soften the result of this operator!"); 63 64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break; 65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; 66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; 67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; 68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; 69 case ISD::EXTRACT_VECTOR_ELT: 70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; 71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break; 72 case ISD::STRICT_FMINNUM: 73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; 74 case ISD::STRICT_FMAXNUM: 75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; 76 case ISD::STRICT_FADD: 77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break; 78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; 79 case ISD::STRICT_FCEIL: 80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; 81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; 82 case ISD::STRICT_FCOS: 83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; 84 case ISD::STRICT_FDIV: 85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; 86 case ISD::STRICT_FEXP: 87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; 88 case ISD::STRICT_FEXP2: 89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; 90 case ISD::STRICT_FFLOOR: 91 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; 92 case ISD::STRICT_FLOG: 93 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; 94 case ISD::STRICT_FLOG2: 95 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; 96 case ISD::STRICT_FLOG10: 97 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; 98 case ISD::STRICT_FMA: 99 case ISD::FMA: R = SoftenFloatRes_FMA(N); break; 100 case ISD::STRICT_FMUL: 101 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; 102 case ISD::STRICT_FNEARBYINT: 103 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; 104 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; 105 case ISD::STRICT_FP_EXTEND: 106 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; 107 case ISD::STRICT_FP_ROUND: 108 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; 109 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; 110 case ISD::STRICT_FPOW: 111 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; 112 case ISD::STRICT_FPOWI: 113 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break; 114 case ISD::STRICT_FREM: 115 case ISD::FREM: R = SoftenFloatRes_FREM(N); break; 116 case ISD::STRICT_FRINT: 117 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; 118 case ISD::STRICT_FROUND: 119 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; 120 case ISD::STRICT_FROUNDEVEN: 121 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; 122 case ISD::STRICT_FSIN: 123 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; 124 case ISD::STRICT_FSQRT: 125 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; 126 case ISD::STRICT_FSUB: 127 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; 128 case ISD::STRICT_FTRUNC: 129 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; 130 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; 131 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 132 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; 133 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; 134 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; 135 case ISD::STRICT_SINT_TO_FP: 136 case ISD::STRICT_UINT_TO_FP: 137 case ISD::SINT_TO_FP: 138 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; 139 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; 140 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; 141 case ISD::VECREDUCE_FADD: 142 case ISD::VECREDUCE_FMUL: 143 case ISD::VECREDUCE_FMIN: 144 case ISD::VECREDUCE_FMAX: 145 R = SoftenFloatRes_VECREDUCE(N); 146 break; 147 case ISD::VECREDUCE_SEQ_FADD: 148 case ISD::VECREDUCE_SEQ_FMUL: 149 R = SoftenFloatRes_VECREDUCE_SEQ(N); 150 break; 151 } 152 153 // If R is null, the sub-method took care of registering the result. 154 if (R.getNode()) { 155 assert(R.getNode() != N); 156 SetSoftenedFloat(SDValue(N, ResNo), R); 157 } 158 } 159 160 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { 161 bool IsStrict = N->isStrictFPOpcode(); 162 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 163 unsigned Offset = IsStrict ? 1 : 0; 164 assert(N->getNumOperands() == (1 + Offset) && 165 "Unexpected number of operands!"); 166 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 167 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 168 TargetLowering::MakeLibCallOptions CallOptions; 169 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 170 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 171 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 172 CallOptions, SDLoc(N), 173 Chain); 174 if (IsStrict) 175 ReplaceValueWith(SDValue(N, 1), Tmp.second); 176 return Tmp.first; 177 } 178 179 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { 180 bool IsStrict = N->isStrictFPOpcode(); 181 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 182 unsigned Offset = IsStrict ? 1 : 0; 183 assert(N->getNumOperands() == (2 + Offset) && 184 "Unexpected number of operands!"); 185 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 186 GetSoftenedFloat(N->getOperand(1 + Offset)) }; 187 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 188 TargetLowering::MakeLibCallOptions CallOptions; 189 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 190 N->getOperand(1 + Offset).getValueType() }; 191 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 192 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 193 CallOptions, SDLoc(N), 194 Chain); 195 if (IsStrict) 196 ReplaceValueWith(SDValue(N, 1), Tmp.second); 197 return Tmp.first; 198 } 199 200 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { 201 return BitConvertToInteger(N->getOperand(0)); 202 } 203 204 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { 205 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 206 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty, 207 GetSoftenedFloat(N->getOperand(0))); 208 } 209 210 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) { 211 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 212 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty, 213 GetSoftenedFloat(N->getOperand(0))); 214 return NewFence; 215 } 216 217 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, 218 unsigned ResNo) { 219 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 220 return BitConvertToInteger(Op); 221 } 222 223 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { 224 // Convert the inputs to integers, and build a new pair out of them. 225 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), 226 TLI.getTypeToTransformTo(*DAG.getContext(), 227 N->getValueType(0)), 228 BitConvertToInteger(N->getOperand(0)), 229 BitConvertToInteger(N->getOperand(1))); 230 } 231 232 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { 233 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 234 // In ppcf128, the high 64 bits are always first in memory regardless 235 // of Endianness. LLVM's APFloat representation is not Endian sensitive, 236 // and so always converts into a 128-bit APInt in a non-Endian-sensitive 237 // way. However, APInt's are serialized in an Endian-sensitive fashion, 238 // so on big-Endian targets, the two doubles are output in the wrong 239 // order. Fix this by manually flipping the order of the high 64 bits 240 // and the low 64 bits here. 241 if (DAG.getDataLayout().isBigEndian() && 242 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) { 243 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], 244 CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; 245 APInt Val(128, words); 246 return DAG.getConstant(Val, SDLoc(CN), 247 TLI.getTypeToTransformTo(*DAG.getContext(), 248 CN->getValueType(0))); 249 } else { 250 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 251 TLI.getTypeToTransformTo(*DAG.getContext(), 252 CN->getValueType(0))); 253 } 254 } 255 256 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) { 257 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 258 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 259 NewOp.getValueType().getVectorElementType(), 260 NewOp, N->getOperand(1)); 261 } 262 263 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { 264 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 265 unsigned Size = NVT.getSizeInBits(); 266 267 // Mask = ~(1 << (Size-1)) 268 APInt API = APInt::getAllOnes(Size); 269 API.clearBit(Size - 1); 270 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT); 271 SDValue Op = GetSoftenedFloat(N->getOperand(0)); 272 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask); 273 } 274 275 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { 276 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 277 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 278 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 279 RTLIB::FMIN_F32, 280 RTLIB::FMIN_F64, 281 RTLIB::FMIN_F80, 282 RTLIB::FMIN_F128, 283 RTLIB::FMIN_PPCF128)); 284 } 285 286 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { 287 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 288 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 289 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 290 RTLIB::FMAX_F32, 291 RTLIB::FMAX_F64, 292 RTLIB::FMAX_F80, 293 RTLIB::FMAX_F128, 294 RTLIB::FMAX_PPCF128)); 295 } 296 297 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { 298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 299 RTLIB::ADD_F32, 300 RTLIB::ADD_F64, 301 RTLIB::ADD_F80, 302 RTLIB::ADD_F128, 303 RTLIB::ADD_PPCF128)); 304 } 305 306 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { 307 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 308 RTLIB::CBRT_F32, 309 RTLIB::CBRT_F64, 310 RTLIB::CBRT_F80, 311 RTLIB::CBRT_F128, 312 RTLIB::CBRT_PPCF128)); 313 } 314 315 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { 316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 317 RTLIB::CEIL_F32, 318 RTLIB::CEIL_F64, 319 RTLIB::CEIL_F80, 320 RTLIB::CEIL_F128, 321 RTLIB::CEIL_PPCF128)); 322 } 323 324 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { 325 SDValue LHS = GetSoftenedFloat(N->getOperand(0)); 326 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 327 SDLoc dl(N); 328 329 EVT LVT = LHS.getValueType(); 330 EVT RVT = RHS.getValueType(); 331 332 unsigned LSize = LVT.getSizeInBits(); 333 unsigned RSize = RVT.getSizeInBits(); 334 335 // First get the sign bit of second operand. 336 SDValue SignBit = DAG.getNode( 337 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 338 DAG.getConstant(RSize - 1, dl, 339 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 340 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 341 342 // Shift right or sign-extend it if the two operands have different types. 343 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 344 if (SizeDiff > 0) { 345 SignBit = 346 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 347 DAG.getConstant(SizeDiff, dl, 348 TLI.getShiftAmountTy(SignBit.getValueType(), 349 DAG.getDataLayout()))); 350 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 351 } else if (SizeDiff < 0) { 352 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 353 SignBit = 354 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 355 DAG.getConstant(-SizeDiff, dl, 356 TLI.getShiftAmountTy(SignBit.getValueType(), 357 DAG.getDataLayout()))); 358 } 359 360 // Clear the sign bit of the first operand. 361 SDValue Mask = DAG.getNode( 362 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 363 DAG.getConstant(LSize - 1, dl, 364 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 365 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 366 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 367 368 // Or the value with the sign bit. 369 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 370 } 371 372 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { 373 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 374 RTLIB::COS_F32, 375 RTLIB::COS_F64, 376 RTLIB::COS_F80, 377 RTLIB::COS_F128, 378 RTLIB::COS_PPCF128)); 379 } 380 381 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { 382 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 383 RTLIB::DIV_F32, 384 RTLIB::DIV_F64, 385 RTLIB::DIV_F80, 386 RTLIB::DIV_F128, 387 RTLIB::DIV_PPCF128)); 388 } 389 390 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { 391 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 392 RTLIB::EXP_F32, 393 RTLIB::EXP_F64, 394 RTLIB::EXP_F80, 395 RTLIB::EXP_F128, 396 RTLIB::EXP_PPCF128)); 397 } 398 399 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { 400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 401 RTLIB::EXP2_F32, 402 RTLIB::EXP2_F64, 403 RTLIB::EXP2_F80, 404 RTLIB::EXP2_F128, 405 RTLIB::EXP2_PPCF128)); 406 } 407 408 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { 409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 410 RTLIB::FLOOR_F32, 411 RTLIB::FLOOR_F64, 412 RTLIB::FLOOR_F80, 413 RTLIB::FLOOR_F128, 414 RTLIB::FLOOR_PPCF128)); 415 } 416 417 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { 418 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 419 RTLIB::LOG_F32, 420 RTLIB::LOG_F64, 421 RTLIB::LOG_F80, 422 RTLIB::LOG_F128, 423 RTLIB::LOG_PPCF128)); 424 } 425 426 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { 427 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 428 RTLIB::LOG2_F32, 429 RTLIB::LOG2_F64, 430 RTLIB::LOG2_F80, 431 RTLIB::LOG2_F128, 432 RTLIB::LOG2_PPCF128)); 433 } 434 435 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { 436 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 437 RTLIB::LOG10_F32, 438 RTLIB::LOG10_F64, 439 RTLIB::LOG10_F80, 440 RTLIB::LOG10_F128, 441 RTLIB::LOG10_PPCF128)); 442 } 443 444 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { 445 bool IsStrict = N->isStrictFPOpcode(); 446 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 447 unsigned Offset = IsStrict ? 1 : 0; 448 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 449 GetSoftenedFloat(N->getOperand(1 + Offset)), 450 GetSoftenedFloat(N->getOperand(2 + Offset)) }; 451 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 452 TargetLowering::MakeLibCallOptions CallOptions; 453 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(), 454 N->getOperand(1 + Offset).getValueType(), 455 N->getOperand(2 + Offset).getValueType() }; 456 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 457 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, 458 GetFPLibCall(N->getValueType(0), 459 RTLIB::FMA_F32, 460 RTLIB::FMA_F64, 461 RTLIB::FMA_F80, 462 RTLIB::FMA_F128, 463 RTLIB::FMA_PPCF128), 464 NVT, Ops, CallOptions, SDLoc(N), Chain); 465 if (IsStrict) 466 ReplaceValueWith(SDValue(N, 1), Tmp.second); 467 return Tmp.first; 468 } 469 470 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { 471 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 472 RTLIB::MUL_F32, 473 RTLIB::MUL_F64, 474 RTLIB::MUL_F80, 475 RTLIB::MUL_F128, 476 RTLIB::MUL_PPCF128)); 477 } 478 479 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { 480 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 481 RTLIB::NEARBYINT_F32, 482 RTLIB::NEARBYINT_F64, 483 RTLIB::NEARBYINT_F80, 484 RTLIB::NEARBYINT_F128, 485 RTLIB::NEARBYINT_PPCF128)); 486 } 487 488 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { 489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 490 SDLoc dl(N); 491 492 // Expand Y = FNEG(X) -> Y = X ^ sign mask 493 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); 494 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), 495 DAG.getConstant(SignMask, dl, NVT)); 496 } 497 498 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { 499 bool IsStrict = N->isStrictFPOpcode(); 500 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 501 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 502 503 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 504 505 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { 506 Op = GetPromotedFloat(Op); 507 // If the promotion did the FP_EXTEND to the destination type for us, 508 // there's nothing left to do here. 509 if (Op.getValueType() == N->getValueType(0)) 510 return BitConvertToInteger(Op); 511 } 512 513 // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's 514 // entirely possible for both f16 and f32 to be legal, so use the fully 515 // hard-float FP_EXTEND rather than FP16_TO_FP. 516 if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) { 517 if (IsStrict) { 518 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 519 { MVT::f32, MVT::Other }, { Chain, Op }); 520 Chain = Op.getValue(1); 521 } else { 522 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); 523 } 524 } 525 526 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); 527 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 528 TargetLowering::MakeLibCallOptions CallOptions; 529 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 530 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 531 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 532 CallOptions, SDLoc(N), 533 Chain); 534 if (IsStrict) 535 ReplaceValueWith(SDValue(N, 1), Tmp.second); 536 return Tmp.first; 537 } 538 539 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 540 // nodes? 541 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { 542 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 543 SDValue Op = N->getOperand(0); 544 TargetLowering::MakeLibCallOptions CallOptions; 545 EVT OpsVT[1] = { N->getOperand(0).getValueType() }; 546 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 547 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op, 548 CallOptions, SDLoc(N)).first; 549 if (N->getValueType(0) == MVT::f32) 550 return Res32; 551 552 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 553 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0)); 554 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 555 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first; 556 } 557 558 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { 559 bool IsStrict = N->isStrictFPOpcode(); 560 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 561 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 562 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 563 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); 564 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 565 TargetLowering::MakeLibCallOptions CallOptions; 566 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 567 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 568 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 569 CallOptions, SDLoc(N), 570 Chain); 571 if (IsStrict) 572 ReplaceValueWith(SDValue(N, 1), Tmp.second); 573 return Tmp.first; 574 } 575 576 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { 577 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 578 RTLIB::POW_F32, 579 RTLIB::POW_F64, 580 RTLIB::POW_F80, 581 RTLIB::POW_F128, 582 RTLIB::POW_PPCF128)); 583 } 584 585 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) { 586 bool IsStrict = N->isStrictFPOpcode(); 587 unsigned Offset = IsStrict ? 1 : 0; 588 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 || 589 N->getOperand(1 + Offset).getValueType() == MVT::i32) && 590 "Unsupported power type!"); 591 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0)); 592 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 593 if (!TLI.getLibcallName(LC)) { 594 // Some targets don't have a powi libcall; use pow instead. 595 // FIXME: Implement this if some target needs it. 596 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow"); 597 return DAG.getUNDEF(N->getValueType(0)); 598 } 599 600 if (DAG.getLibInfo().getIntSize() != 601 N->getOperand(1 + Offset).getValueType().getSizeInBits()) { 602 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI 603 // would use the wrong type for the argument. 604 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); 605 return DAG.getUNDEF(N->getValueType(0)); 606 } 607 608 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 609 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 610 N->getOperand(1 + Offset) }; 611 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 612 TargetLowering::MakeLibCallOptions CallOptions; 613 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 614 N->getOperand(1 + Offset).getValueType() }; 615 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 616 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 617 CallOptions, SDLoc(N), 618 Chain); 619 if (IsStrict) 620 ReplaceValueWith(SDValue(N, 1), Tmp.second); 621 return Tmp.first; 622 } 623 624 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { 625 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 626 RTLIB::REM_F32, 627 RTLIB::REM_F64, 628 RTLIB::REM_F80, 629 RTLIB::REM_F128, 630 RTLIB::REM_PPCF128)); 631 } 632 633 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { 634 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 635 RTLIB::RINT_F32, 636 RTLIB::RINT_F64, 637 RTLIB::RINT_F80, 638 RTLIB::RINT_F128, 639 RTLIB::RINT_PPCF128)); 640 } 641 642 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { 643 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 644 RTLIB::ROUND_F32, 645 RTLIB::ROUND_F64, 646 RTLIB::ROUND_F80, 647 RTLIB::ROUND_F128, 648 RTLIB::ROUND_PPCF128)); 649 } 650 651 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { 652 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 653 RTLIB::ROUNDEVEN_F32, 654 RTLIB::ROUNDEVEN_F64, 655 RTLIB::ROUNDEVEN_F80, 656 RTLIB::ROUNDEVEN_F128, 657 RTLIB::ROUNDEVEN_PPCF128)); 658 } 659 660 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { 661 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 662 RTLIB::SIN_F32, 663 RTLIB::SIN_F64, 664 RTLIB::SIN_F80, 665 RTLIB::SIN_F128, 666 RTLIB::SIN_PPCF128)); 667 } 668 669 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { 670 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 671 RTLIB::SQRT_F32, 672 RTLIB::SQRT_F64, 673 RTLIB::SQRT_F80, 674 RTLIB::SQRT_F128, 675 RTLIB::SQRT_PPCF128)); 676 } 677 678 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { 679 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 680 RTLIB::SUB_F32, 681 RTLIB::SUB_F64, 682 RTLIB::SUB_F80, 683 RTLIB::SUB_F128, 684 RTLIB::SUB_PPCF128)); 685 } 686 687 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { 688 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 689 RTLIB::TRUNC_F32, 690 RTLIB::TRUNC_F64, 691 RTLIB::TRUNC_F80, 692 RTLIB::TRUNC_F128, 693 RTLIB::TRUNC_PPCF128)); 694 } 695 696 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { 697 LoadSDNode *L = cast<LoadSDNode>(N); 698 EVT VT = N->getValueType(0); 699 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 700 SDLoc dl(N); 701 702 auto MMOFlags = 703 L->getMemOperand()->getFlags() & 704 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 705 SDValue NewL; 706 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 707 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, 708 L->getChain(), L->getBasePtr(), L->getOffset(), 709 L->getPointerInfo(), NVT, L->getOriginalAlign(), 710 MMOFlags, L->getAAInfo()); 711 // Legalized the chain result - switch anything that used the old chain to 712 // use the new one. 713 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 714 return NewL; 715 } 716 717 // Do a non-extending load followed by FP_EXTEND. 718 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(), 719 dl, L->getChain(), L->getBasePtr(), L->getOffset(), 720 L->getPointerInfo(), L->getMemoryVT(), 721 L->getOriginalAlign(), MMOFlags, L->getAAInfo()); 722 // Legalized the chain result - switch anything that used the old chain to 723 // use the new one. 724 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 725 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL); 726 return BitConvertToInteger(ExtendNode); 727 } 728 729 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { 730 SDValue LHS = GetSoftenedFloat(N->getOperand(1)); 731 SDValue RHS = GetSoftenedFloat(N->getOperand(2)); 732 return DAG.getSelect(SDLoc(N), 733 LHS.getValueType(), N->getOperand(0), LHS, RHS); 734 } 735 736 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { 737 SDValue LHS = GetSoftenedFloat(N->getOperand(2)); 738 SDValue RHS = GetSoftenedFloat(N->getOperand(3)); 739 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 740 LHS.getValueType(), N->getOperand(0), 741 N->getOperand(1), LHS, RHS, N->getOperand(4)); 742 } 743 744 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { 745 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 746 N->getValueType(0))); 747 } 748 749 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { 750 SDValue Chain = N->getOperand(0); // Get the chain. 751 SDValue Ptr = N->getOperand(1); // Get the pointer. 752 EVT VT = N->getValueType(0); 753 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 754 SDLoc dl(N); 755 756 SDValue NewVAARG; 757 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), 758 N->getConstantOperandVal(3)); 759 760 // Legalized the chain result - switch anything that used the old chain to 761 // use the new one. 762 if (N != NewVAARG.getValue(1).getNode()) 763 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1)); 764 return NewVAARG; 765 } 766 767 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { 768 bool IsStrict = N->isStrictFPOpcode(); 769 bool Signed = N->getOpcode() == ISD::SINT_TO_FP || 770 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 771 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 772 EVT RVT = N->getValueType(0); 773 EVT NVT = EVT(); 774 SDLoc dl(N); 775 776 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to 777 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly 778 // match. Look for an appropriate libcall. 779 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 780 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 781 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { 782 NVT = (MVT::SimpleValueType)t; 783 // The source needs to big enough to hold the operand. 784 if (NVT.bitsGE(SVT)) 785 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT); 786 } 787 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 788 789 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 790 // Sign/zero extend the argument if the libcall takes a larger type. 791 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 792 NVT, N->getOperand(IsStrict ? 1 : 0)); 793 TargetLowering::MakeLibCallOptions CallOptions; 794 CallOptions.setSExt(Signed); 795 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 796 std::pair<SDValue, SDValue> Tmp = 797 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), 798 Op, CallOptions, dl, Chain); 799 800 if (IsStrict) 801 ReplaceValueWith(SDValue(N, 1), Tmp.second); 802 return Tmp.first; 803 } 804 805 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) { 806 // Expand and soften recursively. 807 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 808 return SDValue(); 809 } 810 811 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) { 812 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 813 return SDValue(); 814 } 815 816 //===----------------------------------------------------------------------===// 817 // Convert Float Operand to Integer 818 //===----------------------------------------------------------------------===// 819 820 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { 821 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG); 822 dbgs() << "\n"); 823 SDValue Res = SDValue(); 824 825 switch (N->getOpcode()) { 826 default: 827 #ifndef NDEBUG 828 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": "; 829 N->dump(&DAG); dbgs() << "\n"; 830 #endif 831 llvm_unreachable("Do not know how to soften this operator's operand!"); 832 833 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; 834 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; 835 case ISD::STRICT_FP_TO_FP16: 836 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes 837 case ISD::FP_TO_BF16: 838 case ISD::STRICT_FP_ROUND: 839 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; 840 case ISD::STRICT_FP_TO_SINT: 841 case ISD::STRICT_FP_TO_UINT: 842 case ISD::FP_TO_SINT: 843 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; 844 case ISD::FP_TO_SINT_SAT: 845 case ISD::FP_TO_UINT_SAT: 846 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; 847 case ISD::STRICT_LROUND: 848 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; 849 case ISD::STRICT_LLROUND: 850 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; 851 case ISD::STRICT_LRINT: 852 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; 853 case ISD::STRICT_LLRINT: 854 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; 855 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; 856 case ISD::STRICT_FSETCC: 857 case ISD::STRICT_FSETCCS: 858 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; 859 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; 860 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; 861 } 862 863 // If the result is null, the sub-method took care of registering results etc. 864 if (!Res.getNode()) return false; 865 866 // If the result is N, the sub-method updated N in place. Tell the legalizer 867 // core about this to re-analyze. 868 if (Res.getNode() == N) 869 return true; 870 871 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 872 "Invalid operand softening"); 873 874 ReplaceValueWith(SDValue(N, 0), Res); 875 return false; 876 } 877 878 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { 879 SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); 880 881 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 882 } 883 884 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { 885 // We actually deal with the partially-softened FP_TO_FP16 node too, which 886 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. 887 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || 888 N->getOpcode() == ISD::STRICT_FP_TO_FP16 || 889 N->getOpcode() == ISD::FP_TO_BF16 || 890 N->getOpcode() == ISD::STRICT_FP_ROUND); 891 892 bool IsStrict = N->isStrictFPOpcode(); 893 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 894 EVT SVT = Op.getValueType(); 895 EVT RVT = N->getValueType(0); 896 EVT FloatRVT = RVT; 897 if (N->getOpcode() == ISD::FP_TO_FP16 || 898 N->getOpcode() == ISD::STRICT_FP_TO_FP16) 899 FloatRVT = MVT::f16; 900 else if (N->getOpcode() == ISD::FP_TO_BF16) 901 FloatRVT = MVT::bf16; 902 903 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT); 904 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 905 906 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 907 Op = GetSoftenedFloat(Op); 908 TargetLowering::MakeLibCallOptions CallOptions; 909 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 910 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 911 CallOptions, SDLoc(N), 912 Chain); 913 if (IsStrict) { 914 ReplaceValueWith(SDValue(N, 1), Tmp.second); 915 ReplaceValueWith(SDValue(N, 0), Tmp.first); 916 return SDValue(); 917 } 918 return Tmp.first; 919 } 920 921 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 922 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 923 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 924 925 EVT VT = NewLHS.getValueType(); 926 NewLHS = GetSoftenedFloat(NewLHS); 927 NewRHS = GetSoftenedFloat(NewRHS); 928 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 929 N->getOperand(2), N->getOperand(3)); 930 931 // If softenSetCCOperands returned a scalar, we need to compare the result 932 // against zero to select between true and false values. 933 if (!NewRHS.getNode()) { 934 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 935 CCCode = ISD::SETNE; 936 } 937 938 // Update N to have the operands specified. 939 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 940 DAG.getCondCode(CCCode), NewLHS, NewRHS, 941 N->getOperand(4)), 942 0); 943 } 944 945 // Even if the result type is legal, no libcall may exactly match. (e.g. We 946 // don't have FP-i8 conversions) This helper method looks for an appropriate 947 // promoted libcall. 948 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, 949 bool Signed) { 950 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 951 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 952 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 953 ++IntVT) { 954 Promoted = (MVT::SimpleValueType)IntVT; 955 // The type needs to big enough to hold the result. 956 if (Promoted.bitsGE(RetVT)) 957 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted) 958 : RTLIB::getFPTOUINT(SrcVT, Promoted); 959 } 960 return LC; 961 } 962 963 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { 964 bool IsStrict = N->isStrictFPOpcode(); 965 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 966 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 967 968 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 969 EVT SVT = Op.getValueType(); 970 EVT RVT = N->getValueType(0); 971 EVT NVT = EVT(); 972 SDLoc dl(N); 973 974 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to 975 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly 976 // match, eg. we don't have fp -> i8 conversions. 977 // Look for an appropriate libcall. 978 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed); 979 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 980 "Unsupported FP_TO_XINT!"); 981 982 Op = GetSoftenedFloat(Op); 983 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 984 TargetLowering::MakeLibCallOptions CallOptions; 985 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 986 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 987 CallOptions, dl, Chain); 988 989 // Truncate the result if the libcall returns a larger type. 990 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first); 991 992 if (!IsStrict) 993 return Res; 994 995 ReplaceValueWith(SDValue(N, 1), Tmp.second); 996 ReplaceValueWith(SDValue(N, 0), Res); 997 return SDValue(); 998 } 999 1000 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { 1001 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 1002 return Res; 1003 } 1004 1005 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { 1006 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1007 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1008 1009 EVT VT = NewLHS.getValueType(); 1010 NewLHS = GetSoftenedFloat(NewLHS); 1011 NewRHS = GetSoftenedFloat(NewRHS); 1012 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1013 N->getOperand(0), N->getOperand(1)); 1014 1015 // If softenSetCCOperands returned a scalar, we need to compare the result 1016 // against zero to select between true and false values. 1017 if (!NewRHS.getNode()) { 1018 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1019 CCCode = ISD::SETNE; 1020 } 1021 1022 // Update N to have the operands specified. 1023 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1024 N->getOperand(2), N->getOperand(3), 1025 DAG.getCondCode(CCCode)), 1026 0); 1027 } 1028 1029 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { 1030 bool IsStrict = N->isStrictFPOpcode(); 1031 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); 1032 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1); 1033 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1034 ISD::CondCode CCCode = 1035 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1036 1037 EVT VT = Op0.getValueType(); 1038 SDValue NewLHS = GetSoftenedFloat(Op0); 1039 SDValue NewRHS = GetSoftenedFloat(Op1); 1040 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1, 1041 Chain, N->getOpcode() == ISD::STRICT_FSETCCS); 1042 1043 // Update N to have the operands specified. 1044 if (NewRHS.getNode()) { 1045 if (IsStrict) 1046 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS, 1047 NewRHS, DAG.getCondCode(CCCode)); 1048 else 1049 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1050 DAG.getCondCode(CCCode)), 0); 1051 } 1052 1053 // Otherwise, softenSetCCOperands returned a scalar, use it. 1054 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && 1055 "Unexpected setcc expansion!"); 1056 1057 if (IsStrict) { 1058 ReplaceValueWith(SDValue(N, 0), NewLHS); 1059 ReplaceValueWith(SDValue(N, 1), Chain); 1060 return SDValue(); 1061 } 1062 return NewLHS; 1063 } 1064 1065 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { 1066 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1067 assert(OpNo == 1 && "Can only soften the stored value!"); 1068 StoreSDNode *ST = cast<StoreSDNode>(N); 1069 SDValue Val = ST->getValue(); 1070 SDLoc dl(N); 1071 1072 if (ST->isTruncatingStore()) 1073 // Do an FP_ROUND followed by a non-truncating store. 1074 Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), 1075 Val, DAG.getIntPtrConstant(0, dl))); 1076 else 1077 Val = GetSoftenedFloat(Val); 1078 1079 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(), 1080 ST->getMemOperand()); 1081 } 1082 1083 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { 1084 SDValue LHS = N->getOperand(0); 1085 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 1086 SDLoc dl(N); 1087 1088 EVT LVT = LHS.getValueType(); 1089 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits()); 1090 EVT RVT = RHS.getValueType(); 1091 1092 unsigned LSize = LVT.getSizeInBits(); 1093 unsigned RSize = RVT.getSizeInBits(); 1094 1095 // Shift right or sign-extend it if the two operands have different types. 1096 int SizeDiff = RSize - LSize; 1097 if (SizeDiff > 0) { 1098 RHS = 1099 DAG.getNode(ISD::SRL, dl, RVT, RHS, 1100 DAG.getConstant(SizeDiff, dl, 1101 TLI.getShiftAmountTy(RHS.getValueType(), 1102 DAG.getDataLayout()))); 1103 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS); 1104 } else if (SizeDiff < 0) { 1105 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS); 1106 RHS = 1107 DAG.getNode(ISD::SHL, dl, ILVT, RHS, 1108 DAG.getConstant(-SizeDiff, dl, 1109 TLI.getShiftAmountTy(RHS.getValueType(), 1110 DAG.getDataLayout()))); 1111 } 1112 1113 RHS = DAG.getBitcast(LVT, RHS); 1114 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS); 1115 } 1116 1117 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { 1118 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1119 bool IsStrict = N->isStrictFPOpcode(); 1120 unsigned Offset = IsStrict ? 1 : 0; 1121 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 1122 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1123 TargetLowering::MakeLibCallOptions CallOptions; 1124 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 1125 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 1126 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1127 CallOptions, SDLoc(N), 1128 Chain); 1129 if (IsStrict) { 1130 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1131 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1132 return SDValue(); 1133 } 1134 1135 return Tmp.first; 1136 } 1137 1138 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { 1139 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1140 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1141 RTLIB::LROUND_F32, 1142 RTLIB::LROUND_F64, 1143 RTLIB::LROUND_F80, 1144 RTLIB::LROUND_F128, 1145 RTLIB::LROUND_PPCF128)); 1146 } 1147 1148 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { 1149 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1150 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1151 RTLIB::LLROUND_F32, 1152 RTLIB::LLROUND_F64, 1153 RTLIB::LLROUND_F80, 1154 RTLIB::LLROUND_F128, 1155 RTLIB::LLROUND_PPCF128)); 1156 } 1157 1158 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { 1159 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1160 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1161 RTLIB::LRINT_F32, 1162 RTLIB::LRINT_F64, 1163 RTLIB::LRINT_F80, 1164 RTLIB::LRINT_F128, 1165 RTLIB::LRINT_PPCF128)); 1166 } 1167 1168 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { 1169 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1170 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1171 RTLIB::LLRINT_F32, 1172 RTLIB::LLRINT_F64, 1173 RTLIB::LLRINT_F80, 1174 RTLIB::LLRINT_F128, 1175 RTLIB::LLRINT_PPCF128)); 1176 } 1177 1178 //===----------------------------------------------------------------------===// 1179 // Float Result Expansion 1180 //===----------------------------------------------------------------------===// 1181 1182 /// ExpandFloatResult - This method is called when the specified result of the 1183 /// specified node is found to need expansion. At this point, the node may also 1184 /// have invalid operands or may have other results that need promotion, we just 1185 /// know that (at least) one result needs expansion. 1186 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { 1187 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n"); 1188 SDValue Lo, Hi; 1189 Lo = Hi = SDValue(); 1190 1191 // See if the target wants to custom expand this node. 1192 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1193 return; 1194 1195 switch (N->getOpcode()) { 1196 default: 1197 #ifndef NDEBUG 1198 dbgs() << "ExpandFloatResult #" << ResNo << ": "; 1199 N->dump(&DAG); dbgs() << "\n"; 1200 #endif 1201 llvm_unreachable("Do not know how to expand the result of this operator!"); 1202 1203 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1204 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 1205 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1206 1207 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1208 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1209 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1210 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1211 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1212 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1213 1214 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; 1215 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; 1216 case ISD::STRICT_FMINNUM: 1217 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; 1218 case ISD::STRICT_FMAXNUM: 1219 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; 1220 case ISD::STRICT_FADD: 1221 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; 1222 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; 1223 case ISD::STRICT_FCEIL: 1224 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; 1225 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; 1226 case ISD::STRICT_FCOS: 1227 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; 1228 case ISD::STRICT_FDIV: 1229 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; 1230 case ISD::STRICT_FEXP: 1231 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; 1232 case ISD::STRICT_FEXP2: 1233 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; 1234 case ISD::STRICT_FFLOOR: 1235 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; 1236 case ISD::STRICT_FLOG: 1237 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; 1238 case ISD::STRICT_FLOG2: 1239 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; 1240 case ISD::STRICT_FLOG10: 1241 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; 1242 case ISD::STRICT_FMA: 1243 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; 1244 case ISD::STRICT_FMUL: 1245 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; 1246 case ISD::STRICT_FNEARBYINT: 1247 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; 1248 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; 1249 case ISD::STRICT_FP_EXTEND: 1250 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; 1251 case ISD::STRICT_FPOW: 1252 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; 1253 case ISD::STRICT_FPOWI: 1254 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; 1255 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; 1256 case ISD::STRICT_FRINT: 1257 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; 1258 case ISD::STRICT_FROUND: 1259 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; 1260 case ISD::STRICT_FROUNDEVEN: 1261 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; 1262 case ISD::STRICT_FSIN: 1263 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; 1264 case ISD::STRICT_FSQRT: 1265 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; 1266 case ISD::STRICT_FSUB: 1267 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; 1268 case ISD::STRICT_FTRUNC: 1269 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; 1270 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; 1271 case ISD::STRICT_SINT_TO_FP: 1272 case ISD::STRICT_UINT_TO_FP: 1273 case ISD::SINT_TO_FP: 1274 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; 1275 case ISD::STRICT_FREM: 1276 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; 1277 } 1278 1279 // If Lo/Hi is null, the sub-method took care of registering results etc. 1280 if (Lo.getNode()) 1281 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi); 1282 } 1283 1284 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, 1285 SDValue &Hi) { 1286 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1287 assert(NVT.getSizeInBits() == 64 && 1288 "Do not know how to expand this float constant!"); 1289 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); 1290 SDLoc dl(N); 1291 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1292 APInt(64, C.getRawData()[1])), 1293 dl, NVT); 1294 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1295 APInt(64, C.getRawData()[0])), 1296 dl, NVT); 1297 } 1298 1299 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, 1300 SDValue &Lo, SDValue &Hi) { 1301 bool IsStrict = N->isStrictFPOpcode(); 1302 unsigned Offset = IsStrict ? 1 : 0; 1303 SDValue Op = N->getOperand(0 + Offset); 1304 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1305 TargetLowering::MakeLibCallOptions CallOptions; 1306 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1307 Op, CallOptions, SDLoc(N), 1308 Chain); 1309 if (IsStrict) 1310 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1311 GetPairElements(Tmp.first, Lo, Hi); 1312 } 1313 1314 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, 1315 SDValue &Lo, SDValue &Hi) { 1316 bool IsStrict = N->isStrictFPOpcode(); 1317 unsigned Offset = IsStrict ? 1 : 0; 1318 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) }; 1319 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1320 TargetLowering::MakeLibCallOptions CallOptions; 1321 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1322 Ops, CallOptions, SDLoc(N), 1323 Chain); 1324 if (IsStrict) 1325 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1326 GetPairElements(Tmp.first, Lo, Hi); 1327 } 1328 1329 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, 1330 SDValue &Hi) { 1331 assert(N->getValueType(0) == MVT::ppcf128 && 1332 "Logic only correct for ppcf128!"); 1333 SDLoc dl(N); 1334 SDValue Tmp; 1335 GetExpandedFloat(N->getOperand(0), Lo, Tmp); 1336 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp); 1337 // Lo = Hi==fabs(Hi) ? Lo : -Lo; 1338 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo, 1339 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo), 1340 ISD::SETEQ); 1341 } 1342 1343 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, 1344 SDValue &Hi) { 1345 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1346 RTLIB::FMIN_F32, RTLIB::FMIN_F64, 1347 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 1348 RTLIB::FMIN_PPCF128), Lo, Hi); 1349 } 1350 1351 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, 1352 SDValue &Hi) { 1353 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1354 RTLIB::FMAX_F32, RTLIB::FMAX_F64, 1355 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 1356 RTLIB::FMAX_PPCF128), Lo, Hi); 1357 } 1358 1359 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, 1360 SDValue &Hi) { 1361 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1362 RTLIB::ADD_F32, RTLIB::ADD_F64, 1363 RTLIB::ADD_F80, RTLIB::ADD_F128, 1364 RTLIB::ADD_PPCF128), Lo, Hi); 1365 } 1366 1367 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, 1368 SDValue &Hi) { 1369 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32, 1370 RTLIB::CBRT_F64, RTLIB::CBRT_F80, 1371 RTLIB::CBRT_F128, 1372 RTLIB::CBRT_PPCF128), Lo, Hi); 1373 } 1374 1375 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, 1376 SDValue &Lo, SDValue &Hi) { 1377 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1378 RTLIB::CEIL_F32, RTLIB::CEIL_F64, 1379 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 1380 RTLIB::CEIL_PPCF128), Lo, Hi); 1381 } 1382 1383 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, 1384 SDValue &Lo, SDValue &Hi) { 1385 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1386 RTLIB::COPYSIGN_F32, 1387 RTLIB::COPYSIGN_F64, 1388 RTLIB::COPYSIGN_F80, 1389 RTLIB::COPYSIGN_F128, 1390 RTLIB::COPYSIGN_PPCF128), Lo, Hi); 1391 } 1392 1393 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, 1394 SDValue &Lo, SDValue &Hi) { 1395 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1396 RTLIB::COS_F32, RTLIB::COS_F64, 1397 RTLIB::COS_F80, RTLIB::COS_F128, 1398 RTLIB::COS_PPCF128), Lo, Hi); 1399 } 1400 1401 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, 1402 SDValue &Hi) { 1403 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1404 RTLIB::DIV_F32, 1405 RTLIB::DIV_F64, 1406 RTLIB::DIV_F80, 1407 RTLIB::DIV_F128, 1408 RTLIB::DIV_PPCF128), Lo, Hi); 1409 } 1410 1411 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, 1412 SDValue &Lo, SDValue &Hi) { 1413 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1414 RTLIB::EXP_F32, RTLIB::EXP_F64, 1415 RTLIB::EXP_F80, RTLIB::EXP_F128, 1416 RTLIB::EXP_PPCF128), Lo, Hi); 1417 } 1418 1419 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, 1420 SDValue &Lo, SDValue &Hi) { 1421 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1422 RTLIB::EXP2_F32, RTLIB::EXP2_F64, 1423 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 1424 RTLIB::EXP2_PPCF128), Lo, Hi); 1425 } 1426 1427 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, 1428 SDValue &Lo, SDValue &Hi) { 1429 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1430 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 1431 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 1432 RTLIB::FLOOR_PPCF128), Lo, Hi); 1433 } 1434 1435 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, 1436 SDValue &Lo, SDValue &Hi) { 1437 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1438 RTLIB::LOG_F32, RTLIB::LOG_F64, 1439 RTLIB::LOG_F80, RTLIB::LOG_F128, 1440 RTLIB::LOG_PPCF128), Lo, Hi); 1441 } 1442 1443 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, 1444 SDValue &Lo, SDValue &Hi) { 1445 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1446 RTLIB::LOG2_F32, RTLIB::LOG2_F64, 1447 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 1448 RTLIB::LOG2_PPCF128), Lo, Hi); 1449 } 1450 1451 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, 1452 SDValue &Lo, SDValue &Hi) { 1453 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1454 RTLIB::LOG10_F32, RTLIB::LOG10_F64, 1455 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 1456 RTLIB::LOG10_PPCF128), Lo, Hi); 1457 } 1458 1459 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, 1460 SDValue &Hi) { 1461 bool IsStrict = N->isStrictFPOpcode(); 1462 unsigned Offset = IsStrict ? 1 : 0; 1463 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset), 1464 N->getOperand(2 + Offset) }; 1465 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1466 TargetLowering::MakeLibCallOptions CallOptions; 1467 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), 1468 RTLIB::FMA_F32, 1469 RTLIB::FMA_F64, 1470 RTLIB::FMA_F80, 1471 RTLIB::FMA_F128, 1472 RTLIB::FMA_PPCF128), 1473 N->getValueType(0), Ops, CallOptions, 1474 SDLoc(N), Chain); 1475 if (IsStrict) 1476 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1477 GetPairElements(Tmp.first, Lo, Hi); 1478 } 1479 1480 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, 1481 SDValue &Hi) { 1482 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1483 RTLIB::MUL_F32, 1484 RTLIB::MUL_F64, 1485 RTLIB::MUL_F80, 1486 RTLIB::MUL_F128, 1487 RTLIB::MUL_PPCF128), Lo, Hi); 1488 } 1489 1490 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, 1491 SDValue &Lo, SDValue &Hi) { 1492 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1493 RTLIB::NEARBYINT_F32, 1494 RTLIB::NEARBYINT_F64, 1495 RTLIB::NEARBYINT_F80, 1496 RTLIB::NEARBYINT_F128, 1497 RTLIB::NEARBYINT_PPCF128), Lo, Hi); 1498 } 1499 1500 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, 1501 SDValue &Hi) { 1502 SDLoc dl(N); 1503 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1504 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo); 1505 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); 1506 } 1507 1508 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, 1509 SDValue &Hi) { 1510 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1511 SDLoc dl(N); 1512 bool IsStrict = N->isStrictFPOpcode(); 1513 1514 SDValue Chain; 1515 if (IsStrict) { 1516 // If the expanded type is the same as the input type, just bypass the node. 1517 if (NVT == N->getOperand(1).getValueType()) { 1518 Hi = N->getOperand(1); 1519 Chain = N->getOperand(0); 1520 } else { 1521 // Other we need to extend. 1522 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, 1523 { N->getOperand(0), N->getOperand(1) }); 1524 Chain = Hi.getValue(1); 1525 } 1526 } else { 1527 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0)); 1528 } 1529 1530 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1531 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1532 1533 if (IsStrict) 1534 ReplaceValueWith(SDValue(N, 1), Chain); 1535 } 1536 1537 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, 1538 SDValue &Lo, SDValue &Hi) { 1539 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1540 RTLIB::POW_F32, RTLIB::POW_F64, 1541 RTLIB::POW_F80, RTLIB::POW_F128, 1542 RTLIB::POW_PPCF128), Lo, Hi); 1543 } 1544 1545 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, 1546 SDValue &Lo, SDValue &Hi) { 1547 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi); 1548 } 1549 1550 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, 1551 SDValue &Lo, SDValue &Hi) { 1552 assert(N->getValueType(0) == MVT::ppcf128 && 1553 "Logic only correct for ppcf128!"); 1554 1555 SDLoc dl(N); 1556 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1557 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo); 1558 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi); 1559 } 1560 1561 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, 1562 SDValue &Lo, SDValue &Hi) { 1563 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1564 RTLIB::REM_F32, RTLIB::REM_F64, 1565 RTLIB::REM_F80, RTLIB::REM_F128, 1566 RTLIB::REM_PPCF128), Lo, Hi); 1567 } 1568 1569 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, 1570 SDValue &Lo, SDValue &Hi) { 1571 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1572 RTLIB::RINT_F32, RTLIB::RINT_F64, 1573 RTLIB::RINT_F80, RTLIB::RINT_F128, 1574 RTLIB::RINT_PPCF128), Lo, Hi); 1575 } 1576 1577 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, 1578 SDValue &Lo, SDValue &Hi) { 1579 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1580 RTLIB::ROUND_F32, 1581 RTLIB::ROUND_F64, 1582 RTLIB::ROUND_F80, 1583 RTLIB::ROUND_F128, 1584 RTLIB::ROUND_PPCF128), Lo, Hi); 1585 } 1586 1587 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, 1588 SDValue &Lo, SDValue &Hi) { 1589 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1590 RTLIB::ROUNDEVEN_F32, 1591 RTLIB::ROUNDEVEN_F64, 1592 RTLIB::ROUNDEVEN_F80, 1593 RTLIB::ROUNDEVEN_F128, 1594 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); 1595 } 1596 1597 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, 1598 SDValue &Lo, SDValue &Hi) { 1599 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1600 RTLIB::SIN_F32, RTLIB::SIN_F64, 1601 RTLIB::SIN_F80, RTLIB::SIN_F128, 1602 RTLIB::SIN_PPCF128), Lo, Hi); 1603 } 1604 1605 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, 1606 SDValue &Lo, SDValue &Hi) { 1607 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1608 RTLIB::SQRT_F32, RTLIB::SQRT_F64, 1609 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 1610 RTLIB::SQRT_PPCF128), Lo, Hi); 1611 } 1612 1613 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, 1614 SDValue &Hi) { 1615 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1616 RTLIB::SUB_F32, 1617 RTLIB::SUB_F64, 1618 RTLIB::SUB_F80, 1619 RTLIB::SUB_F128, 1620 RTLIB::SUB_PPCF128), Lo, Hi); 1621 } 1622 1623 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, 1624 SDValue &Lo, SDValue &Hi) { 1625 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1626 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 1627 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 1628 RTLIB::TRUNC_PPCF128), Lo, Hi); 1629 } 1630 1631 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, 1632 SDValue &Hi) { 1633 if (ISD::isNormalLoad(N)) { 1634 ExpandRes_NormalLoad(N, Lo, Hi); 1635 return; 1636 } 1637 1638 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1639 LoadSDNode *LD = cast<LoadSDNode>(N); 1640 SDValue Chain = LD->getChain(); 1641 SDValue Ptr = LD->getBasePtr(); 1642 SDLoc dl(N); 1643 1644 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1645 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1646 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1647 1648 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr, 1649 LD->getMemoryVT(), LD->getMemOperand()); 1650 1651 // Remember the chain. 1652 Chain = Hi.getValue(1); 1653 1654 // The low part is zero. 1655 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1656 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1657 1658 // Modified the chain - switch anything that used the old chain to use the 1659 // new one. 1660 ReplaceValueWith(SDValue(LD, 1), Chain); 1661 } 1662 1663 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, 1664 SDValue &Hi) { 1665 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!"); 1666 EVT VT = N->getValueType(0); 1667 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1668 bool Strict = N->isStrictFPOpcode(); 1669 SDValue Src = N->getOperand(Strict ? 1 : 0); 1670 EVT SrcVT = Src.getValueType(); 1671 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP || 1672 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 1673 SDLoc dl(N); 1674 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode(); 1675 1676 // TODO: Any other flags to propagate? 1677 SDNodeFlags Flags; 1678 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept()); 1679 1680 // First do an SINT_TO_FP, whether the original was signed or unsigned. 1681 // When promoting partial word types to i32 we must honor the signedness, 1682 // though. 1683 if (SrcVT.bitsLE(MVT::i32)) { 1684 // The integer can be represented exactly in an f64. 1685 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1686 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1687 if (Strict) { 1688 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), 1689 {Chain, Src}, Flags); 1690 Chain = Hi.getValue(1); 1691 } else 1692 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src); 1693 } else { 1694 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1695 if (SrcVT.bitsLE(MVT::i64)) { 1696 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1697 MVT::i64, Src); 1698 LC = RTLIB::SINTTOFP_I64_PPCF128; 1699 } else if (SrcVT.bitsLE(MVT::i128)) { 1700 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); 1701 LC = RTLIB::SINTTOFP_I128_PPCF128; 1702 } 1703 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 1704 1705 TargetLowering::MakeLibCallOptions CallOptions; 1706 CallOptions.setSExt(true); 1707 std::pair<SDValue, SDValue> Tmp = 1708 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); 1709 if (Strict) 1710 Chain = Tmp.second; 1711 GetPairElements(Tmp.first, Lo, Hi); 1712 } 1713 1714 // No need to complement for unsigned 32-bit integers 1715 if (isSigned || SrcVT.bitsLE(MVT::i32)) { 1716 if (Strict) 1717 ReplaceValueWith(SDValue(N, 1), Chain); 1718 1719 return; 1720 } 1721 1722 // Unsigned - fix up the SINT_TO_FP value just calculated. 1723 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully 1724 // keep semantics correctness if the integer is not exactly representable 1725 // here. See ExpandLegalINT_TO_FP. 1726 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 1727 SrcVT = Src.getValueType(); 1728 1729 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. 1730 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 1731 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 1732 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; 1733 ArrayRef<uint64_t> Parts; 1734 1735 switch (SrcVT.getSimpleVT().SimpleTy) { 1736 default: 1737 llvm_unreachable("Unsupported UINT_TO_FP!"); 1738 case MVT::i32: 1739 Parts = TwoE32; 1740 break; 1741 case MVT::i64: 1742 Parts = TwoE64; 1743 break; 1744 case MVT::i128: 1745 Parts = TwoE128; 1746 break; 1747 } 1748 1749 // TODO: Are there other fast-math-flags to propagate to this FADD? 1750 SDValue NewLo = DAG.getConstantFP( 1751 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128); 1752 if (Strict) { 1753 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other), 1754 {Chain, Hi, NewLo}, Flags); 1755 Chain = Lo.getValue(1); 1756 ReplaceValueWith(SDValue(N, 1), Chain); 1757 } else 1758 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo); 1759 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT), 1760 Lo, Hi, ISD::SETLT); 1761 GetPairElements(Lo, Lo, Hi); 1762 } 1763 1764 1765 //===----------------------------------------------------------------------===// 1766 // Float Operand Expansion 1767 //===----------------------------------------------------------------------===// 1768 1769 /// ExpandFloatOperand - This method is called when the specified operand of the 1770 /// specified node is found to need expansion. At this point, all of the result 1771 /// types of the node are known to be legal, but other operands of the node may 1772 /// need promotion or expansion as well as the specified one. 1773 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { 1774 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n"); 1775 SDValue Res = SDValue(); 1776 1777 // See if the target wants to custom expand this node. 1778 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1779 return false; 1780 1781 switch (N->getOpcode()) { 1782 default: 1783 #ifndef NDEBUG 1784 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": "; 1785 N->dump(&DAG); dbgs() << "\n"; 1786 #endif 1787 llvm_unreachable("Do not know how to expand this operator's operand!"); 1788 1789 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 1790 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1791 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1792 1793 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; 1794 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; 1795 case ISD::STRICT_FP_ROUND: 1796 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; 1797 case ISD::STRICT_FP_TO_SINT: 1798 case ISD::STRICT_FP_TO_UINT: 1799 case ISD::FP_TO_SINT: 1800 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break; 1801 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; 1802 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; 1803 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; 1804 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; 1805 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; 1806 case ISD::STRICT_FSETCC: 1807 case ISD::STRICT_FSETCCS: 1808 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; 1809 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N), 1810 OpNo); break; 1811 } 1812 1813 // If the result is null, the sub-method took care of registering results etc. 1814 if (!Res.getNode()) return false; 1815 1816 // If the result is N, the sub-method updated N in place. Tell the legalizer 1817 // core about this. 1818 if (Res.getNode() == N) 1819 return true; 1820 1821 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1822 "Invalid operand expansion"); 1823 1824 ReplaceValueWith(SDValue(N, 0), Res); 1825 return false; 1826 } 1827 1828 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code 1829 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1830 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, 1831 SDValue &NewRHS, 1832 ISD::CondCode &CCCode, 1833 const SDLoc &dl, SDValue &Chain, 1834 bool IsSignaling) { 1835 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 1836 GetExpandedFloat(NewLHS, LHSLo, LHSHi); 1837 GetExpandedFloat(NewRHS, RHSLo, RHSHi); 1838 1839 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); 1840 1841 // FIXME: This generated code sucks. We want to generate 1842 // FCMPU crN, hi1, hi2 1843 // BNE crN, L: 1844 // FCMPU crN, lo1, lo2 1845 // The following can be improved, but not that much. 1846 SDValue Tmp1, Tmp2, Tmp3, OutputChain; 1847 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1848 RHSHi, ISD::SETOEQ, Chain, IsSignaling); 1849 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1850 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 1851 RHSLo, CCCode, OutputChain, IsSignaling); 1852 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1853 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1854 Tmp1 = 1855 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi, 1856 ISD::SETUNE, OutputChain, IsSignaling); 1857 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1858 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1859 RHSHi, CCCode, OutputChain, IsSignaling); 1860 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1861 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1862 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 1863 NewRHS = SDValue(); // LHS is the result, not a compare. 1864 Chain = OutputChain; 1865 } 1866 1867 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { 1868 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1869 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1870 SDValue Chain; 1871 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 1872 1873 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1874 // against zero to select between true and false values. 1875 if (!NewRHS.getNode()) { 1876 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1877 CCCode = ISD::SETNE; 1878 } 1879 1880 // Update N to have the operands specified. 1881 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1882 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1883 N->getOperand(4)), 0); 1884 } 1885 1886 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { 1887 assert(N->getOperand(1).getValueType() == MVT::ppcf128 && 1888 "Logic only correct for ppcf128!"); 1889 SDValue Lo, Hi; 1890 GetExpandedFloat(N->getOperand(1), Lo, Hi); 1891 // The ppcf128 value is providing only the sign; take it from the 1892 // higher-order double (which must have the larger magnitude). 1893 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), 1894 N->getValueType(0), N->getOperand(0), Hi); 1895 } 1896 1897 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { 1898 bool IsStrict = N->isStrictFPOpcode(); 1899 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && 1900 "Logic only correct for ppcf128!"); 1901 SDValue Lo, Hi; 1902 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi); 1903 1904 if (!IsStrict) 1905 // Round it the rest of the way (e.g. to f32) if needed. 1906 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 1907 N->getValueType(0), Hi, N->getOperand(1)); 1908 1909 // Eliminate the node if the input float type is the same as the output float 1910 // type. 1911 if (Hi.getValueType() == N->getValueType(0)) { 1912 // Connect the output chain to the input chain, unlinking the node. 1913 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 1914 ReplaceValueWith(SDValue(N, 0), Hi); 1915 return SDValue(); 1916 } 1917 1918 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 1919 {N->getValueType(0), MVT::Other}, 1920 {N->getOperand(0), Hi, N->getOperand(2)}); 1921 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1)); 1922 ReplaceValueWith(SDValue(N, 0), Expansion); 1923 return SDValue(); 1924 } 1925 1926 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) { 1927 EVT RVT = N->getValueType(0); 1928 SDLoc dl(N); 1929 1930 bool IsStrict = N->isStrictFPOpcode(); 1931 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 1932 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 1933 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1934 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1935 1936 EVT NVT; 1937 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed); 1938 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 1939 "Unsupported FP_TO_XINT!"); 1940 TargetLowering::MakeLibCallOptions CallOptions; 1941 std::pair<SDValue, SDValue> Tmp = 1942 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); 1943 if (!IsStrict) 1944 return Tmp.first; 1945 1946 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1947 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1948 return SDValue(); 1949 } 1950 1951 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { 1952 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1953 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1954 SDValue Chain; 1955 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 1956 1957 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1958 // against zero to select between true and false values. 1959 if (!NewRHS.getNode()) { 1960 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1961 CCCode = ISD::SETNE; 1962 } 1963 1964 // Update N to have the operands specified. 1965 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1966 N->getOperand(2), N->getOperand(3), 1967 DAG.getCondCode(CCCode)), 0); 1968 } 1969 1970 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { 1971 bool IsStrict = N->isStrictFPOpcode(); 1972 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0); 1973 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1); 1974 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1975 ISD::CondCode CCCode = 1976 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1977 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain, 1978 N->getOpcode() == ISD::STRICT_FSETCCS); 1979 1980 // FloatExpandSetCCOperands always returned a scalar. 1981 assert(!NewRHS.getNode() && "Expect to return scalar"); 1982 assert(NewLHS.getValueType() == N->getValueType(0) && 1983 "Unexpected setcc expansion!"); 1984 if (Chain) { 1985 ReplaceValueWith(SDValue(N, 0), NewLHS); 1986 ReplaceValueWith(SDValue(N, 1), Chain); 1987 return SDValue(); 1988 } 1989 return NewLHS; 1990 } 1991 1992 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { 1993 if (ISD::isNormalStore(N)) 1994 return ExpandOp_NormalStore(N, OpNo); 1995 1996 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1997 assert(OpNo == 1 && "Can only expand the stored value so far"); 1998 StoreSDNode *ST = cast<StoreSDNode>(N); 1999 2000 SDValue Chain = ST->getChain(); 2001 SDValue Ptr = ST->getBasePtr(); 2002 2003 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 2004 ST->getValue().getValueType()); 2005 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2006 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 2007 (void)NVT; 2008 2009 SDValue Lo, Hi; 2010 GetExpandedOp(ST->getValue(), Lo, Hi); 2011 2012 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr, 2013 ST->getMemoryVT(), ST->getMemOperand()); 2014 } 2015 2016 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { 2017 EVT RVT = N->getValueType(0); 2018 EVT RetVT = N->getOperand(0).getValueType(); 2019 TargetLowering::MakeLibCallOptions CallOptions; 2020 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2021 RTLIB::LROUND_F32, 2022 RTLIB::LROUND_F64, 2023 RTLIB::LROUND_F80, 2024 RTLIB::LROUND_F128, 2025 RTLIB::LROUND_PPCF128), 2026 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2027 } 2028 2029 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { 2030 EVT RVT = N->getValueType(0); 2031 EVT RetVT = N->getOperand(0).getValueType(); 2032 TargetLowering::MakeLibCallOptions CallOptions; 2033 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2034 RTLIB::LLROUND_F32, 2035 RTLIB::LLROUND_F64, 2036 RTLIB::LLROUND_F80, 2037 RTLIB::LLROUND_F128, 2038 RTLIB::LLROUND_PPCF128), 2039 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2040 } 2041 2042 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { 2043 EVT RVT = N->getValueType(0); 2044 EVT RetVT = N->getOperand(0).getValueType(); 2045 TargetLowering::MakeLibCallOptions CallOptions; 2046 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2047 RTLIB::LRINT_F32, 2048 RTLIB::LRINT_F64, 2049 RTLIB::LRINT_F80, 2050 RTLIB::LRINT_F128, 2051 RTLIB::LRINT_PPCF128), 2052 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2053 } 2054 2055 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { 2056 EVT RVT = N->getValueType(0); 2057 EVT RetVT = N->getOperand(0).getValueType(); 2058 TargetLowering::MakeLibCallOptions CallOptions; 2059 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2060 RTLIB::LLRINT_F32, 2061 RTLIB::LLRINT_F64, 2062 RTLIB::LLRINT_F80, 2063 RTLIB::LLRINT_F128, 2064 RTLIB::LLRINT_PPCF128), 2065 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2066 } 2067 2068 //===----------------------------------------------------------------------===// 2069 // Float Operand Promotion 2070 //===----------------------------------------------------------------------===// 2071 // 2072 2073 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { 2074 if (OpVT == MVT::f16) { 2075 return ISD::FP16_TO_FP; 2076 } else if (RetVT == MVT::f16) { 2077 return ISD::FP_TO_FP16; 2078 } else if (OpVT == MVT::bf16) { 2079 return ISD::BF16_TO_FP; 2080 } else if (RetVT == MVT::bf16) { 2081 return ISD::FP_TO_BF16; 2082 } 2083 2084 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2085 } 2086 2087 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { 2088 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG); 2089 dbgs() << "\n"); 2090 SDValue R = SDValue(); 2091 2092 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2093 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2094 return false; 2095 } 2096 2097 // Nodes that use a promotion-requiring floating point operand, but doesn't 2098 // produce a promotion-requiring floating point result, need to be legalized 2099 // to use the promoted float operand. Nodes that produce at least one 2100 // promotion-requiring floating point result have their operands legalized as 2101 // a part of PromoteFloatResult. 2102 switch (N->getOpcode()) { 2103 default: 2104 #ifndef NDEBUG 2105 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; 2106 N->dump(&DAG); dbgs() << "\n"; 2107 #endif 2108 llvm_unreachable("Do not know how to promote this operator's operand!"); 2109 2110 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; 2111 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; 2112 case ISD::FP_TO_SINT: 2113 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break; 2114 case ISD::FP_TO_SINT_SAT: 2115 case ISD::FP_TO_UINT_SAT: 2116 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; 2117 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; 2118 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; 2119 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; 2120 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; 2121 } 2122 2123 if (R.getNode()) 2124 ReplaceValueWith(SDValue(N, 0), R); 2125 return false; 2126 } 2127 2128 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { 2129 SDValue Op = N->getOperand(0); 2130 EVT OpVT = Op->getValueType(0); 2131 2132 SDValue Promoted = GetPromotedFloat(N->getOperand(0)); 2133 EVT PromotedVT = Promoted->getValueType(0); 2134 2135 // Convert the promoted float value to the desired IVT. 2136 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); 2137 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), 2138 IVT, Promoted); 2139 // The final result type might not be an scalar so we need a bitcast. The 2140 // bitcast will be further legalized if needed. 2141 return DAG.getBitcast(N->getValueType(0), Convert); 2142 } 2143 2144 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by 2145 // PromoteFloatRes_FCOPYSIGN. 2146 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { 2147 assert (OpNo == 1 && "Only Operand 1 must need promotion here"); 2148 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2149 2150 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2151 N->getOperand(0), Op1); 2152 } 2153 2154 // Convert the promoted float value to the desired integer type 2155 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) { 2156 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2157 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); 2158 } 2159 2160 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, 2161 unsigned OpNo) { 2162 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2163 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, 2164 N->getOperand(1)); 2165 } 2166 2167 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { 2168 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2169 EVT VT = N->getValueType(0); 2170 2171 // Desired VT is same as promoted type. Use promoted float directly. 2172 if (VT == Op->getValueType(0)) 2173 return Op; 2174 2175 // Else, extend the promoted float value to the desired VT. 2176 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); 2177 } 2178 2179 // Promote the float operands used for comparison. The true- and false- 2180 // operands have the same type as the result and are promoted, if needed, by 2181 // PromoteFloatRes_SELECT_CC 2182 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2183 SDValue LHS = GetPromotedFloat(N->getOperand(0)); 2184 SDValue RHS = GetPromotedFloat(N->getOperand(1)); 2185 2186 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 2187 LHS, RHS, N->getOperand(2), N->getOperand(3), 2188 N->getOperand(4)); 2189 } 2190 2191 // Construct a SETCC that compares the promoted values and sets the conditional 2192 // code. 2193 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { 2194 EVT VT = N->getValueType(0); 2195 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2196 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2197 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2198 2199 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); 2200 2201 } 2202 2203 // Lower the promoted Float down to the integer value of same size and construct 2204 // a STORE of the integer value. 2205 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { 2206 StoreSDNode *ST = cast<StoreSDNode>(N); 2207 SDValue Val = ST->getValue(); 2208 SDLoc DL(N); 2209 2210 SDValue Promoted = GetPromotedFloat(Val); 2211 EVT VT = ST->getOperand(1).getValueType(); 2212 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2213 2214 SDValue NewVal; 2215 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, 2216 IVT, Promoted); 2217 2218 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), 2219 ST->getMemOperand()); 2220 } 2221 2222 //===----------------------------------------------------------------------===// 2223 // Float Result Promotion 2224 //===----------------------------------------------------------------------===// 2225 2226 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { 2227 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG); 2228 dbgs() << "\n"); 2229 SDValue R = SDValue(); 2230 2231 // See if the target wants to custom expand this node. 2232 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2233 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2234 return; 2235 } 2236 2237 switch (N->getOpcode()) { 2238 // These opcodes cannot appear if promotion of FP16 is done in the backend 2239 // instead of Clang 2240 case ISD::FP16_TO_FP: 2241 case ISD::FP_TO_FP16: 2242 default: 2243 #ifndef NDEBUG 2244 dbgs() << "PromoteFloatResult #" << ResNo << ": "; 2245 N->dump(&DAG); dbgs() << "\n"; 2246 #endif 2247 llvm_unreachable("Do not know how to promote this operator's result!"); 2248 2249 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; 2250 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; 2251 case ISD::EXTRACT_VECTOR_ELT: 2252 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; 2253 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; 2254 2255 // Unary FP Operations 2256 case ISD::FABS: 2257 case ISD::FCBRT: 2258 case ISD::FCEIL: 2259 case ISD::FCOS: 2260 case ISD::FEXP: 2261 case ISD::FEXP2: 2262 case ISD::FFLOOR: 2263 case ISD::FLOG: 2264 case ISD::FLOG2: 2265 case ISD::FLOG10: 2266 case ISD::FNEARBYINT: 2267 case ISD::FNEG: 2268 case ISD::FRINT: 2269 case ISD::FROUND: 2270 case ISD::FROUNDEVEN: 2271 case ISD::FSIN: 2272 case ISD::FSQRT: 2273 case ISD::FTRUNC: 2274 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; 2275 2276 // Binary FP Operations 2277 case ISD::FADD: 2278 case ISD::FDIV: 2279 case ISD::FMAXIMUM: 2280 case ISD::FMINIMUM: 2281 case ISD::FMAXNUM: 2282 case ISD::FMINNUM: 2283 case ISD::FMUL: 2284 case ISD::FPOW: 2285 case ISD::FREM: 2286 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; 2287 2288 case ISD::FMA: // FMA is same as FMAD 2289 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; 2290 2291 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break; 2292 2293 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; 2294 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; 2295 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; 2296 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; 2297 2298 case ISD::SINT_TO_FP: 2299 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; 2300 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; 2301 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2302 case ISD::VECREDUCE_FADD: 2303 case ISD::VECREDUCE_FMUL: 2304 case ISD::VECREDUCE_FMIN: 2305 case ISD::VECREDUCE_FMAX: 2306 R = PromoteFloatRes_VECREDUCE(N); 2307 break; 2308 case ISD::VECREDUCE_SEQ_FADD: 2309 case ISD::VECREDUCE_SEQ_FMUL: 2310 R = PromoteFloatRes_VECREDUCE_SEQ(N); 2311 break; 2312 } 2313 2314 if (R.getNode()) 2315 SetPromotedFloat(SDValue(N, ResNo), R); 2316 } 2317 2318 // Bitcast from i16 to f16: convert the i16 to a f32 value instead. 2319 // At this point, it is not possible to determine if the bitcast value is 2320 // eventually stored to memory or promoted to f32 or promoted to a floating 2321 // point at a higher precision. Some of these cases are handled by FP_EXTEND, 2322 // STORE promotion handlers. 2323 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { 2324 EVT VT = N->getValueType(0); 2325 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2326 // Input type isn't guaranteed to be a scalar int so bitcast if not. The 2327 // bitcast will be legalized further if necessary. 2328 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), 2329 N->getOperand(0).getValueType().getSizeInBits()); 2330 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); 2331 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); 2332 } 2333 2334 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { 2335 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); 2336 EVT VT = N->getValueType(0); 2337 SDLoc DL(N); 2338 2339 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2340 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2341 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, 2342 IVT); 2343 2344 // Convert the Constant to the desired FP type 2345 // FIXME We might be able to do the conversion during compilation and get rid 2346 // of it from the object code 2347 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2348 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); 2349 } 2350 2351 // If the Index operand is a constant, try to redirect the extract operation to 2352 // the correct legalized vector. If not, bit-convert the input vector to 2353 // equivalent integer vector. Extract the element as an (bit-cast) integer 2354 // value and convert it to the promoted type. 2355 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2356 SDLoc DL(N); 2357 2358 // If the index is constant, try to extract the value from the legalized 2359 // vector type. 2360 if (isa<ConstantSDNode>(N->getOperand(1))) { 2361 SDValue Vec = N->getOperand(0); 2362 SDValue Idx = N->getOperand(1); 2363 EVT VecVT = Vec->getValueType(0); 2364 EVT EltVT = VecVT.getVectorElementType(); 2365 2366 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2367 2368 switch (getTypeAction(VecVT)) { 2369 default: break; 2370 case TargetLowering::TypeScalarizeVector: { 2371 SDValue Res = GetScalarizedVector(N->getOperand(0)); 2372 ReplaceValueWith(SDValue(N, 0), Res); 2373 return SDValue(); 2374 } 2375 case TargetLowering::TypeWidenVector: { 2376 Vec = GetWidenedVector(Vec); 2377 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); 2378 ReplaceValueWith(SDValue(N, 0), Res); 2379 return SDValue(); 2380 } 2381 case TargetLowering::TypeSplitVector: { 2382 SDValue Lo, Hi; 2383 GetSplitVector(Vec, Lo, Hi); 2384 2385 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2386 SDValue Res; 2387 if (IdxVal < LoElts) 2388 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); 2389 else 2390 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, 2391 DAG.getConstant(IdxVal - LoElts, DL, 2392 Idx.getValueType())); 2393 ReplaceValueWith(SDValue(N, 0), Res); 2394 return SDValue(); 2395 } 2396 2397 } 2398 } 2399 2400 // Bit-convert the input vector to the equivalent integer vector 2401 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2402 EVT IVT = NewOp.getValueType().getVectorElementType(); 2403 2404 // Extract the element as an (bit-cast) integer value 2405 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, 2406 NewOp, N->getOperand(1)); 2407 2408 // Convert the element to the desired FP type 2409 EVT VT = N->getValueType(0); 2410 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2411 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); 2412 } 2413 2414 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result 2415 // needs promotion, so does the argument X. Note that Y, if needed, will be 2416 // handled during operand promotion. 2417 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { 2418 EVT VT = N->getValueType(0); 2419 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2420 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2421 2422 SDValue Op1 = N->getOperand(1); 2423 2424 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2425 } 2426 2427 // Unary operation where the result and the operand have PromoteFloat type 2428 // action. Construct a new SDNode with the promoted float value of the old 2429 // operand. 2430 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { 2431 EVT VT = N->getValueType(0); 2432 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2433 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2434 2435 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); 2436 } 2437 2438 // Binary operations where the result and both operands have PromoteFloat type 2439 // action. Construct a new SDNode with the promoted float values of the old 2440 // operands. 2441 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { 2442 EVT VT = N->getValueType(0); 2443 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2444 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2445 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2446 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); 2447 } 2448 2449 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { 2450 EVT VT = N->getValueType(0); 2451 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2452 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2453 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2454 SDValue Op2 = GetPromotedFloat(N->getOperand(2)); 2455 2456 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); 2457 } 2458 2459 // Promote the Float (first) operand and retain the Integer (second) operand 2460 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) { 2461 EVT VT = N->getValueType(0); 2462 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2463 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2464 SDValue Op1 = N->getOperand(1); 2465 2466 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2467 } 2468 2469 // Explicit operation to reduce precision. Reduce the value to half precision 2470 // and promote it back to the legal type. 2471 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { 2472 SDLoc DL(N); 2473 2474 SDValue Op = N->getOperand(0); 2475 EVT VT = N->getValueType(0); 2476 EVT OpVT = Op->getValueType(0); 2477 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2478 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2479 2480 // Round promoted float to desired precision 2481 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); 2482 // Promote it back to the legal output type 2483 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); 2484 } 2485 2486 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { 2487 LoadSDNode *L = cast<LoadSDNode>(N); 2488 EVT VT = N->getValueType(0); 2489 2490 // Load the value as an integer value with the same number of bits. 2491 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2492 SDValue newL = DAG.getLoad( 2493 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), 2494 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, 2495 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); 2496 // Legalize the chain result by replacing uses of the old value chain with the 2497 // new one 2498 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 2499 2500 // Convert the integer value to the desired FP type 2501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2502 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); 2503 } 2504 2505 // Construct a new SELECT node with the promoted true- and false- values. 2506 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { 2507 SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); 2508 SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); 2509 2510 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), 2511 N->getOperand(0), TrueVal, FalseVal); 2512 } 2513 2514 // Construct a new SELECT_CC node with the promoted true- and false- values. 2515 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. 2516 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { 2517 SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); 2518 SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); 2519 2520 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2521 TrueVal.getNode()->getValueType(0), N->getOperand(0), 2522 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); 2523 } 2524 2525 // Construct a SDNode that transforms the SINT or UINT operand to the promoted 2526 // float type. 2527 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { 2528 SDLoc DL(N); 2529 EVT VT = N->getValueType(0); 2530 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2531 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); 2532 // Round the value to the desired precision (that of the source type). 2533 return DAG.getNode( 2534 ISD::FP_EXTEND, DL, NVT, 2535 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL))); 2536 } 2537 2538 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { 2539 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 2540 N->getValueType(0))); 2541 } 2542 2543 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { 2544 // Expand and promote recursively. 2545 // TODO: This is non-optimal, but dealing with the concurrently happening 2546 // vector-legalization is non-trivial. We could do something similar to 2547 // PromoteFloatRes_EXTRACT_VECTOR_ELT here. 2548 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2549 return SDValue(); 2550 } 2551 2552 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { 2553 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2554 return SDValue(); 2555 } 2556 2557 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { 2558 EVT VT = N->getValueType(0); 2559 2560 AtomicSDNode *AM = cast<AtomicSDNode>(N); 2561 SDLoc SL(N); 2562 2563 SDValue CastVal = BitConvertToInteger(AM->getVal()); 2564 EVT CastVT = CastVal.getValueType(); 2565 2566 SDValue NewAtomic 2567 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, 2568 DAG.getVTList(CastVT, MVT::Other), 2569 { AM->getChain(), AM->getBasePtr(), CastVal }, 2570 AM->getMemOperand()); 2571 2572 SDValue Result = NewAtomic; 2573 2574 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { 2575 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2576 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, 2577 NewAtomic); 2578 } 2579 2580 // Legalize the chain result by replacing uses of the old value chain with the 2581 // new one 2582 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); 2583 2584 return Result; 2585 2586 } 2587 2588 //===----------------------------------------------------------------------===// 2589 // Half Result Soft Promotion 2590 //===----------------------------------------------------------------------===// 2591 2592 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { 2593 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": "; 2594 N->dump(&DAG); dbgs() << "\n"); 2595 SDValue R = SDValue(); 2596 2597 // See if the target wants to custom expand this node. 2598 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2599 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2600 return; 2601 } 2602 2603 switch (N->getOpcode()) { 2604 default: 2605 #ifndef NDEBUG 2606 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": "; 2607 N->dump(&DAG); dbgs() << "\n"; 2608 #endif 2609 llvm_unreachable("Do not know how to soft promote this operator's result!"); 2610 2611 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; 2612 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; 2613 case ISD::EXTRACT_VECTOR_ELT: 2614 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; 2615 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; 2616 case ISD::STRICT_FP_ROUND: 2617 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; 2618 2619 // Unary FP Operations 2620 case ISD::FABS: 2621 case ISD::FCBRT: 2622 case ISD::FCEIL: 2623 case ISD::FCOS: 2624 case ISD::FEXP: 2625 case ISD::FEXP2: 2626 case ISD::FFLOOR: 2627 case ISD::FLOG: 2628 case ISD::FLOG2: 2629 case ISD::FLOG10: 2630 case ISD::FNEARBYINT: 2631 case ISD::FNEG: 2632 case ISD::FREEZE: 2633 case ISD::FRINT: 2634 case ISD::FROUND: 2635 case ISD::FROUNDEVEN: 2636 case ISD::FSIN: 2637 case ISD::FSQRT: 2638 case ISD::FTRUNC: 2639 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; 2640 2641 // Binary FP Operations 2642 case ISD::FADD: 2643 case ISD::FDIV: 2644 case ISD::FMAXIMUM: 2645 case ISD::FMINIMUM: 2646 case ISD::FMAXNUM: 2647 case ISD::FMINNUM: 2648 case ISD::FMUL: 2649 case ISD::FPOW: 2650 case ISD::FREM: 2651 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; 2652 2653 case ISD::FMA: // FMA is same as FMAD 2654 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; 2655 2656 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break; 2657 2658 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; 2659 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; 2660 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; 2661 case ISD::SINT_TO_FP: 2662 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; 2663 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; 2664 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2665 case ISD::VECREDUCE_FADD: 2666 case ISD::VECREDUCE_FMUL: 2667 case ISD::VECREDUCE_FMIN: 2668 case ISD::VECREDUCE_FMAX: 2669 R = SoftPromoteHalfRes_VECREDUCE(N); 2670 break; 2671 case ISD::VECREDUCE_SEQ_FADD: 2672 case ISD::VECREDUCE_SEQ_FMUL: 2673 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N); 2674 break; 2675 } 2676 2677 if (R.getNode()) 2678 SetSoftPromotedHalf(SDValue(N, ResNo), R); 2679 } 2680 2681 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { 2682 return BitConvertToInteger(N->getOperand(0)); 2683 } 2684 2685 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { 2686 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 2687 2688 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2689 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 2690 MVT::i16); 2691 } 2692 2693 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2694 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2695 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2696 NewOp.getValueType().getVectorElementType(), NewOp, 2697 N->getOperand(1)); 2698 } 2699 2700 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { 2701 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0)); 2702 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 2703 SDLoc dl(N); 2704 2705 EVT LVT = LHS.getValueType(); 2706 EVT RVT = RHS.getValueType(); 2707 2708 unsigned LSize = LVT.getSizeInBits(); 2709 unsigned RSize = RVT.getSizeInBits(); 2710 2711 // First get the sign bit of second operand. 2712 SDValue SignBit = DAG.getNode( 2713 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 2714 DAG.getConstant(RSize - 1, dl, 2715 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 2716 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 2717 2718 // Shift right or sign-extend it if the two operands have different types. 2719 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 2720 if (SizeDiff > 0) { 2721 SignBit = 2722 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 2723 DAG.getConstant(SizeDiff, dl, 2724 TLI.getShiftAmountTy(SignBit.getValueType(), 2725 DAG.getDataLayout()))); 2726 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 2727 } else if (SizeDiff < 0) { 2728 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 2729 SignBit = 2730 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 2731 DAG.getConstant(-SizeDiff, dl, 2732 TLI.getShiftAmountTy(SignBit.getValueType(), 2733 DAG.getDataLayout()))); 2734 } 2735 2736 // Clear the sign bit of the first operand. 2737 SDValue Mask = DAG.getNode( 2738 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 2739 DAG.getConstant(LSize - 1, dl, 2740 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 2741 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 2742 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 2743 2744 // Or the value with the sign bit. 2745 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 2746 } 2747 2748 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { 2749 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2750 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2751 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2752 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2753 SDLoc dl(N); 2754 2755 // Promote to the larger FP type. 2756 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2757 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2758 Op2 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op2); 2759 2760 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2); 2761 2762 // Convert back to FP16 as an integer. 2763 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2764 } 2765 2766 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) { 2767 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2768 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2769 SDValue Op1 = N->getOperand(1); 2770 SDLoc dl(N); 2771 2772 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2773 2774 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2775 2776 // Convert back to FP16 as an integer. 2777 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2778 } 2779 2780 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { 2781 if (N->isStrictFPOpcode()) { 2782 SDValue Res = 2783 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other}, 2784 {N->getOperand(0), N->getOperand(1)}); 2785 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2786 return Res; 2787 } 2788 2789 return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), MVT::i16, N->getOperand(0)); 2790 } 2791 2792 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { 2793 LoadSDNode *L = cast<LoadSDNode>(N); 2794 2795 // Load the value as an integer value with the same number of bits. 2796 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!"); 2797 SDValue NewL = 2798 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, 2799 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), 2800 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), 2801 L->getMemOperand()->getFlags(), L->getAAInfo()); 2802 // Legalize the chain result by replacing uses of the old value chain with the 2803 // new one 2804 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 2805 return NewL; 2806 } 2807 2808 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { 2809 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2810 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2811 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1, 2812 Op2); 2813 } 2814 2815 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { 2816 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2817 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3)); 2818 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(), 2819 N->getOperand(0), N->getOperand(1), Op2, Op3, 2820 N->getOperand(4)); 2821 } 2822 2823 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { 2824 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2825 SDLoc dl(N); 2826 2827 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 2828 2829 // Round the value to the softened type. 2830 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2831 } 2832 2833 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { 2834 return DAG.getUNDEF(MVT::i16); 2835 } 2836 2837 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { 2838 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2839 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 2840 SDLoc dl(N); 2841 2842 // Promote to the larger FP type. 2843 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); 2844 2845 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op); 2846 2847 // Convert back to FP16 as an integer. 2848 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2849 } 2850 2851 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { 2852 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2853 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2854 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2855 SDLoc dl(N); 2856 2857 // Promote to the larger FP type. 2858 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2859 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2860 2861 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2862 2863 // Convert back to FP16 as an integer. 2864 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2865 } 2866 2867 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) { 2868 // Expand and soften recursively. 2869 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2870 return SDValue(); 2871 } 2872 2873 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) { 2874 // Expand and soften. 2875 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2876 return SDValue(); 2877 } 2878 2879 //===----------------------------------------------------------------------===// 2880 // Half Operand Soft Promotion 2881 //===----------------------------------------------------------------------===// 2882 2883 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { 2884 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": "; 2885 N->dump(&DAG); dbgs() << "\n"); 2886 SDValue Res = SDValue(); 2887 2888 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2889 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2890 return false; 2891 } 2892 2893 // Nodes that use a promotion-requiring floating point operand, but doesn't 2894 // produce a soft promotion-requiring floating point result, need to be 2895 // legalized to use the soft promoted float operand. Nodes that produce at 2896 // least one soft promotion-requiring floating point result have their 2897 // operands legalized as a part of PromoteFloatResult. 2898 switch (N->getOpcode()) { 2899 default: 2900 #ifndef NDEBUG 2901 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": "; 2902 N->dump(&DAG); dbgs() << "\n"; 2903 #endif 2904 llvm_unreachable("Do not know how to soft promote this operator's operand!"); 2905 2906 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; 2907 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; 2908 case ISD::FP_TO_SINT: 2909 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; 2910 case ISD::FP_TO_SINT_SAT: 2911 case ISD::FP_TO_UINT_SAT: 2912 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; 2913 case ISD::STRICT_FP_EXTEND: 2914 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; 2915 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; 2916 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; 2917 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; 2918 case ISD::STACKMAP: 2919 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); 2920 break; 2921 case ISD::PATCHPOINT: 2922 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo); 2923 break; 2924 } 2925 2926 if (!Res.getNode()) 2927 return false; 2928 2929 assert(Res.getNode() != N && "Expected a new node!"); 2930 2931 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2932 "Invalid operand expansion"); 2933 2934 ReplaceValueWith(SDValue(N, 0), Res); 2935 return false; 2936 } 2937 2938 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { 2939 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2940 2941 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 2942 } 2943 2944 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, 2945 unsigned OpNo) { 2946 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 2947 SDValue Op1 = N->getOperand(1); 2948 SDLoc dl(N); 2949 2950 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType()); 2951 2952 Op1 = GetSoftPromotedHalf(Op1); 2953 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2954 2955 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0), 2956 Op1); 2957 } 2958 2959 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { 2960 bool IsStrict = N->isStrictFPOpcode(); 2961 SDValue Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0)); 2962 2963 if (IsStrict) { 2964 SDValue Res = 2965 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N), 2966 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op}); 2967 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2968 ReplaceValueWith(SDValue(N, 0), Res); 2969 return SDValue(); 2970 } 2971 2972 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), Op); 2973 } 2974 2975 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { 2976 SDValue Op = N->getOperand(0); 2977 SDLoc dl(N); 2978 2979 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 2980 2981 Op = GetSoftPromotedHalf(Op); 2982 2983 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); 2984 2985 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res); 2986 } 2987 2988 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { 2989 SDValue Op = N->getOperand(0); 2990 SDLoc dl(N); 2991 2992 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 2993 2994 Op = GetSoftPromotedHalf(Op); 2995 2996 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); 2997 2998 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res, 2999 N->getOperand(1)); 3000 } 3001 3002 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, 3003 unsigned OpNo) { 3004 assert(OpNo == 0 && "Can only soften the comparison values"); 3005 SDValue Op0 = N->getOperand(0); 3006 SDValue Op1 = N->getOperand(1); 3007 SDLoc dl(N); 3008 3009 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 3010 3011 Op0 = GetSoftPromotedHalf(Op0); 3012 Op1 = GetSoftPromotedHalf(Op1); 3013 3014 // Promote to the larger FP type. 3015 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 3016 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 3017 3018 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1, 3019 N->getOperand(2), N->getOperand(3), N->getOperand(4)); 3020 } 3021 3022 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { 3023 SDValue Op0 = N->getOperand(0); 3024 SDValue Op1 = N->getOperand(1); 3025 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3026 SDLoc dl(N); 3027 3028 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 3029 3030 Op0 = GetSoftPromotedHalf(Op0); 3031 Op1 = GetSoftPromotedHalf(Op1); 3032 3033 // Promote to the larger FP type. 3034 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 3035 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 3036 3037 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode); 3038 } 3039 3040 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { 3041 assert(OpNo == 1 && "Can only soften the stored value!"); 3042 StoreSDNode *ST = cast<StoreSDNode>(N); 3043 SDValue Val = ST->getValue(); 3044 SDLoc dl(N); 3045 3046 assert(!ST->isTruncatingStore() && "Unexpected truncating store."); 3047 SDValue Promoted = GetSoftPromotedHalf(Val); 3048 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), 3049 ST->getMemOperand()); 3050 } 3051 3052 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { 3053 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 3054 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3055 SDValue Op = N->getOperand(OpNo); 3056 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3057 SDValue NewNode = 3058 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3059 3060 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3061 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3062 3063 return SDValue(); // Signal that we replaced the node ourselves. 3064 } 3065 3066 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N, 3067 unsigned OpNo) { 3068 assert(OpNo >= 7); 3069 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3070 SDValue Op = N->getOperand(OpNo); 3071 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3072 SDValue NewNode = 3073 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3074 3075 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3076 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3077 3078 return SDValue(); // Signal that we replaced the node ourselves. 3079 } 3080