1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements float type expansion and softening for LegalizeTypes. 10 // Softening is the act of turning a computation in an illegal floating point 11 // type into a computation in an integer type of the same size; also known as 12 // "soft float". For example, turning f32 arithmetic into operations using i32. 13 // The resulting integer value is the same as what you would get by performing 14 // the floating point operation and bitcasting the result to the integer type. 15 // Expansion is the act of changing a computation in an illegal type to be a 16 // computation in two identical registers of a smaller type. For example, 17 // implementing ppcf128 arithmetic in two f64 registers. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/Support/ErrorHandling.h" 23 #include "llvm/Support/raw_ostream.h" 24 using namespace llvm; 25 26 #define DEBUG_TYPE "legalize-types" 27 28 /// GetFPLibCall - Return the right libcall for the given floating point type. 29 static RTLIB::Libcall GetFPLibCall(EVT VT, 30 RTLIB::Libcall Call_F32, 31 RTLIB::Libcall Call_F64, 32 RTLIB::Libcall Call_F80, 33 RTLIB::Libcall Call_F128, 34 RTLIB::Libcall Call_PPCF128) { 35 return 36 VT == MVT::f32 ? Call_F32 : 37 VT == MVT::f64 ? Call_F64 : 38 VT == MVT::f80 ? Call_F80 : 39 VT == MVT::f128 ? Call_F128 : 40 VT == MVT::ppcf128 ? Call_PPCF128 : 41 RTLIB::UNKNOWN_LIBCALL; 42 } 43 44 //===----------------------------------------------------------------------===// 45 // Convert Float Results to Integer 46 //===----------------------------------------------------------------------===// 47 48 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { 49 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG); 50 dbgs() << "\n"); 51 SDValue R = SDValue(); 52 53 switch (N->getOpcode()) { 54 default: 55 #ifndef NDEBUG 56 dbgs() << "SoftenFloatResult #" << ResNo << ": "; 57 N->dump(&DAG); dbgs() << "\n"; 58 #endif 59 llvm_unreachable("Do not know how to soften the result of this operator!"); 60 61 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; 62 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; 63 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; 64 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; 65 case ISD::EXTRACT_VECTOR_ELT: 66 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; 67 case ISD::FABS: R = SoftenFloatRes_FABS(N); break; 68 case ISD::STRICT_FMINNUM: 69 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; 70 case ISD::STRICT_FMAXNUM: 71 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; 72 case ISD::STRICT_FADD: 73 case ISD::FADD: R = SoftenFloatRes_FADD(N); break; 74 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; 75 case ISD::STRICT_FCEIL: 76 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; 77 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; 78 case ISD::STRICT_FCOS: 79 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; 80 case ISD::STRICT_FDIV: 81 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; 82 case ISD::STRICT_FEXP: 83 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; 84 case ISD::STRICT_FEXP2: 85 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; 86 case ISD::STRICT_FFLOOR: 87 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; 88 case ISD::STRICT_FLOG: 89 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; 90 case ISD::STRICT_FLOG2: 91 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; 92 case ISD::STRICT_FLOG10: 93 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; 94 case ISD::STRICT_FMA: 95 case ISD::FMA: R = SoftenFloatRes_FMA(N); break; 96 case ISD::STRICT_FMUL: 97 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; 98 case ISD::STRICT_FNEARBYINT: 99 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; 100 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; 101 case ISD::STRICT_FP_EXTEND: 102 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; 103 case ISD::STRICT_FP_ROUND: 104 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; 105 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; 106 case ISD::STRICT_FPOW: 107 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; 108 case ISD::STRICT_FPOWI: 109 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break; 110 case ISD::STRICT_FREM: 111 case ISD::FREM: R = SoftenFloatRes_FREM(N); break; 112 case ISD::STRICT_FRINT: 113 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; 114 case ISD::STRICT_FROUND: 115 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; 116 case ISD::STRICT_FROUNDEVEN: 117 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; 118 case ISD::STRICT_FSIN: 119 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; 120 case ISD::STRICT_FSQRT: 121 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; 122 case ISD::STRICT_FSUB: 123 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; 124 case ISD::STRICT_FTRUNC: 125 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; 126 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; 127 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 128 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; 129 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; 130 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; 131 case ISD::STRICT_SINT_TO_FP: 132 case ISD::STRICT_UINT_TO_FP: 133 case ISD::SINT_TO_FP: 134 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; 135 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; 136 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; 137 } 138 139 // If R is null, the sub-method took care of registering the result. 140 if (R.getNode()) { 141 assert(R.getNode() != N); 142 SetSoftenedFloat(SDValue(N, ResNo), R); 143 } 144 } 145 146 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { 147 bool IsStrict = N->isStrictFPOpcode(); 148 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 149 unsigned Offset = IsStrict ? 1 : 0; 150 assert(N->getNumOperands() == (1 + Offset) && 151 "Unexpected number of operands!"); 152 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 153 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 154 TargetLowering::MakeLibCallOptions CallOptions; 155 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 156 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 157 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 158 CallOptions, SDLoc(N), 159 Chain); 160 if (IsStrict) 161 ReplaceValueWith(SDValue(N, 1), Tmp.second); 162 return Tmp.first; 163 } 164 165 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { 166 bool IsStrict = N->isStrictFPOpcode(); 167 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 168 unsigned Offset = IsStrict ? 1 : 0; 169 assert(N->getNumOperands() == (2 + Offset) && 170 "Unexpected number of operands!"); 171 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 172 GetSoftenedFloat(N->getOperand(1 + Offset)) }; 173 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 174 TargetLowering::MakeLibCallOptions CallOptions; 175 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 176 N->getOperand(1 + Offset).getValueType() }; 177 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 178 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 179 CallOptions, SDLoc(N), 180 Chain); 181 if (IsStrict) 182 ReplaceValueWith(SDValue(N, 1), Tmp.second); 183 return Tmp.first; 184 } 185 186 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { 187 return BitConvertToInteger(N->getOperand(0)); 188 } 189 190 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { 191 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 192 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty, 193 GetSoftenedFloat(N->getOperand(0))); 194 } 195 196 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, 197 unsigned ResNo) { 198 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 199 return BitConvertToInteger(Op); 200 } 201 202 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { 203 // Convert the inputs to integers, and build a new pair out of them. 204 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), 205 TLI.getTypeToTransformTo(*DAG.getContext(), 206 N->getValueType(0)), 207 BitConvertToInteger(N->getOperand(0)), 208 BitConvertToInteger(N->getOperand(1))); 209 } 210 211 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { 212 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 213 // In ppcf128, the high 64 bits are always first in memory regardless 214 // of Endianness. LLVM's APFloat representation is not Endian sensitive, 215 // and so always converts into a 128-bit APInt in a non-Endian-sensitive 216 // way. However, APInt's are serialized in an Endian-sensitive fashion, 217 // so on big-Endian targets, the two doubles are output in the wrong 218 // order. Fix this by manually flipping the order of the high 64 bits 219 // and the low 64 bits here. 220 if (DAG.getDataLayout().isBigEndian() && 221 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) { 222 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], 223 CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; 224 APInt Val(128, words); 225 return DAG.getConstant(Val, SDLoc(CN), 226 TLI.getTypeToTransformTo(*DAG.getContext(), 227 CN->getValueType(0))); 228 } else { 229 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 230 TLI.getTypeToTransformTo(*DAG.getContext(), 231 CN->getValueType(0))); 232 } 233 } 234 235 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) { 236 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 237 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 238 NewOp.getValueType().getVectorElementType(), 239 NewOp, N->getOperand(1)); 240 } 241 242 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { 243 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 244 unsigned Size = NVT.getSizeInBits(); 245 246 // Mask = ~(1 << (Size-1)) 247 APInt API = APInt::getAllOnesValue(Size); 248 API.clearBit(Size - 1); 249 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT); 250 SDValue Op = GetSoftenedFloat(N->getOperand(0)); 251 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask); 252 } 253 254 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { 255 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 256 RTLIB::FMIN_F32, 257 RTLIB::FMIN_F64, 258 RTLIB::FMIN_F80, 259 RTLIB::FMIN_F128, 260 RTLIB::FMIN_PPCF128)); 261 } 262 263 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { 264 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 265 RTLIB::FMAX_F32, 266 RTLIB::FMAX_F64, 267 RTLIB::FMAX_F80, 268 RTLIB::FMAX_F128, 269 RTLIB::FMAX_PPCF128)); 270 } 271 272 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { 273 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 274 RTLIB::ADD_F32, 275 RTLIB::ADD_F64, 276 RTLIB::ADD_F80, 277 RTLIB::ADD_F128, 278 RTLIB::ADD_PPCF128)); 279 } 280 281 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { 282 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 283 RTLIB::CBRT_F32, 284 RTLIB::CBRT_F64, 285 RTLIB::CBRT_F80, 286 RTLIB::CBRT_F128, 287 RTLIB::CBRT_PPCF128)); 288 } 289 290 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { 291 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 292 RTLIB::CEIL_F32, 293 RTLIB::CEIL_F64, 294 RTLIB::CEIL_F80, 295 RTLIB::CEIL_F128, 296 RTLIB::CEIL_PPCF128)); 297 } 298 299 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { 300 SDValue LHS = GetSoftenedFloat(N->getOperand(0)); 301 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 302 SDLoc dl(N); 303 304 EVT LVT = LHS.getValueType(); 305 EVT RVT = RHS.getValueType(); 306 307 unsigned LSize = LVT.getSizeInBits(); 308 unsigned RSize = RVT.getSizeInBits(); 309 310 // First get the sign bit of second operand. 311 SDValue SignBit = DAG.getNode( 312 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 313 DAG.getConstant(RSize - 1, dl, 314 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 315 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 316 317 // Shift right or sign-extend it if the two operands have different types. 318 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 319 if (SizeDiff > 0) { 320 SignBit = 321 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 322 DAG.getConstant(SizeDiff, dl, 323 TLI.getShiftAmountTy(SignBit.getValueType(), 324 DAG.getDataLayout()))); 325 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 326 } else if (SizeDiff < 0) { 327 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 328 SignBit = 329 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 330 DAG.getConstant(-SizeDiff, dl, 331 TLI.getShiftAmountTy(SignBit.getValueType(), 332 DAG.getDataLayout()))); 333 } 334 335 // Clear the sign bit of the first operand. 336 SDValue Mask = DAG.getNode( 337 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 338 DAG.getConstant(LSize - 1, dl, 339 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 340 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 341 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 342 343 // Or the value with the sign bit. 344 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 345 } 346 347 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { 348 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 349 RTLIB::COS_F32, 350 RTLIB::COS_F64, 351 RTLIB::COS_F80, 352 RTLIB::COS_F128, 353 RTLIB::COS_PPCF128)); 354 } 355 356 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { 357 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 358 RTLIB::DIV_F32, 359 RTLIB::DIV_F64, 360 RTLIB::DIV_F80, 361 RTLIB::DIV_F128, 362 RTLIB::DIV_PPCF128)); 363 } 364 365 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { 366 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 367 RTLIB::EXP_F32, 368 RTLIB::EXP_F64, 369 RTLIB::EXP_F80, 370 RTLIB::EXP_F128, 371 RTLIB::EXP_PPCF128)); 372 } 373 374 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { 375 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 376 RTLIB::EXP2_F32, 377 RTLIB::EXP2_F64, 378 RTLIB::EXP2_F80, 379 RTLIB::EXP2_F128, 380 RTLIB::EXP2_PPCF128)); 381 } 382 383 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { 384 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 385 RTLIB::FLOOR_F32, 386 RTLIB::FLOOR_F64, 387 RTLIB::FLOOR_F80, 388 RTLIB::FLOOR_F128, 389 RTLIB::FLOOR_PPCF128)); 390 } 391 392 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { 393 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 394 RTLIB::LOG_F32, 395 RTLIB::LOG_F64, 396 RTLIB::LOG_F80, 397 RTLIB::LOG_F128, 398 RTLIB::LOG_PPCF128)); 399 } 400 401 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { 402 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 403 RTLIB::LOG2_F32, 404 RTLIB::LOG2_F64, 405 RTLIB::LOG2_F80, 406 RTLIB::LOG2_F128, 407 RTLIB::LOG2_PPCF128)); 408 } 409 410 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { 411 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 412 RTLIB::LOG10_F32, 413 RTLIB::LOG10_F64, 414 RTLIB::LOG10_F80, 415 RTLIB::LOG10_F128, 416 RTLIB::LOG10_PPCF128)); 417 } 418 419 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { 420 bool IsStrict = N->isStrictFPOpcode(); 421 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 422 unsigned Offset = IsStrict ? 1 : 0; 423 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 424 GetSoftenedFloat(N->getOperand(1 + Offset)), 425 GetSoftenedFloat(N->getOperand(2 + Offset)) }; 426 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 427 TargetLowering::MakeLibCallOptions CallOptions; 428 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(), 429 N->getOperand(1 + Offset).getValueType(), 430 N->getOperand(2 + Offset).getValueType() }; 431 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 432 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, 433 GetFPLibCall(N->getValueType(0), 434 RTLIB::FMA_F32, 435 RTLIB::FMA_F64, 436 RTLIB::FMA_F80, 437 RTLIB::FMA_F128, 438 RTLIB::FMA_PPCF128), 439 NVT, Ops, CallOptions, SDLoc(N), Chain); 440 if (IsStrict) 441 ReplaceValueWith(SDValue(N, 1), Tmp.second); 442 return Tmp.first; 443 } 444 445 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { 446 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 447 RTLIB::MUL_F32, 448 RTLIB::MUL_F64, 449 RTLIB::MUL_F80, 450 RTLIB::MUL_F128, 451 RTLIB::MUL_PPCF128)); 452 } 453 454 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { 455 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 456 RTLIB::NEARBYINT_F32, 457 RTLIB::NEARBYINT_F64, 458 RTLIB::NEARBYINT_F80, 459 RTLIB::NEARBYINT_F128, 460 RTLIB::NEARBYINT_PPCF128)); 461 } 462 463 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { 464 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 465 SDLoc dl(N); 466 467 // Expand Y = FNEG(X) -> Y = X ^ sign mask 468 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); 469 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), 470 DAG.getConstant(SignMask, dl, NVT)); 471 } 472 473 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { 474 bool IsStrict = N->isStrictFPOpcode(); 475 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 476 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 477 478 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 479 480 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { 481 Op = GetPromotedFloat(Op); 482 // If the promotion did the FP_EXTEND to the destination type for us, 483 // there's nothing left to do here. 484 if (Op.getValueType() == N->getValueType(0)) 485 return BitConvertToInteger(Op); 486 } 487 488 // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's 489 // entirely possible for both f16 and f32 to be legal, so use the fully 490 // hard-float FP_EXTEND rather than FP16_TO_FP. 491 if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) { 492 if (IsStrict) { 493 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 494 { MVT::f32, MVT::Other }, { Chain, Op }); 495 Chain = Op.getValue(1); 496 } else { 497 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); 498 } 499 } 500 501 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); 502 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 503 TargetLowering::MakeLibCallOptions CallOptions; 504 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 505 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 506 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 507 CallOptions, SDLoc(N), 508 Chain); 509 if (IsStrict) 510 ReplaceValueWith(SDValue(N, 1), Tmp.second); 511 return Tmp.first; 512 } 513 514 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 515 // nodes? 516 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { 517 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 518 SDValue Op = N->getOperand(0); 519 TargetLowering::MakeLibCallOptions CallOptions; 520 EVT OpsVT[1] = { N->getOperand(0).getValueType() }; 521 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 522 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op, 523 CallOptions, SDLoc(N)).first; 524 if (N->getValueType(0) == MVT::f32) 525 return Res32; 526 527 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 528 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0)); 529 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 530 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first; 531 } 532 533 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { 534 bool IsStrict = N->isStrictFPOpcode(); 535 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 536 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 537 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 538 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); 539 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 540 TargetLowering::MakeLibCallOptions CallOptions; 541 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 542 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 543 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 544 CallOptions, SDLoc(N), 545 Chain); 546 if (IsStrict) 547 ReplaceValueWith(SDValue(N, 1), Tmp.second); 548 return Tmp.first; 549 } 550 551 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { 552 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 553 RTLIB::POW_F32, 554 RTLIB::POW_F64, 555 RTLIB::POW_F80, 556 RTLIB::POW_F128, 557 RTLIB::POW_PPCF128)); 558 } 559 560 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) { 561 bool IsStrict = N->isStrictFPOpcode(); 562 unsigned Offset = IsStrict ? 1 : 0; 563 assert(N->getOperand(1 + Offset).getValueType() == MVT::i32 && 564 "Unsupported power type!"); 565 RTLIB::Libcall LC = GetFPLibCall(N->getValueType(0), 566 RTLIB::POWI_F32, 567 RTLIB::POWI_F64, 568 RTLIB::POWI_F80, 569 RTLIB::POWI_F128, 570 RTLIB::POWI_PPCF128); 571 if (!TLI.getLibcallName(LC)) { 572 // Some targets don't have a powi libcall; use pow instead. 573 // FIXME: Implement this if some target needs it. 574 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow"); 575 return DAG.getUNDEF(N->getValueType(0)); 576 } 577 578 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 579 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 580 N->getOperand(1 + Offset) }; 581 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 582 TargetLowering::MakeLibCallOptions CallOptions; 583 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 584 N->getOperand(1 + Offset).getValueType() }; 585 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 586 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 587 CallOptions, SDLoc(N), 588 Chain); 589 if (IsStrict) 590 ReplaceValueWith(SDValue(N, 1), Tmp.second); 591 return Tmp.first; 592 } 593 594 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { 595 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 596 RTLIB::REM_F32, 597 RTLIB::REM_F64, 598 RTLIB::REM_F80, 599 RTLIB::REM_F128, 600 RTLIB::REM_PPCF128)); 601 } 602 603 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { 604 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 605 RTLIB::RINT_F32, 606 RTLIB::RINT_F64, 607 RTLIB::RINT_F80, 608 RTLIB::RINT_F128, 609 RTLIB::RINT_PPCF128)); 610 } 611 612 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { 613 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 614 RTLIB::ROUND_F32, 615 RTLIB::ROUND_F64, 616 RTLIB::ROUND_F80, 617 RTLIB::ROUND_F128, 618 RTLIB::ROUND_PPCF128)); 619 } 620 621 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { 622 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 623 RTLIB::ROUNDEVEN_F32, 624 RTLIB::ROUNDEVEN_F64, 625 RTLIB::ROUNDEVEN_F80, 626 RTLIB::ROUNDEVEN_F128, 627 RTLIB::ROUNDEVEN_PPCF128)); 628 } 629 630 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { 631 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 632 RTLIB::SIN_F32, 633 RTLIB::SIN_F64, 634 RTLIB::SIN_F80, 635 RTLIB::SIN_F128, 636 RTLIB::SIN_PPCF128)); 637 } 638 639 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { 640 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 641 RTLIB::SQRT_F32, 642 RTLIB::SQRT_F64, 643 RTLIB::SQRT_F80, 644 RTLIB::SQRT_F128, 645 RTLIB::SQRT_PPCF128)); 646 } 647 648 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { 649 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 650 RTLIB::SUB_F32, 651 RTLIB::SUB_F64, 652 RTLIB::SUB_F80, 653 RTLIB::SUB_F128, 654 RTLIB::SUB_PPCF128)); 655 } 656 657 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { 658 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 659 RTLIB::TRUNC_F32, 660 RTLIB::TRUNC_F64, 661 RTLIB::TRUNC_F80, 662 RTLIB::TRUNC_F128, 663 RTLIB::TRUNC_PPCF128)); 664 } 665 666 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { 667 LoadSDNode *L = cast<LoadSDNode>(N); 668 EVT VT = N->getValueType(0); 669 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 670 SDLoc dl(N); 671 672 auto MMOFlags = 673 L->getMemOperand()->getFlags() & 674 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 675 SDValue NewL; 676 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 677 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, 678 L->getChain(), L->getBasePtr(), L->getOffset(), 679 L->getPointerInfo(), NVT, L->getOriginalAlign(), 680 MMOFlags, L->getAAInfo()); 681 // Legalized the chain result - switch anything that used the old chain to 682 // use the new one. 683 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 684 return NewL; 685 } 686 687 // Do a non-extending load followed by FP_EXTEND. 688 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(), 689 dl, L->getChain(), L->getBasePtr(), L->getOffset(), 690 L->getPointerInfo(), L->getMemoryVT(), 691 L->getOriginalAlign(), MMOFlags, L->getAAInfo()); 692 // Legalized the chain result - switch anything that used the old chain to 693 // use the new one. 694 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 695 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL); 696 return BitConvertToInteger(ExtendNode); 697 } 698 699 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { 700 SDValue LHS = GetSoftenedFloat(N->getOperand(1)); 701 SDValue RHS = GetSoftenedFloat(N->getOperand(2)); 702 return DAG.getSelect(SDLoc(N), 703 LHS.getValueType(), N->getOperand(0), LHS, RHS); 704 } 705 706 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { 707 SDValue LHS = GetSoftenedFloat(N->getOperand(2)); 708 SDValue RHS = GetSoftenedFloat(N->getOperand(3)); 709 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 710 LHS.getValueType(), N->getOperand(0), 711 N->getOperand(1), LHS, RHS, N->getOperand(4)); 712 } 713 714 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { 715 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 716 N->getValueType(0))); 717 } 718 719 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { 720 SDValue Chain = N->getOperand(0); // Get the chain. 721 SDValue Ptr = N->getOperand(1); // Get the pointer. 722 EVT VT = N->getValueType(0); 723 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 724 SDLoc dl(N); 725 726 SDValue NewVAARG; 727 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), 728 N->getConstantOperandVal(3)); 729 730 // Legalized the chain result - switch anything that used the old chain to 731 // use the new one. 732 if (N != NewVAARG.getValue(1).getNode()) 733 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1)); 734 return NewVAARG; 735 } 736 737 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { 738 bool IsStrict = N->isStrictFPOpcode(); 739 bool Signed = N->getOpcode() == ISD::SINT_TO_FP || 740 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 741 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 742 EVT RVT = N->getValueType(0); 743 EVT NVT = EVT(); 744 SDLoc dl(N); 745 746 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to 747 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly 748 // match. Look for an appropriate libcall. 749 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 750 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 751 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { 752 NVT = (MVT::SimpleValueType)t; 753 // The source needs to big enough to hold the operand. 754 if (NVT.bitsGE(SVT)) 755 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT); 756 } 757 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 758 759 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 760 // Sign/zero extend the argument if the libcall takes a larger type. 761 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 762 NVT, N->getOperand(IsStrict ? 1 : 0)); 763 TargetLowering::MakeLibCallOptions CallOptions; 764 CallOptions.setSExt(Signed); 765 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 766 std::pair<SDValue, SDValue> Tmp = 767 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), 768 Op, CallOptions, dl, Chain); 769 770 if (IsStrict) 771 ReplaceValueWith(SDValue(N, 1), Tmp.second); 772 return Tmp.first; 773 } 774 775 776 //===----------------------------------------------------------------------===// 777 // Convert Float Operand to Integer 778 //===----------------------------------------------------------------------===// 779 780 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { 781 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG); 782 dbgs() << "\n"); 783 SDValue Res = SDValue(); 784 785 switch (N->getOpcode()) { 786 default: 787 #ifndef NDEBUG 788 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": "; 789 N->dump(&DAG); dbgs() << "\n"; 790 #endif 791 llvm_unreachable("Do not know how to soften this operator's operand!"); 792 793 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; 794 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; 795 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes 796 case ISD::STRICT_FP_ROUND: 797 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; 798 case ISD::STRICT_FP_TO_SINT: 799 case ISD::STRICT_FP_TO_UINT: 800 case ISD::FP_TO_SINT: 801 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; 802 case ISD::STRICT_LROUND: 803 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; 804 case ISD::STRICT_LLROUND: 805 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; 806 case ISD::STRICT_LRINT: 807 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; 808 case ISD::STRICT_LLRINT: 809 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; 810 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; 811 case ISD::STRICT_FSETCC: 812 case ISD::STRICT_FSETCCS: 813 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; 814 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; 815 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; 816 } 817 818 // If the result is null, the sub-method took care of registering results etc. 819 if (!Res.getNode()) return false; 820 821 // If the result is N, the sub-method updated N in place. Tell the legalizer 822 // core about this to re-analyze. 823 if (Res.getNode() == N) 824 return true; 825 826 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 827 "Invalid operand softening"); 828 829 ReplaceValueWith(SDValue(N, 0), Res); 830 return false; 831 } 832 833 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { 834 SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); 835 836 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 837 } 838 839 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { 840 // We actually deal with the partially-softened FP_TO_FP16 node too, which 841 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. 842 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || 843 N->getOpcode() == ISD::STRICT_FP_ROUND); 844 845 bool IsStrict = N->isStrictFPOpcode(); 846 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 847 EVT SVT = Op.getValueType(); 848 EVT RVT = N->getValueType(0); 849 EVT FloatRVT = N->getOpcode() == ISD::FP_TO_FP16 ? MVT::f16 : RVT; 850 851 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT); 852 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 853 854 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 855 Op = GetSoftenedFloat(Op); 856 TargetLowering::MakeLibCallOptions CallOptions; 857 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 858 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 859 CallOptions, SDLoc(N), 860 Chain); 861 if (IsStrict) { 862 ReplaceValueWith(SDValue(N, 1), Tmp.second); 863 ReplaceValueWith(SDValue(N, 0), Tmp.first); 864 return SDValue(); 865 } 866 return Tmp.first; 867 } 868 869 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 870 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 871 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 872 873 EVT VT = NewLHS.getValueType(); 874 NewLHS = GetSoftenedFloat(NewLHS); 875 NewRHS = GetSoftenedFloat(NewRHS); 876 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 877 N->getOperand(2), N->getOperand(3)); 878 879 // If softenSetCCOperands returned a scalar, we need to compare the result 880 // against zero to select between true and false values. 881 if (!NewRHS.getNode()) { 882 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 883 CCCode = ISD::SETNE; 884 } 885 886 // Update N to have the operands specified. 887 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 888 DAG.getCondCode(CCCode), NewLHS, NewRHS, 889 N->getOperand(4)), 890 0); 891 } 892 893 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { 894 bool IsStrict = N->isStrictFPOpcode(); 895 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 896 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 897 898 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 899 EVT SVT = Op.getValueType(); 900 EVT RVT = N->getValueType(0); 901 EVT NVT = EVT(); 902 SDLoc dl(N); 903 904 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to 905 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly 906 // match, eg. we don't have fp -> i8 conversions. 907 // Look for an appropriate libcall. 908 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 909 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 910 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 911 ++IntVT) { 912 NVT = (MVT::SimpleValueType)IntVT; 913 // The type needs to big enough to hold the result. 914 if (NVT.bitsGE(RVT)) 915 LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT) : RTLIB::getFPTOUINT(SVT, NVT); 916 } 917 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_XINT!"); 918 919 Op = GetSoftenedFloat(Op); 920 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 921 TargetLowering::MakeLibCallOptions CallOptions; 922 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 923 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 924 CallOptions, dl, Chain); 925 926 // Truncate the result if the libcall returns a larger type. 927 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first); 928 929 if (!IsStrict) 930 return Res; 931 932 ReplaceValueWith(SDValue(N, 1), Tmp.second); 933 ReplaceValueWith(SDValue(N, 0), Res); 934 return SDValue(); 935 } 936 937 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { 938 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 939 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 940 941 EVT VT = NewLHS.getValueType(); 942 NewLHS = GetSoftenedFloat(NewLHS); 943 NewRHS = GetSoftenedFloat(NewRHS); 944 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 945 N->getOperand(0), N->getOperand(1)); 946 947 // If softenSetCCOperands returned a scalar, we need to compare the result 948 // against zero to select between true and false values. 949 if (!NewRHS.getNode()) { 950 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 951 CCCode = ISD::SETNE; 952 } 953 954 // Update N to have the operands specified. 955 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 956 N->getOperand(2), N->getOperand(3), 957 DAG.getCondCode(CCCode)), 958 0); 959 } 960 961 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { 962 bool IsStrict = N->isStrictFPOpcode(); 963 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); 964 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1); 965 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 966 ISD::CondCode CCCode = 967 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 968 969 EVT VT = Op0.getValueType(); 970 SDValue NewLHS = GetSoftenedFloat(Op0); 971 SDValue NewRHS = GetSoftenedFloat(Op1); 972 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1, 973 Chain, N->getOpcode() == ISD::STRICT_FSETCCS); 974 975 // Update N to have the operands specified. 976 if (NewRHS.getNode()) { 977 if (IsStrict) 978 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS, 979 NewRHS, DAG.getCondCode(CCCode)); 980 else 981 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 982 DAG.getCondCode(CCCode)), 0); 983 } 984 985 // Otherwise, softenSetCCOperands returned a scalar, use it. 986 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && 987 "Unexpected setcc expansion!"); 988 989 if (IsStrict) { 990 ReplaceValueWith(SDValue(N, 0), NewLHS); 991 ReplaceValueWith(SDValue(N, 1), Chain); 992 return SDValue(); 993 } 994 return NewLHS; 995 } 996 997 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { 998 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 999 assert(OpNo == 1 && "Can only soften the stored value!"); 1000 StoreSDNode *ST = cast<StoreSDNode>(N); 1001 SDValue Val = ST->getValue(); 1002 SDLoc dl(N); 1003 1004 if (ST->isTruncatingStore()) 1005 // Do an FP_ROUND followed by a non-truncating store. 1006 Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), 1007 Val, DAG.getIntPtrConstant(0, dl))); 1008 else 1009 Val = GetSoftenedFloat(Val); 1010 1011 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(), 1012 ST->getMemOperand()); 1013 } 1014 1015 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { 1016 SDValue LHS = N->getOperand(0); 1017 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 1018 SDLoc dl(N); 1019 1020 EVT LVT = LHS.getValueType(); 1021 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits()); 1022 EVT RVT = RHS.getValueType(); 1023 1024 unsigned LSize = LVT.getSizeInBits(); 1025 unsigned RSize = RVT.getSizeInBits(); 1026 1027 // Shift right or sign-extend it if the two operands have different types. 1028 int SizeDiff = RSize - LSize; 1029 if (SizeDiff > 0) { 1030 RHS = 1031 DAG.getNode(ISD::SRL, dl, RVT, RHS, 1032 DAG.getConstant(SizeDiff, dl, 1033 TLI.getShiftAmountTy(RHS.getValueType(), 1034 DAG.getDataLayout()))); 1035 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS); 1036 } else if (SizeDiff < 0) { 1037 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS); 1038 RHS = 1039 DAG.getNode(ISD::SHL, dl, ILVT, RHS, 1040 DAG.getConstant(-SizeDiff, dl, 1041 TLI.getShiftAmountTy(RHS.getValueType(), 1042 DAG.getDataLayout()))); 1043 } 1044 1045 RHS = DAG.getBitcast(LVT, RHS); 1046 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS); 1047 } 1048 1049 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { 1050 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1051 bool IsStrict = N->isStrictFPOpcode(); 1052 unsigned Offset = IsStrict ? 1 : 0; 1053 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 1054 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1055 TargetLowering::MakeLibCallOptions CallOptions; 1056 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 1057 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 1058 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1059 CallOptions, SDLoc(N), 1060 Chain); 1061 if (IsStrict) { 1062 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1063 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1064 return SDValue(); 1065 } 1066 1067 return Tmp.first; 1068 } 1069 1070 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { 1071 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1072 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1073 RTLIB::LROUND_F32, 1074 RTLIB::LROUND_F64, 1075 RTLIB::LROUND_F80, 1076 RTLIB::LROUND_F128, 1077 RTLIB::LROUND_PPCF128)); 1078 } 1079 1080 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { 1081 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1082 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1083 RTLIB::LLROUND_F32, 1084 RTLIB::LLROUND_F64, 1085 RTLIB::LLROUND_F80, 1086 RTLIB::LLROUND_F128, 1087 RTLIB::LLROUND_PPCF128)); 1088 } 1089 1090 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { 1091 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1092 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1093 RTLIB::LRINT_F32, 1094 RTLIB::LRINT_F64, 1095 RTLIB::LRINT_F80, 1096 RTLIB::LRINT_F128, 1097 RTLIB::LRINT_PPCF128)); 1098 } 1099 1100 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { 1101 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1102 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1103 RTLIB::LLRINT_F32, 1104 RTLIB::LLRINT_F64, 1105 RTLIB::LLRINT_F80, 1106 RTLIB::LLRINT_F128, 1107 RTLIB::LLRINT_PPCF128)); 1108 } 1109 1110 //===----------------------------------------------------------------------===// 1111 // Float Result Expansion 1112 //===----------------------------------------------------------------------===// 1113 1114 /// ExpandFloatResult - This method is called when the specified result of the 1115 /// specified node is found to need expansion. At this point, the node may also 1116 /// have invalid operands or may have other results that need promotion, we just 1117 /// know that (at least) one result needs expansion. 1118 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { 1119 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n"); 1120 SDValue Lo, Hi; 1121 Lo = Hi = SDValue(); 1122 1123 // See if the target wants to custom expand this node. 1124 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1125 return; 1126 1127 switch (N->getOpcode()) { 1128 default: 1129 #ifndef NDEBUG 1130 dbgs() << "ExpandFloatResult #" << ResNo << ": "; 1131 N->dump(&DAG); dbgs() << "\n"; 1132 #endif 1133 llvm_unreachable("Do not know how to expand the result of this operator!"); 1134 1135 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1136 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1137 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1138 1139 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1140 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1141 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1142 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1143 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1144 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1145 1146 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; 1147 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; 1148 case ISD::STRICT_FMINNUM: 1149 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; 1150 case ISD::STRICT_FMAXNUM: 1151 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; 1152 case ISD::STRICT_FADD: 1153 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; 1154 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; 1155 case ISD::STRICT_FCEIL: 1156 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; 1157 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; 1158 case ISD::STRICT_FCOS: 1159 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; 1160 case ISD::STRICT_FDIV: 1161 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; 1162 case ISD::STRICT_FEXP: 1163 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; 1164 case ISD::STRICT_FEXP2: 1165 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; 1166 case ISD::STRICT_FFLOOR: 1167 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; 1168 case ISD::STRICT_FLOG: 1169 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; 1170 case ISD::STRICT_FLOG2: 1171 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; 1172 case ISD::STRICT_FLOG10: 1173 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; 1174 case ISD::STRICT_FMA: 1175 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; 1176 case ISD::STRICT_FMUL: 1177 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; 1178 case ISD::STRICT_FNEARBYINT: 1179 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; 1180 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; 1181 case ISD::STRICT_FP_EXTEND: 1182 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; 1183 case ISD::STRICT_FPOW: 1184 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; 1185 case ISD::STRICT_FPOWI: 1186 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; 1187 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; 1188 case ISD::STRICT_FRINT: 1189 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; 1190 case ISD::STRICT_FROUND: 1191 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; 1192 case ISD::STRICT_FROUNDEVEN: 1193 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; 1194 case ISD::STRICT_FSIN: 1195 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; 1196 case ISD::STRICT_FSQRT: 1197 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; 1198 case ISD::STRICT_FSUB: 1199 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; 1200 case ISD::STRICT_FTRUNC: 1201 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; 1202 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; 1203 case ISD::SINT_TO_FP: 1204 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; 1205 case ISD::STRICT_FREM: 1206 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; 1207 } 1208 1209 // If Lo/Hi is null, the sub-method took care of registering results etc. 1210 if (Lo.getNode()) 1211 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi); 1212 } 1213 1214 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, 1215 SDValue &Hi) { 1216 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1217 assert(NVT.getSizeInBits() == 64 && 1218 "Do not know how to expand this float constant!"); 1219 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); 1220 SDLoc dl(N); 1221 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1222 APInt(64, C.getRawData()[1])), 1223 dl, NVT); 1224 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1225 APInt(64, C.getRawData()[0])), 1226 dl, NVT); 1227 } 1228 1229 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, 1230 SDValue &Lo, SDValue &Hi) { 1231 bool IsStrict = N->isStrictFPOpcode(); 1232 unsigned Offset = IsStrict ? 1 : 0; 1233 SDValue Op = N->getOperand(0 + Offset); 1234 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1235 TargetLowering::MakeLibCallOptions CallOptions; 1236 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1237 Op, CallOptions, SDLoc(N), 1238 Chain); 1239 if (IsStrict) 1240 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1241 GetPairElements(Tmp.first, Lo, Hi); 1242 } 1243 1244 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, 1245 SDValue &Lo, SDValue &Hi) { 1246 bool IsStrict = N->isStrictFPOpcode(); 1247 unsigned Offset = IsStrict ? 1 : 0; 1248 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) }; 1249 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1250 TargetLowering::MakeLibCallOptions CallOptions; 1251 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1252 Ops, CallOptions, SDLoc(N), 1253 Chain); 1254 if (IsStrict) 1255 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1256 GetPairElements(Tmp.first, Lo, Hi); 1257 } 1258 1259 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, 1260 SDValue &Hi) { 1261 assert(N->getValueType(0) == MVT::ppcf128 && 1262 "Logic only correct for ppcf128!"); 1263 SDLoc dl(N); 1264 SDValue Tmp; 1265 GetExpandedFloat(N->getOperand(0), Lo, Tmp); 1266 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp); 1267 // Lo = Hi==fabs(Hi) ? Lo : -Lo; 1268 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo, 1269 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo), 1270 ISD::SETEQ); 1271 } 1272 1273 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, 1274 SDValue &Hi) { 1275 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1276 RTLIB::FMIN_F32, RTLIB::FMIN_F64, 1277 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 1278 RTLIB::FMIN_PPCF128), Lo, Hi); 1279 } 1280 1281 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, 1282 SDValue &Hi) { 1283 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1284 RTLIB::FMAX_F32, RTLIB::FMAX_F64, 1285 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 1286 RTLIB::FMAX_PPCF128), Lo, Hi); 1287 } 1288 1289 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, 1290 SDValue &Hi) { 1291 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1292 RTLIB::ADD_F32, RTLIB::ADD_F64, 1293 RTLIB::ADD_F80, RTLIB::ADD_F128, 1294 RTLIB::ADD_PPCF128), Lo, Hi); 1295 } 1296 1297 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, 1298 SDValue &Hi) { 1299 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32, 1300 RTLIB::CBRT_F64, RTLIB::CBRT_F80, 1301 RTLIB::CBRT_F128, 1302 RTLIB::CBRT_PPCF128), Lo, Hi); 1303 } 1304 1305 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, 1306 SDValue &Lo, SDValue &Hi) { 1307 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1308 RTLIB::CEIL_F32, RTLIB::CEIL_F64, 1309 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 1310 RTLIB::CEIL_PPCF128), Lo, Hi); 1311 } 1312 1313 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, 1314 SDValue &Lo, SDValue &Hi) { 1315 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1316 RTLIB::COPYSIGN_F32, 1317 RTLIB::COPYSIGN_F64, 1318 RTLIB::COPYSIGN_F80, 1319 RTLIB::COPYSIGN_F128, 1320 RTLIB::COPYSIGN_PPCF128), Lo, Hi); 1321 } 1322 1323 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, 1324 SDValue &Lo, SDValue &Hi) { 1325 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1326 RTLIB::COS_F32, RTLIB::COS_F64, 1327 RTLIB::COS_F80, RTLIB::COS_F128, 1328 RTLIB::COS_PPCF128), Lo, Hi); 1329 } 1330 1331 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, 1332 SDValue &Hi) { 1333 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1334 RTLIB::DIV_F32, 1335 RTLIB::DIV_F64, 1336 RTLIB::DIV_F80, 1337 RTLIB::DIV_F128, 1338 RTLIB::DIV_PPCF128), Lo, Hi); 1339 } 1340 1341 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, 1342 SDValue &Lo, SDValue &Hi) { 1343 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1344 RTLIB::EXP_F32, RTLIB::EXP_F64, 1345 RTLIB::EXP_F80, RTLIB::EXP_F128, 1346 RTLIB::EXP_PPCF128), Lo, Hi); 1347 } 1348 1349 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, 1350 SDValue &Lo, SDValue &Hi) { 1351 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1352 RTLIB::EXP2_F32, RTLIB::EXP2_F64, 1353 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 1354 RTLIB::EXP2_PPCF128), Lo, Hi); 1355 } 1356 1357 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, 1358 SDValue &Lo, SDValue &Hi) { 1359 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1360 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 1361 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 1362 RTLIB::FLOOR_PPCF128), Lo, Hi); 1363 } 1364 1365 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, 1366 SDValue &Lo, SDValue &Hi) { 1367 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1368 RTLIB::LOG_F32, RTLIB::LOG_F64, 1369 RTLIB::LOG_F80, RTLIB::LOG_F128, 1370 RTLIB::LOG_PPCF128), Lo, Hi); 1371 } 1372 1373 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, 1374 SDValue &Lo, SDValue &Hi) { 1375 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1376 RTLIB::LOG2_F32, RTLIB::LOG2_F64, 1377 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 1378 RTLIB::LOG2_PPCF128), Lo, Hi); 1379 } 1380 1381 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, 1382 SDValue &Lo, SDValue &Hi) { 1383 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1384 RTLIB::LOG10_F32, RTLIB::LOG10_F64, 1385 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 1386 RTLIB::LOG10_PPCF128), Lo, Hi); 1387 } 1388 1389 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, 1390 SDValue &Hi) { 1391 bool IsStrict = N->isStrictFPOpcode(); 1392 unsigned Offset = IsStrict ? 1 : 0; 1393 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset), 1394 N->getOperand(2 + Offset) }; 1395 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1396 TargetLowering::MakeLibCallOptions CallOptions; 1397 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), 1398 RTLIB::FMA_F32, 1399 RTLIB::FMA_F64, 1400 RTLIB::FMA_F80, 1401 RTLIB::FMA_F128, 1402 RTLIB::FMA_PPCF128), 1403 N->getValueType(0), Ops, CallOptions, 1404 SDLoc(N), Chain); 1405 if (IsStrict) 1406 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1407 GetPairElements(Tmp.first, Lo, Hi); 1408 } 1409 1410 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, 1411 SDValue &Hi) { 1412 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1413 RTLIB::MUL_F32, 1414 RTLIB::MUL_F64, 1415 RTLIB::MUL_F80, 1416 RTLIB::MUL_F128, 1417 RTLIB::MUL_PPCF128), Lo, Hi); 1418 } 1419 1420 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, 1421 SDValue &Lo, SDValue &Hi) { 1422 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1423 RTLIB::NEARBYINT_F32, 1424 RTLIB::NEARBYINT_F64, 1425 RTLIB::NEARBYINT_F80, 1426 RTLIB::NEARBYINT_F128, 1427 RTLIB::NEARBYINT_PPCF128), Lo, Hi); 1428 } 1429 1430 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, 1431 SDValue &Hi) { 1432 SDLoc dl(N); 1433 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1434 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo); 1435 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); 1436 } 1437 1438 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, 1439 SDValue &Hi) { 1440 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1441 SDLoc dl(N); 1442 bool IsStrict = N->isStrictFPOpcode(); 1443 1444 SDValue Chain; 1445 if (IsStrict) { 1446 // If the expanded type is the same as the input type, just bypass the node. 1447 if (NVT == N->getOperand(1).getValueType()) { 1448 Hi = N->getOperand(1); 1449 Chain = N->getOperand(0); 1450 } else { 1451 // Other we need to extend. 1452 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, 1453 { N->getOperand(0), N->getOperand(1) }); 1454 Chain = Hi.getValue(1); 1455 } 1456 } else { 1457 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0)); 1458 } 1459 1460 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1461 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1462 1463 if (IsStrict) 1464 ReplaceValueWith(SDValue(N, 1), Chain); 1465 } 1466 1467 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, 1468 SDValue &Lo, SDValue &Hi) { 1469 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1470 RTLIB::POW_F32, RTLIB::POW_F64, 1471 RTLIB::POW_F80, RTLIB::POW_F128, 1472 RTLIB::POW_PPCF128), Lo, Hi); 1473 } 1474 1475 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, 1476 SDValue &Lo, SDValue &Hi) { 1477 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1478 RTLIB::POWI_F32, RTLIB::POWI_F64, 1479 RTLIB::POWI_F80, RTLIB::POWI_F128, 1480 RTLIB::POWI_PPCF128), Lo, Hi); 1481 } 1482 1483 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, 1484 SDValue &Lo, SDValue &Hi) { 1485 assert(N->getValueType(0) == MVT::ppcf128 && 1486 "Logic only correct for ppcf128!"); 1487 1488 SDLoc dl(N); 1489 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1490 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo); 1491 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi); 1492 } 1493 1494 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, 1495 SDValue &Lo, SDValue &Hi) { 1496 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1497 RTLIB::REM_F32, RTLIB::REM_F64, 1498 RTLIB::REM_F80, RTLIB::REM_F128, 1499 RTLIB::REM_PPCF128), Lo, Hi); 1500 } 1501 1502 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, 1503 SDValue &Lo, SDValue &Hi) { 1504 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1505 RTLIB::RINT_F32, RTLIB::RINT_F64, 1506 RTLIB::RINT_F80, RTLIB::RINT_F128, 1507 RTLIB::RINT_PPCF128), Lo, Hi); 1508 } 1509 1510 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, 1511 SDValue &Lo, SDValue &Hi) { 1512 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1513 RTLIB::ROUND_F32, 1514 RTLIB::ROUND_F64, 1515 RTLIB::ROUND_F80, 1516 RTLIB::ROUND_F128, 1517 RTLIB::ROUND_PPCF128), Lo, Hi); 1518 } 1519 1520 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, 1521 SDValue &Lo, SDValue &Hi) { 1522 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1523 RTLIB::ROUNDEVEN_F32, 1524 RTLIB::ROUNDEVEN_F64, 1525 RTLIB::ROUNDEVEN_F80, 1526 RTLIB::ROUNDEVEN_F128, 1527 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); 1528 } 1529 1530 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, 1531 SDValue &Lo, SDValue &Hi) { 1532 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1533 RTLIB::SIN_F32, RTLIB::SIN_F64, 1534 RTLIB::SIN_F80, RTLIB::SIN_F128, 1535 RTLIB::SIN_PPCF128), Lo, Hi); 1536 } 1537 1538 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, 1539 SDValue &Lo, SDValue &Hi) { 1540 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1541 RTLIB::SQRT_F32, RTLIB::SQRT_F64, 1542 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 1543 RTLIB::SQRT_PPCF128), Lo, Hi); 1544 } 1545 1546 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, 1547 SDValue &Hi) { 1548 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1549 RTLIB::SUB_F32, 1550 RTLIB::SUB_F64, 1551 RTLIB::SUB_F80, 1552 RTLIB::SUB_F128, 1553 RTLIB::SUB_PPCF128), Lo, Hi); 1554 } 1555 1556 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, 1557 SDValue &Lo, SDValue &Hi) { 1558 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1559 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 1560 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 1561 RTLIB::TRUNC_PPCF128), Lo, Hi); 1562 } 1563 1564 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, 1565 SDValue &Hi) { 1566 if (ISD::isNormalLoad(N)) { 1567 ExpandRes_NormalLoad(N, Lo, Hi); 1568 return; 1569 } 1570 1571 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1572 LoadSDNode *LD = cast<LoadSDNode>(N); 1573 SDValue Chain = LD->getChain(); 1574 SDValue Ptr = LD->getBasePtr(); 1575 SDLoc dl(N); 1576 1577 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1578 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1579 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1580 1581 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr, 1582 LD->getMemoryVT(), LD->getMemOperand()); 1583 1584 // Remember the chain. 1585 Chain = Hi.getValue(1); 1586 1587 // The low part is zero. 1588 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1589 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1590 1591 // Modified the chain - switch anything that used the old chain to use the 1592 // new one. 1593 ReplaceValueWith(SDValue(LD, 1), Chain); 1594 } 1595 1596 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, 1597 SDValue &Hi) { 1598 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!"); 1599 EVT VT = N->getValueType(0); 1600 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1601 SDValue Src = N->getOperand(0); 1602 EVT SrcVT = Src.getValueType(); 1603 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP; 1604 SDLoc dl(N); 1605 1606 // First do an SINT_TO_FP, whether the original was signed or unsigned. 1607 // When promoting partial word types to i32 we must honor the signedness, 1608 // though. 1609 if (SrcVT.bitsLE(MVT::i32)) { 1610 // The integer can be represented exactly in an f64. 1611 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1612 MVT::i32, Src); 1613 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1614 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1615 Hi = DAG.getNode(ISD::SINT_TO_FP, dl, NVT, Src); 1616 } else { 1617 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1618 if (SrcVT.bitsLE(MVT::i64)) { 1619 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1620 MVT::i64, Src); 1621 LC = RTLIB::SINTTOFP_I64_PPCF128; 1622 } else if (SrcVT.bitsLE(MVT::i128)) { 1623 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); 1624 LC = RTLIB::SINTTOFP_I128_PPCF128; 1625 } 1626 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 1627 1628 TargetLowering::MakeLibCallOptions CallOptions; 1629 CallOptions.setSExt(true); 1630 Hi = TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl).first; 1631 GetPairElements(Hi, Lo, Hi); 1632 } 1633 1634 if (isSigned) 1635 return; 1636 1637 // Unsigned - fix up the SINT_TO_FP value just calculated. 1638 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 1639 SrcVT = Src.getValueType(); 1640 1641 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. 1642 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 1643 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 1644 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; 1645 ArrayRef<uint64_t> Parts; 1646 1647 switch (SrcVT.getSimpleVT().SimpleTy) { 1648 default: 1649 llvm_unreachable("Unsupported UINT_TO_FP!"); 1650 case MVT::i32: 1651 Parts = TwoE32; 1652 break; 1653 case MVT::i64: 1654 Parts = TwoE64; 1655 break; 1656 case MVT::i128: 1657 Parts = TwoE128; 1658 break; 1659 } 1660 1661 // TODO: Are there fast-math-flags to propagate to this FADD? 1662 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, 1663 DAG.getConstantFP(APFloat(APFloat::PPCDoubleDouble(), 1664 APInt(128, Parts)), 1665 dl, MVT::ppcf128)); 1666 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT), 1667 Lo, Hi, ISD::SETLT); 1668 GetPairElements(Lo, Lo, Hi); 1669 } 1670 1671 1672 //===----------------------------------------------------------------------===// 1673 // Float Operand Expansion 1674 //===----------------------------------------------------------------------===// 1675 1676 /// ExpandFloatOperand - This method is called when the specified operand of the 1677 /// specified node is found to need expansion. At this point, all of the result 1678 /// types of the node are known to be legal, but other operands of the node may 1679 /// need promotion or expansion as well as the specified one. 1680 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { 1681 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n"); 1682 SDValue Res = SDValue(); 1683 1684 // See if the target wants to custom expand this node. 1685 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1686 return false; 1687 1688 switch (N->getOpcode()) { 1689 default: 1690 #ifndef NDEBUG 1691 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": "; 1692 N->dump(&DAG); dbgs() << "\n"; 1693 #endif 1694 llvm_unreachable("Do not know how to expand this operator's operand!"); 1695 1696 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 1697 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1698 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1699 1700 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; 1701 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; 1702 case ISD::STRICT_FP_ROUND: 1703 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; 1704 case ISD::STRICT_FP_TO_SINT: 1705 case ISD::FP_TO_SINT: Res = ExpandFloatOp_FP_TO_SINT(N); break; 1706 case ISD::STRICT_FP_TO_UINT: 1707 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_UINT(N); break; 1708 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; 1709 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; 1710 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; 1711 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; 1712 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; 1713 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; 1714 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N), 1715 OpNo); break; 1716 } 1717 1718 // If the result is null, the sub-method took care of registering results etc. 1719 if (!Res.getNode()) return false; 1720 1721 // If the result is N, the sub-method updated N in place. Tell the legalizer 1722 // core about this. 1723 if (Res.getNode() == N) 1724 return true; 1725 1726 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1727 "Invalid operand expansion"); 1728 1729 ReplaceValueWith(SDValue(N, 0), Res); 1730 return false; 1731 } 1732 1733 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code 1734 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1735 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, 1736 SDValue &NewRHS, 1737 ISD::CondCode &CCCode, 1738 const SDLoc &dl) { 1739 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 1740 GetExpandedFloat(NewLHS, LHSLo, LHSHi); 1741 GetExpandedFloat(NewRHS, RHSLo, RHSHi); 1742 1743 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); 1744 1745 // FIXME: This generated code sucks. We want to generate 1746 // FCMPU crN, hi1, hi2 1747 // BNE crN, L: 1748 // FCMPU crN, lo1, lo2 1749 // The following can be improved, but not that much. 1750 SDValue Tmp1, Tmp2, Tmp3; 1751 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), 1752 LHSHi, RHSHi, ISD::SETOEQ); 1753 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), 1754 LHSLo, RHSLo, CCCode); 1755 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1756 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), 1757 LHSHi, RHSHi, ISD::SETUNE); 1758 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), 1759 LHSHi, RHSHi, CCCode); 1760 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1761 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 1762 NewRHS = SDValue(); // LHS is the result, not a compare. 1763 } 1764 1765 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { 1766 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1767 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1768 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 1769 1770 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1771 // against zero to select between true and false values. 1772 if (!NewRHS.getNode()) { 1773 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1774 CCCode = ISD::SETNE; 1775 } 1776 1777 // Update N to have the operands specified. 1778 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1779 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1780 N->getOperand(4)), 0); 1781 } 1782 1783 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { 1784 assert(N->getOperand(1).getValueType() == MVT::ppcf128 && 1785 "Logic only correct for ppcf128!"); 1786 SDValue Lo, Hi; 1787 GetExpandedFloat(N->getOperand(1), Lo, Hi); 1788 // The ppcf128 value is providing only the sign; take it from the 1789 // higher-order double (which must have the larger magnitude). 1790 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), 1791 N->getValueType(0), N->getOperand(0), Hi); 1792 } 1793 1794 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { 1795 bool IsStrict = N->isStrictFPOpcode(); 1796 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && 1797 "Logic only correct for ppcf128!"); 1798 SDValue Lo, Hi; 1799 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi); 1800 1801 if (!IsStrict) 1802 // Round it the rest of the way (e.g. to f32) if needed. 1803 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 1804 N->getValueType(0), Hi, N->getOperand(1)); 1805 1806 // Eliminate the node if the input float type is the same as the output float 1807 // type. 1808 if (Hi.getValueType() == N->getValueType(0)) { 1809 // Connect the output chain to the input chain, unlinking the node. 1810 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 1811 ReplaceValueWith(SDValue(N, 0), Hi); 1812 return SDValue(); 1813 } 1814 1815 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 1816 {N->getValueType(0), MVT::Other}, 1817 {N->getOperand(0), Hi, N->getOperand(2)}); 1818 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1)); 1819 ReplaceValueWith(SDValue(N, 0), Expansion); 1820 return SDValue(); 1821 } 1822 1823 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_SINT(SDNode *N) { 1824 EVT RVT = N->getValueType(0); 1825 SDLoc dl(N); 1826 1827 bool IsStrict = N->isStrictFPOpcode(); 1828 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1829 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1830 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), RVT); 1831 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_SINT!"); 1832 TargetLowering::MakeLibCallOptions CallOptions; 1833 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 1834 CallOptions, dl, Chain); 1835 if (!IsStrict) 1836 return Tmp.first; 1837 1838 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1839 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1840 return SDValue(); 1841 } 1842 1843 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) { 1844 EVT RVT = N->getValueType(0); 1845 SDLoc dl(N); 1846 1847 bool IsStrict = N->isStrictFPOpcode(); 1848 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1849 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1850 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), RVT); 1851 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_UINT!"); 1852 TargetLowering::MakeLibCallOptions CallOptions; 1853 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 1854 CallOptions, dl, Chain); 1855 if (!IsStrict) 1856 return Tmp.first; 1857 1858 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1859 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1860 return SDValue(); 1861 } 1862 1863 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { 1864 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1865 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1866 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 1867 1868 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1869 // against zero to select between true and false values. 1870 if (!NewRHS.getNode()) { 1871 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1872 CCCode = ISD::SETNE; 1873 } 1874 1875 // Update N to have the operands specified. 1876 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1877 N->getOperand(2), N->getOperand(3), 1878 DAG.getCondCode(CCCode)), 0); 1879 } 1880 1881 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { 1882 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1883 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 1884 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 1885 1886 // If ExpandSetCCOperands returned a scalar, use it. 1887 if (!NewRHS.getNode()) { 1888 assert(NewLHS.getValueType() == N->getValueType(0) && 1889 "Unexpected setcc expansion!"); 1890 return NewLHS; 1891 } 1892 1893 // Otherwise, update N to have the operands specified. 1894 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1895 DAG.getCondCode(CCCode)), 0); 1896 } 1897 1898 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { 1899 if (ISD::isNormalStore(N)) 1900 return ExpandOp_NormalStore(N, OpNo); 1901 1902 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1903 assert(OpNo == 1 && "Can only expand the stored value so far"); 1904 StoreSDNode *ST = cast<StoreSDNode>(N); 1905 1906 SDValue Chain = ST->getChain(); 1907 SDValue Ptr = ST->getBasePtr(); 1908 1909 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 1910 ST->getValue().getValueType()); 1911 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1912 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1913 (void)NVT; 1914 1915 SDValue Lo, Hi; 1916 GetExpandedOp(ST->getValue(), Lo, Hi); 1917 1918 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr, 1919 ST->getMemoryVT(), ST->getMemOperand()); 1920 } 1921 1922 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { 1923 EVT RVT = N->getValueType(0); 1924 EVT RetVT = N->getOperand(0).getValueType(); 1925 TargetLowering::MakeLibCallOptions CallOptions; 1926 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 1927 RTLIB::LROUND_F32, 1928 RTLIB::LROUND_F64, 1929 RTLIB::LROUND_F80, 1930 RTLIB::LROUND_F128, 1931 RTLIB::LROUND_PPCF128), 1932 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 1933 } 1934 1935 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { 1936 EVT RVT = N->getValueType(0); 1937 EVT RetVT = N->getOperand(0).getValueType(); 1938 TargetLowering::MakeLibCallOptions CallOptions; 1939 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 1940 RTLIB::LLROUND_F32, 1941 RTLIB::LLROUND_F64, 1942 RTLIB::LLROUND_F80, 1943 RTLIB::LLROUND_F128, 1944 RTLIB::LLROUND_PPCF128), 1945 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 1946 } 1947 1948 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { 1949 EVT RVT = N->getValueType(0); 1950 EVT RetVT = N->getOperand(0).getValueType(); 1951 TargetLowering::MakeLibCallOptions CallOptions; 1952 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 1953 RTLIB::LRINT_F32, 1954 RTLIB::LRINT_F64, 1955 RTLIB::LRINT_F80, 1956 RTLIB::LRINT_F128, 1957 RTLIB::LRINT_PPCF128), 1958 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 1959 } 1960 1961 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { 1962 EVT RVT = N->getValueType(0); 1963 EVT RetVT = N->getOperand(0).getValueType(); 1964 TargetLowering::MakeLibCallOptions CallOptions; 1965 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 1966 RTLIB::LLRINT_F32, 1967 RTLIB::LLRINT_F64, 1968 RTLIB::LLRINT_F80, 1969 RTLIB::LLRINT_F128, 1970 RTLIB::LLRINT_PPCF128), 1971 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 1972 } 1973 1974 //===----------------------------------------------------------------------===// 1975 // Float Operand Promotion 1976 //===----------------------------------------------------------------------===// 1977 // 1978 1979 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { 1980 if (OpVT == MVT::f16) { 1981 return ISD::FP16_TO_FP; 1982 } else if (RetVT == MVT::f16) { 1983 return ISD::FP_TO_FP16; 1984 } 1985 1986 report_fatal_error("Attempt at an invalid promotion-related conversion"); 1987 } 1988 1989 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { 1990 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG); 1991 dbgs() << "\n"); 1992 SDValue R = SDValue(); 1993 1994 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 1995 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 1996 return false; 1997 } 1998 1999 // Nodes that use a promotion-requiring floating point operand, but doesn't 2000 // produce a promotion-requiring floating point result, need to be legalized 2001 // to use the promoted float operand. Nodes that produce at least one 2002 // promotion-requiring floating point result have their operands legalized as 2003 // a part of PromoteFloatResult. 2004 switch (N->getOpcode()) { 2005 default: 2006 #ifndef NDEBUG 2007 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; 2008 N->dump(&DAG); dbgs() << "\n"; 2009 #endif 2010 llvm_unreachable("Do not know how to promote this operator's operand!"); 2011 2012 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; 2013 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; 2014 case ISD::FP_TO_SINT: 2015 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break; 2016 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; 2017 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; 2018 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; 2019 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; 2020 } 2021 2022 if (R.getNode()) 2023 ReplaceValueWith(SDValue(N, 0), R); 2024 return false; 2025 } 2026 2027 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { 2028 SDValue Op = N->getOperand(0); 2029 EVT OpVT = Op->getValueType(0); 2030 2031 SDValue Promoted = GetPromotedFloat(N->getOperand(0)); 2032 EVT PromotedVT = Promoted->getValueType(0); 2033 2034 // Convert the promoted float value to the desired IVT. 2035 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); 2036 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), 2037 IVT, Promoted); 2038 // The final result type might not be an scalar so we need a bitcast. The 2039 // bitcast will be further legalized if needed. 2040 return DAG.getBitcast(N->getValueType(0), Convert); 2041 } 2042 2043 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by 2044 // PromoteFloatRes_FCOPYSIGN. 2045 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { 2046 assert (OpNo == 1 && "Only Operand 1 must need promotion here"); 2047 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2048 2049 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2050 N->getOperand(0), Op1); 2051 } 2052 2053 // Convert the promoted float value to the desired integer type 2054 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) { 2055 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2056 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); 2057 } 2058 2059 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { 2060 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2061 EVT VT = N->getValueType(0); 2062 2063 // Desired VT is same as promoted type. Use promoted float directly. 2064 if (VT == Op->getValueType(0)) 2065 return Op; 2066 2067 // Else, extend the promoted float value to the desired VT. 2068 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); 2069 } 2070 2071 // Promote the float operands used for comparison. The true- and false- 2072 // operands have the same type as the result and are promoted, if needed, by 2073 // PromoteFloatRes_SELECT_CC 2074 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2075 SDValue LHS = GetPromotedFloat(N->getOperand(0)); 2076 SDValue RHS = GetPromotedFloat(N->getOperand(1)); 2077 2078 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 2079 LHS, RHS, N->getOperand(2), N->getOperand(3), 2080 N->getOperand(4)); 2081 } 2082 2083 // Construct a SETCC that compares the promoted values and sets the conditional 2084 // code. 2085 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { 2086 EVT VT = N->getValueType(0); 2087 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2088 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2089 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2090 2091 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); 2092 2093 } 2094 2095 // Lower the promoted Float down to the integer value of same size and construct 2096 // a STORE of the integer value. 2097 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { 2098 StoreSDNode *ST = cast<StoreSDNode>(N); 2099 SDValue Val = ST->getValue(); 2100 SDLoc DL(N); 2101 2102 SDValue Promoted = GetPromotedFloat(Val); 2103 EVT VT = ST->getOperand(1).getValueType(); 2104 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2105 2106 SDValue NewVal; 2107 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, 2108 IVT, Promoted); 2109 2110 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), 2111 ST->getMemOperand()); 2112 } 2113 2114 //===----------------------------------------------------------------------===// 2115 // Float Result Promotion 2116 //===----------------------------------------------------------------------===// 2117 2118 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { 2119 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG); 2120 dbgs() << "\n"); 2121 SDValue R = SDValue(); 2122 2123 // See if the target wants to custom expand this node. 2124 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2125 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2126 return; 2127 } 2128 2129 switch (N->getOpcode()) { 2130 // These opcodes cannot appear if promotion of FP16 is done in the backend 2131 // instead of Clang 2132 case ISD::FP16_TO_FP: 2133 case ISD::FP_TO_FP16: 2134 default: 2135 #ifndef NDEBUG 2136 dbgs() << "PromoteFloatResult #" << ResNo << ": "; 2137 N->dump(&DAG); dbgs() << "\n"; 2138 #endif 2139 llvm_unreachable("Do not know how to promote this operator's result!"); 2140 2141 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; 2142 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; 2143 case ISD::EXTRACT_VECTOR_ELT: 2144 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; 2145 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; 2146 2147 // Unary FP Operations 2148 case ISD::FABS: 2149 case ISD::FCBRT: 2150 case ISD::FCEIL: 2151 case ISD::FCOS: 2152 case ISD::FEXP: 2153 case ISD::FEXP2: 2154 case ISD::FFLOOR: 2155 case ISD::FLOG: 2156 case ISD::FLOG2: 2157 case ISD::FLOG10: 2158 case ISD::FNEARBYINT: 2159 case ISD::FNEG: 2160 case ISD::FRINT: 2161 case ISD::FROUND: 2162 case ISD::FROUNDEVEN: 2163 case ISD::FSIN: 2164 case ISD::FSQRT: 2165 case ISD::FTRUNC: 2166 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; 2167 2168 // Binary FP Operations 2169 case ISD::FADD: 2170 case ISD::FDIV: 2171 case ISD::FMAXIMUM: 2172 case ISD::FMINIMUM: 2173 case ISD::FMAXNUM: 2174 case ISD::FMINNUM: 2175 case ISD::FMUL: 2176 case ISD::FPOW: 2177 case ISD::FREM: 2178 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; 2179 2180 case ISD::FMA: // FMA is same as FMAD 2181 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; 2182 2183 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break; 2184 2185 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; 2186 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; 2187 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; 2188 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; 2189 2190 case ISD::SINT_TO_FP: 2191 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; 2192 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; 2193 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2194 } 2195 2196 if (R.getNode()) 2197 SetPromotedFloat(SDValue(N, ResNo), R); 2198 } 2199 2200 // Bitcast from i16 to f16: convert the i16 to a f32 value instead. 2201 // At this point, it is not possible to determine if the bitcast value is 2202 // eventually stored to memory or promoted to f32 or promoted to a floating 2203 // point at a higher precision. Some of these cases are handled by FP_EXTEND, 2204 // STORE promotion handlers. 2205 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { 2206 EVT VT = N->getValueType(0); 2207 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2208 // Input type isn't guaranteed to be a scalar int so bitcast if not. The 2209 // bitcast will be legalized further if necessary. 2210 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), 2211 N->getOperand(0).getValueType().getSizeInBits()); 2212 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); 2213 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); 2214 } 2215 2216 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { 2217 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); 2218 EVT VT = N->getValueType(0); 2219 SDLoc DL(N); 2220 2221 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2222 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2223 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, 2224 IVT); 2225 2226 // Convert the Constant to the desired FP type 2227 // FIXME We might be able to do the conversion during compilation and get rid 2228 // of it from the object code 2229 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2230 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); 2231 } 2232 2233 // If the Index operand is a constant, try to redirect the extract operation to 2234 // the correct legalized vector. If not, bit-convert the input vector to 2235 // equivalent integer vector. Extract the element as an (bit-cast) integer 2236 // value and convert it to the promoted type. 2237 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2238 SDLoc DL(N); 2239 2240 // If the index is constant, try to extract the value from the legalized 2241 // vector type. 2242 if (isa<ConstantSDNode>(N->getOperand(1))) { 2243 SDValue Vec = N->getOperand(0); 2244 SDValue Idx = N->getOperand(1); 2245 EVT VecVT = Vec->getValueType(0); 2246 EVT EltVT = VecVT.getVectorElementType(); 2247 2248 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2249 2250 switch (getTypeAction(VecVT)) { 2251 default: break; 2252 case TargetLowering::TypeScalarizeVector: { 2253 SDValue Res = GetScalarizedVector(N->getOperand(0)); 2254 ReplaceValueWith(SDValue(N, 0), Res); 2255 return SDValue(); 2256 } 2257 case TargetLowering::TypeWidenVector: { 2258 Vec = GetWidenedVector(Vec); 2259 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); 2260 ReplaceValueWith(SDValue(N, 0), Res); 2261 return SDValue(); 2262 } 2263 case TargetLowering::TypeSplitVector: { 2264 SDValue Lo, Hi; 2265 GetSplitVector(Vec, Lo, Hi); 2266 2267 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2268 SDValue Res; 2269 if (IdxVal < LoElts) 2270 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); 2271 else 2272 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, 2273 DAG.getConstant(IdxVal - LoElts, DL, 2274 Idx.getValueType())); 2275 ReplaceValueWith(SDValue(N, 0), Res); 2276 return SDValue(); 2277 } 2278 2279 } 2280 } 2281 2282 // Bit-convert the input vector to the equivalent integer vector 2283 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2284 EVT IVT = NewOp.getValueType().getVectorElementType(); 2285 2286 // Extract the element as an (bit-cast) integer value 2287 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, 2288 NewOp, N->getOperand(1)); 2289 2290 // Convert the element to the desired FP type 2291 EVT VT = N->getValueType(0); 2292 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2293 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); 2294 } 2295 2296 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result 2297 // needs promotion, so does the argument X. Note that Y, if needed, will be 2298 // handled during operand promotion. 2299 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { 2300 EVT VT = N->getValueType(0); 2301 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2302 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2303 2304 SDValue Op1 = N->getOperand(1); 2305 2306 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2307 } 2308 2309 // Unary operation where the result and the operand have PromoteFloat type 2310 // action. Construct a new SDNode with the promoted float value of the old 2311 // operand. 2312 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { 2313 EVT VT = N->getValueType(0); 2314 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2315 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2316 2317 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); 2318 } 2319 2320 // Binary operations where the result and both operands have PromoteFloat type 2321 // action. Construct a new SDNode with the promoted float values of the old 2322 // operands. 2323 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { 2324 EVT VT = N->getValueType(0); 2325 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2326 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2327 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2328 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); 2329 } 2330 2331 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { 2332 EVT VT = N->getValueType(0); 2333 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2334 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2335 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2336 SDValue Op2 = GetPromotedFloat(N->getOperand(2)); 2337 2338 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); 2339 } 2340 2341 // Promote the Float (first) operand and retain the Integer (second) operand 2342 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) { 2343 EVT VT = N->getValueType(0); 2344 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2345 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2346 SDValue Op1 = N->getOperand(1); 2347 2348 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2349 } 2350 2351 // Explicit operation to reduce precision. Reduce the value to half precision 2352 // and promote it back to the legal type. 2353 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { 2354 SDLoc DL(N); 2355 2356 SDValue Op = N->getOperand(0); 2357 EVT VT = N->getValueType(0); 2358 EVT OpVT = Op->getValueType(0); 2359 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2360 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2361 2362 // Round promoted float to desired precision 2363 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); 2364 // Promote it back to the legal output type 2365 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); 2366 } 2367 2368 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { 2369 LoadSDNode *L = cast<LoadSDNode>(N); 2370 EVT VT = N->getValueType(0); 2371 2372 // Load the value as an integer value with the same number of bits. 2373 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2374 SDValue newL = DAG.getLoad( 2375 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), 2376 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, 2377 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); 2378 // Legalize the chain result by replacing uses of the old value chain with the 2379 // new one 2380 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 2381 2382 // Convert the integer value to the desired FP type 2383 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2384 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); 2385 } 2386 2387 // Construct a new SELECT node with the promoted true- and false- values. 2388 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { 2389 SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); 2390 SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); 2391 2392 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), 2393 N->getOperand(0), TrueVal, FalseVal); 2394 } 2395 2396 // Construct a new SELECT_CC node with the promoted true- and false- values. 2397 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. 2398 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { 2399 SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); 2400 SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); 2401 2402 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2403 TrueVal.getNode()->getValueType(0), N->getOperand(0), 2404 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); 2405 } 2406 2407 // Construct a SDNode that transforms the SINT or UINT operand to the promoted 2408 // float type. 2409 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { 2410 SDLoc DL(N); 2411 EVT VT = N->getValueType(0); 2412 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2413 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); 2414 // Round the value to the desired precision (that of the source type). 2415 return DAG.getNode( 2416 ISD::FP_EXTEND, DL, NVT, 2417 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL))); 2418 } 2419 2420 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { 2421 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 2422 N->getValueType(0))); 2423 } 2424 2425 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { 2426 EVT VT = N->getValueType(0); 2427 2428 AtomicSDNode *AM = cast<AtomicSDNode>(N); 2429 SDLoc SL(N); 2430 2431 SDValue CastVal = BitConvertToInteger(AM->getVal()); 2432 EVT CastVT = CastVal.getValueType(); 2433 2434 SDValue NewAtomic 2435 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, 2436 DAG.getVTList(CastVT, MVT::Other), 2437 { AM->getChain(), AM->getBasePtr(), CastVal }, 2438 AM->getMemOperand()); 2439 2440 SDValue Result = NewAtomic; 2441 2442 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { 2443 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2444 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, 2445 NewAtomic); 2446 } 2447 2448 // Legalize the chain result by replacing uses of the old value chain with the 2449 // new one 2450 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); 2451 2452 return Result; 2453 2454 } 2455 2456 //===----------------------------------------------------------------------===// 2457 // Half Result Soft Promotion 2458 //===----------------------------------------------------------------------===// 2459 2460 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { 2461 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": "; 2462 N->dump(&DAG); dbgs() << "\n"); 2463 SDValue R = SDValue(); 2464 2465 // See if the target wants to custom expand this node. 2466 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2467 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2468 return; 2469 } 2470 2471 switch (N->getOpcode()) { 2472 default: 2473 #ifndef NDEBUG 2474 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": "; 2475 N->dump(&DAG); dbgs() << "\n"; 2476 #endif 2477 llvm_unreachable("Do not know how to soft promote this operator's result!"); 2478 2479 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; 2480 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; 2481 case ISD::EXTRACT_VECTOR_ELT: 2482 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; 2483 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; 2484 case ISD::STRICT_FP_ROUND: 2485 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; 2486 2487 // Unary FP Operations 2488 case ISD::FABS: 2489 case ISD::FCBRT: 2490 case ISD::FCEIL: 2491 case ISD::FCOS: 2492 case ISD::FEXP: 2493 case ISD::FEXP2: 2494 case ISD::FFLOOR: 2495 case ISD::FLOG: 2496 case ISD::FLOG2: 2497 case ISD::FLOG10: 2498 case ISD::FNEARBYINT: 2499 case ISD::FNEG: 2500 case ISD::FREEZE: 2501 case ISD::FRINT: 2502 case ISD::FROUND: 2503 case ISD::FROUNDEVEN: 2504 case ISD::FSIN: 2505 case ISD::FSQRT: 2506 case ISD::FTRUNC: 2507 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; 2508 2509 // Binary FP Operations 2510 case ISD::FADD: 2511 case ISD::FDIV: 2512 case ISD::FMAXIMUM: 2513 case ISD::FMINIMUM: 2514 case ISD::FMAXNUM: 2515 case ISD::FMINNUM: 2516 case ISD::FMUL: 2517 case ISD::FPOW: 2518 case ISD::FREM: 2519 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; 2520 2521 case ISD::FMA: // FMA is same as FMAD 2522 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; 2523 2524 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break; 2525 2526 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; 2527 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; 2528 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; 2529 case ISD::SINT_TO_FP: 2530 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; 2531 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; 2532 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2533 } 2534 2535 if (R.getNode()) 2536 SetSoftPromotedHalf(SDValue(N, ResNo), R); 2537 } 2538 2539 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { 2540 return BitConvertToInteger(N->getOperand(0)); 2541 } 2542 2543 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { 2544 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 2545 2546 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2547 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 2548 MVT::i16); 2549 } 2550 2551 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2552 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2553 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2554 NewOp.getValueType().getVectorElementType(), NewOp, 2555 N->getOperand(1)); 2556 } 2557 2558 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { 2559 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0)); 2560 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 2561 SDLoc dl(N); 2562 2563 EVT LVT = LHS.getValueType(); 2564 EVT RVT = RHS.getValueType(); 2565 2566 unsigned LSize = LVT.getSizeInBits(); 2567 unsigned RSize = RVT.getSizeInBits(); 2568 2569 // First get the sign bit of second operand. 2570 SDValue SignBit = DAG.getNode( 2571 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 2572 DAG.getConstant(RSize - 1, dl, 2573 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 2574 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 2575 2576 // Shift right or sign-extend it if the two operands have different types. 2577 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 2578 if (SizeDiff > 0) { 2579 SignBit = 2580 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 2581 DAG.getConstant(SizeDiff, dl, 2582 TLI.getShiftAmountTy(SignBit.getValueType(), 2583 DAG.getDataLayout()))); 2584 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 2585 } else if (SizeDiff < 0) { 2586 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 2587 SignBit = 2588 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 2589 DAG.getConstant(-SizeDiff, dl, 2590 TLI.getShiftAmountTy(SignBit.getValueType(), 2591 DAG.getDataLayout()))); 2592 } 2593 2594 // Clear the sign bit of the first operand. 2595 SDValue Mask = DAG.getNode( 2596 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 2597 DAG.getConstant(LSize - 1, dl, 2598 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 2599 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 2600 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 2601 2602 // Or the value with the sign bit. 2603 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 2604 } 2605 2606 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { 2607 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2608 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2609 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2610 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2611 SDLoc dl(N); 2612 2613 // Promote to the larger FP type. 2614 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2615 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2616 Op2 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op2); 2617 2618 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2); 2619 2620 // Convert back to FP16 as an integer. 2621 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2622 } 2623 2624 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) { 2625 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2626 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2627 SDValue Op1 = N->getOperand(1); 2628 SDLoc dl(N); 2629 2630 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2631 2632 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2633 2634 // Convert back to FP16 as an integer. 2635 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2636 } 2637 2638 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { 2639 if (N->isStrictFPOpcode()) { 2640 SDValue Res = 2641 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other}, 2642 {N->getOperand(0), N->getOperand(1)}); 2643 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2644 return Res; 2645 } 2646 2647 return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), MVT::i16, N->getOperand(0)); 2648 } 2649 2650 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { 2651 LoadSDNode *L = cast<LoadSDNode>(N); 2652 2653 // Load the value as an integer value with the same number of bits. 2654 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!"); 2655 SDValue NewL = 2656 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, 2657 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), 2658 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), 2659 L->getMemOperand()->getFlags(), L->getAAInfo()); 2660 // Legalize the chain result by replacing uses of the old value chain with the 2661 // new one 2662 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 2663 return NewL; 2664 } 2665 2666 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { 2667 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2668 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2669 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1, 2670 Op2); 2671 } 2672 2673 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { 2674 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2675 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3)); 2676 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(), 2677 N->getOperand(0), N->getOperand(1), Op2, Op3, 2678 N->getOperand(4)); 2679 } 2680 2681 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { 2682 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2683 SDLoc dl(N); 2684 2685 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 2686 2687 // Round the value to the softened type. 2688 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2689 } 2690 2691 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { 2692 return DAG.getUNDEF(MVT::i16); 2693 } 2694 2695 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { 2696 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2697 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 2698 SDLoc dl(N); 2699 2700 // Promote to the larger FP type. 2701 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); 2702 2703 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op); 2704 2705 // Convert back to FP16 as an integer. 2706 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2707 } 2708 2709 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { 2710 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2711 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2712 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2713 SDLoc dl(N); 2714 2715 // Promote to the larger FP type. 2716 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2717 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2718 2719 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2720 2721 // Convert back to FP16 as an integer. 2722 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2723 } 2724 2725 //===----------------------------------------------------------------------===// 2726 // Half Operand Soft Promotion 2727 //===----------------------------------------------------------------------===// 2728 2729 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { 2730 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": "; 2731 N->dump(&DAG); dbgs() << "\n"); 2732 SDValue Res = SDValue(); 2733 2734 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2735 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2736 return false; 2737 } 2738 2739 // Nodes that use a promotion-requiring floating point operand, but doesn't 2740 // produce a soft promotion-requiring floating point result, need to be 2741 // legalized to use the soft promoted float operand. Nodes that produce at 2742 // least one soft promotion-requiring floating point result have their 2743 // operands legalized as a part of PromoteFloatResult. 2744 switch (N->getOpcode()) { 2745 default: 2746 #ifndef NDEBUG 2747 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": "; 2748 N->dump(&DAG); dbgs() << "\n"; 2749 #endif 2750 llvm_unreachable("Do not know how to soft promote this operator's operand!"); 2751 2752 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; 2753 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; 2754 case ISD::FP_TO_SINT: 2755 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; 2756 case ISD::STRICT_FP_EXTEND: 2757 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; 2758 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; 2759 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; 2760 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; 2761 } 2762 2763 if (!Res.getNode()) 2764 return false; 2765 2766 assert(Res.getNode() != N && "Expected a new node!"); 2767 2768 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2769 "Invalid operand expansion"); 2770 2771 ReplaceValueWith(SDValue(N, 0), Res); 2772 return false; 2773 } 2774 2775 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { 2776 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2777 2778 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 2779 } 2780 2781 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, 2782 unsigned OpNo) { 2783 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 2784 SDValue Op1 = N->getOperand(1); 2785 SDLoc dl(N); 2786 2787 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType()); 2788 2789 Op1 = GetSoftPromotedHalf(Op1); 2790 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2791 2792 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0), 2793 Op1); 2794 } 2795 2796 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { 2797 bool IsStrict = N->isStrictFPOpcode(); 2798 SDValue Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0)); 2799 2800 if (IsStrict) { 2801 SDValue Res = 2802 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N), 2803 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op}); 2804 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2805 ReplaceValueWith(SDValue(N, 0), Res); 2806 return SDValue(); 2807 } 2808 2809 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), Op); 2810 } 2811 2812 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { 2813 SDValue Op = N->getOperand(0); 2814 SDLoc dl(N); 2815 2816 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 2817 2818 Op = GetSoftPromotedHalf(Op); 2819 2820 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); 2821 2822 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res); 2823 } 2824 2825 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, 2826 unsigned OpNo) { 2827 assert(OpNo == 0 && "Can only soften the comparison values"); 2828 SDValue Op0 = N->getOperand(0); 2829 SDValue Op1 = N->getOperand(1); 2830 SDLoc dl(N); 2831 2832 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 2833 2834 Op0 = GetSoftPromotedHalf(Op0); 2835 Op1 = GetSoftPromotedHalf(Op1); 2836 2837 // Promote to the larger FP type. 2838 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2839 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2840 2841 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1, 2842 N->getOperand(2), N->getOperand(3), N->getOperand(4)); 2843 } 2844 2845 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { 2846 SDValue Op0 = N->getOperand(0); 2847 SDValue Op1 = N->getOperand(1); 2848 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2849 SDLoc dl(N); 2850 2851 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 2852 2853 Op0 = GetSoftPromotedHalf(Op0); 2854 Op1 = GetSoftPromotedHalf(Op1); 2855 2856 // Promote to the larger FP type. 2857 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2858 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2859 2860 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode); 2861 } 2862 2863 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { 2864 assert(OpNo == 1 && "Can only soften the stored value!"); 2865 StoreSDNode *ST = cast<StoreSDNode>(N); 2866 SDValue Val = ST->getValue(); 2867 SDLoc dl(N); 2868 2869 assert(!ST->isTruncatingStore() && "Unexpected truncating store."); 2870 SDValue Promoted = GetSoftPromotedHalf(Val); 2871 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), 2872 ST->getMemOperand()); 2873 } 2874