1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements float type expansion and softening for LegalizeTypes. 10 // Softening is the act of turning a computation in an illegal floating point 11 // type into a computation in an integer type of the same size; also known as 12 // "soft float". For example, turning f32 arithmetic into operations using i32. 13 // The resulting integer value is the same as what you would get by performing 14 // the floating point operation and bitcasting the result to the integer type. 15 // Expansion is the act of changing a computation in an illegal type to be a 16 // computation in two identical registers of a smaller type. For example, 17 // implementing ppcf128 arithmetic in two f64 registers. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/Analysis/TargetLibraryInfo.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 /// GetFPLibCall - Return the right libcall for the given floating point type. 30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be 31 /// refactored away (see RTLIB::getPOWI for an example). 32 static RTLIB::Libcall GetFPLibCall(EVT VT, 33 RTLIB::Libcall Call_F32, 34 RTLIB::Libcall Call_F64, 35 RTLIB::Libcall Call_F80, 36 RTLIB::Libcall Call_F128, 37 RTLIB::Libcall Call_PPCF128) { 38 return 39 VT == MVT::f32 ? Call_F32 : 40 VT == MVT::f64 ? Call_F64 : 41 VT == MVT::f80 ? Call_F80 : 42 VT == MVT::f128 ? Call_F128 : 43 VT == MVT::ppcf128 ? Call_PPCF128 : 44 RTLIB::UNKNOWN_LIBCALL; 45 } 46 47 //===----------------------------------------------------------------------===// 48 // Convert Float Results to Integer 49 //===----------------------------------------------------------------------===// 50 51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { 52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG); 53 dbgs() << "\n"); 54 SDValue R = SDValue(); 55 56 switch (N->getOpcode()) { 57 default: 58 #ifndef NDEBUG 59 dbgs() << "SoftenFloatResult #" << ResNo << ": "; 60 N->dump(&DAG); dbgs() << "\n"; 61 #endif 62 llvm_unreachable("Do not know how to soften the result of this operator!"); 63 64 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; 65 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; 66 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; 67 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; 68 case ISD::EXTRACT_VECTOR_ELT: 69 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; 70 case ISD::FABS: R = SoftenFloatRes_FABS(N); break; 71 case ISD::STRICT_FMINNUM: 72 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; 73 case ISD::STRICT_FMAXNUM: 74 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; 75 case ISD::STRICT_FADD: 76 case ISD::FADD: R = SoftenFloatRes_FADD(N); break; 77 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; 78 case ISD::STRICT_FCEIL: 79 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; 80 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; 81 case ISD::STRICT_FCOS: 82 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; 83 case ISD::STRICT_FDIV: 84 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; 85 case ISD::STRICT_FEXP: 86 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; 87 case ISD::STRICT_FEXP2: 88 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; 89 case ISD::STRICT_FFLOOR: 90 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; 91 case ISD::STRICT_FLOG: 92 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; 93 case ISD::STRICT_FLOG2: 94 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; 95 case ISD::STRICT_FLOG10: 96 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; 97 case ISD::STRICT_FMA: 98 case ISD::FMA: R = SoftenFloatRes_FMA(N); break; 99 case ISD::STRICT_FMUL: 100 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; 101 case ISD::STRICT_FNEARBYINT: 102 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; 103 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; 104 case ISD::STRICT_FP_EXTEND: 105 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; 106 case ISD::STRICT_FP_ROUND: 107 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; 108 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; 109 case ISD::STRICT_FPOW: 110 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; 111 case ISD::STRICT_FPOWI: 112 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break; 113 case ISD::STRICT_FREM: 114 case ISD::FREM: R = SoftenFloatRes_FREM(N); break; 115 case ISD::STRICT_FRINT: 116 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; 117 case ISD::STRICT_FROUND: 118 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; 119 case ISD::STRICT_FROUNDEVEN: 120 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; 121 case ISD::STRICT_FSIN: 122 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; 123 case ISD::STRICT_FSQRT: 124 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; 125 case ISD::STRICT_FSUB: 126 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; 127 case ISD::STRICT_FTRUNC: 128 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; 129 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; 130 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 131 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; 132 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; 133 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; 134 case ISD::STRICT_SINT_TO_FP: 135 case ISD::STRICT_UINT_TO_FP: 136 case ISD::SINT_TO_FP: 137 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; 138 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; 139 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; 140 case ISD::VECREDUCE_FADD: 141 case ISD::VECREDUCE_FMUL: 142 case ISD::VECREDUCE_FMIN: 143 case ISD::VECREDUCE_FMAX: 144 R = SoftenFloatRes_VECREDUCE(N); 145 break; 146 case ISD::VECREDUCE_SEQ_FADD: 147 case ISD::VECREDUCE_SEQ_FMUL: 148 R = SoftenFloatRes_VECREDUCE_SEQ(N); 149 break; 150 } 151 152 // If R is null, the sub-method took care of registering the result. 153 if (R.getNode()) { 154 assert(R.getNode() != N); 155 SetSoftenedFloat(SDValue(N, ResNo), R); 156 } 157 } 158 159 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { 160 bool IsStrict = N->isStrictFPOpcode(); 161 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 162 unsigned Offset = IsStrict ? 1 : 0; 163 assert(N->getNumOperands() == (1 + Offset) && 164 "Unexpected number of operands!"); 165 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 166 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 167 TargetLowering::MakeLibCallOptions CallOptions; 168 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 169 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 170 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 171 CallOptions, SDLoc(N), 172 Chain); 173 if (IsStrict) 174 ReplaceValueWith(SDValue(N, 1), Tmp.second); 175 return Tmp.first; 176 } 177 178 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { 179 bool IsStrict = N->isStrictFPOpcode(); 180 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 181 unsigned Offset = IsStrict ? 1 : 0; 182 assert(N->getNumOperands() == (2 + Offset) && 183 "Unexpected number of operands!"); 184 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 185 GetSoftenedFloat(N->getOperand(1 + Offset)) }; 186 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 187 TargetLowering::MakeLibCallOptions CallOptions; 188 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 189 N->getOperand(1 + Offset).getValueType() }; 190 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 191 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 192 CallOptions, SDLoc(N), 193 Chain); 194 if (IsStrict) 195 ReplaceValueWith(SDValue(N, 1), Tmp.second); 196 return Tmp.first; 197 } 198 199 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { 200 return BitConvertToInteger(N->getOperand(0)); 201 } 202 203 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { 204 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 205 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty, 206 GetSoftenedFloat(N->getOperand(0))); 207 } 208 209 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, 210 unsigned ResNo) { 211 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 212 return BitConvertToInteger(Op); 213 } 214 215 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { 216 // Convert the inputs to integers, and build a new pair out of them. 217 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), 218 TLI.getTypeToTransformTo(*DAG.getContext(), 219 N->getValueType(0)), 220 BitConvertToInteger(N->getOperand(0)), 221 BitConvertToInteger(N->getOperand(1))); 222 } 223 224 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { 225 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 226 // In ppcf128, the high 64 bits are always first in memory regardless 227 // of Endianness. LLVM's APFloat representation is not Endian sensitive, 228 // and so always converts into a 128-bit APInt in a non-Endian-sensitive 229 // way. However, APInt's are serialized in an Endian-sensitive fashion, 230 // so on big-Endian targets, the two doubles are output in the wrong 231 // order. Fix this by manually flipping the order of the high 64 bits 232 // and the low 64 bits here. 233 if (DAG.getDataLayout().isBigEndian() && 234 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) { 235 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], 236 CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; 237 APInt Val(128, words); 238 return DAG.getConstant(Val, SDLoc(CN), 239 TLI.getTypeToTransformTo(*DAG.getContext(), 240 CN->getValueType(0))); 241 } else { 242 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 243 TLI.getTypeToTransformTo(*DAG.getContext(), 244 CN->getValueType(0))); 245 } 246 } 247 248 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) { 249 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 250 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 251 NewOp.getValueType().getVectorElementType(), 252 NewOp, N->getOperand(1)); 253 } 254 255 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { 256 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 257 unsigned Size = NVT.getSizeInBits(); 258 259 // Mask = ~(1 << (Size-1)) 260 APInt API = APInt::getAllOnesValue(Size); 261 API.clearBit(Size - 1); 262 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT); 263 SDValue Op = GetSoftenedFloat(N->getOperand(0)); 264 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask); 265 } 266 267 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { 268 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 269 RTLIB::FMIN_F32, 270 RTLIB::FMIN_F64, 271 RTLIB::FMIN_F80, 272 RTLIB::FMIN_F128, 273 RTLIB::FMIN_PPCF128)); 274 } 275 276 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { 277 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 278 RTLIB::FMAX_F32, 279 RTLIB::FMAX_F64, 280 RTLIB::FMAX_F80, 281 RTLIB::FMAX_F128, 282 RTLIB::FMAX_PPCF128)); 283 } 284 285 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { 286 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 287 RTLIB::ADD_F32, 288 RTLIB::ADD_F64, 289 RTLIB::ADD_F80, 290 RTLIB::ADD_F128, 291 RTLIB::ADD_PPCF128)); 292 } 293 294 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { 295 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 296 RTLIB::CBRT_F32, 297 RTLIB::CBRT_F64, 298 RTLIB::CBRT_F80, 299 RTLIB::CBRT_F128, 300 RTLIB::CBRT_PPCF128)); 301 } 302 303 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { 304 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 305 RTLIB::CEIL_F32, 306 RTLIB::CEIL_F64, 307 RTLIB::CEIL_F80, 308 RTLIB::CEIL_F128, 309 RTLIB::CEIL_PPCF128)); 310 } 311 312 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { 313 SDValue LHS = GetSoftenedFloat(N->getOperand(0)); 314 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 315 SDLoc dl(N); 316 317 EVT LVT = LHS.getValueType(); 318 EVT RVT = RHS.getValueType(); 319 320 unsigned LSize = LVT.getSizeInBits(); 321 unsigned RSize = RVT.getSizeInBits(); 322 323 // First get the sign bit of second operand. 324 SDValue SignBit = DAG.getNode( 325 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 326 DAG.getConstant(RSize - 1, dl, 327 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 328 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 329 330 // Shift right or sign-extend it if the two operands have different types. 331 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 332 if (SizeDiff > 0) { 333 SignBit = 334 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 335 DAG.getConstant(SizeDiff, dl, 336 TLI.getShiftAmountTy(SignBit.getValueType(), 337 DAG.getDataLayout()))); 338 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 339 } else if (SizeDiff < 0) { 340 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 341 SignBit = 342 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 343 DAG.getConstant(-SizeDiff, dl, 344 TLI.getShiftAmountTy(SignBit.getValueType(), 345 DAG.getDataLayout()))); 346 } 347 348 // Clear the sign bit of the first operand. 349 SDValue Mask = DAG.getNode( 350 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 351 DAG.getConstant(LSize - 1, dl, 352 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 353 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 354 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 355 356 // Or the value with the sign bit. 357 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 358 } 359 360 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { 361 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 362 RTLIB::COS_F32, 363 RTLIB::COS_F64, 364 RTLIB::COS_F80, 365 RTLIB::COS_F128, 366 RTLIB::COS_PPCF128)); 367 } 368 369 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { 370 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 371 RTLIB::DIV_F32, 372 RTLIB::DIV_F64, 373 RTLIB::DIV_F80, 374 RTLIB::DIV_F128, 375 RTLIB::DIV_PPCF128)); 376 } 377 378 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { 379 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 380 RTLIB::EXP_F32, 381 RTLIB::EXP_F64, 382 RTLIB::EXP_F80, 383 RTLIB::EXP_F128, 384 RTLIB::EXP_PPCF128)); 385 } 386 387 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { 388 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 389 RTLIB::EXP2_F32, 390 RTLIB::EXP2_F64, 391 RTLIB::EXP2_F80, 392 RTLIB::EXP2_F128, 393 RTLIB::EXP2_PPCF128)); 394 } 395 396 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { 397 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 398 RTLIB::FLOOR_F32, 399 RTLIB::FLOOR_F64, 400 RTLIB::FLOOR_F80, 401 RTLIB::FLOOR_F128, 402 RTLIB::FLOOR_PPCF128)); 403 } 404 405 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { 406 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 407 RTLIB::LOG_F32, 408 RTLIB::LOG_F64, 409 RTLIB::LOG_F80, 410 RTLIB::LOG_F128, 411 RTLIB::LOG_PPCF128)); 412 } 413 414 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { 415 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 416 RTLIB::LOG2_F32, 417 RTLIB::LOG2_F64, 418 RTLIB::LOG2_F80, 419 RTLIB::LOG2_F128, 420 RTLIB::LOG2_PPCF128)); 421 } 422 423 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { 424 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 425 RTLIB::LOG10_F32, 426 RTLIB::LOG10_F64, 427 RTLIB::LOG10_F80, 428 RTLIB::LOG10_F128, 429 RTLIB::LOG10_PPCF128)); 430 } 431 432 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { 433 bool IsStrict = N->isStrictFPOpcode(); 434 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 435 unsigned Offset = IsStrict ? 1 : 0; 436 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 437 GetSoftenedFloat(N->getOperand(1 + Offset)), 438 GetSoftenedFloat(N->getOperand(2 + Offset)) }; 439 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 440 TargetLowering::MakeLibCallOptions CallOptions; 441 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(), 442 N->getOperand(1 + Offset).getValueType(), 443 N->getOperand(2 + Offset).getValueType() }; 444 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 445 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, 446 GetFPLibCall(N->getValueType(0), 447 RTLIB::FMA_F32, 448 RTLIB::FMA_F64, 449 RTLIB::FMA_F80, 450 RTLIB::FMA_F128, 451 RTLIB::FMA_PPCF128), 452 NVT, Ops, CallOptions, SDLoc(N), Chain); 453 if (IsStrict) 454 ReplaceValueWith(SDValue(N, 1), Tmp.second); 455 return Tmp.first; 456 } 457 458 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { 459 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 460 RTLIB::MUL_F32, 461 RTLIB::MUL_F64, 462 RTLIB::MUL_F80, 463 RTLIB::MUL_F128, 464 RTLIB::MUL_PPCF128)); 465 } 466 467 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { 468 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 469 RTLIB::NEARBYINT_F32, 470 RTLIB::NEARBYINT_F64, 471 RTLIB::NEARBYINT_F80, 472 RTLIB::NEARBYINT_F128, 473 RTLIB::NEARBYINT_PPCF128)); 474 } 475 476 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { 477 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 478 SDLoc dl(N); 479 480 // Expand Y = FNEG(X) -> Y = X ^ sign mask 481 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); 482 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), 483 DAG.getConstant(SignMask, dl, NVT)); 484 } 485 486 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { 487 bool IsStrict = N->isStrictFPOpcode(); 488 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 489 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 490 491 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 492 493 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { 494 Op = GetPromotedFloat(Op); 495 // If the promotion did the FP_EXTEND to the destination type for us, 496 // there's nothing left to do here. 497 if (Op.getValueType() == N->getValueType(0)) 498 return BitConvertToInteger(Op); 499 } 500 501 // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's 502 // entirely possible for both f16 and f32 to be legal, so use the fully 503 // hard-float FP_EXTEND rather than FP16_TO_FP. 504 if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) { 505 if (IsStrict) { 506 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 507 { MVT::f32, MVT::Other }, { Chain, Op }); 508 Chain = Op.getValue(1); 509 } else { 510 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); 511 } 512 } 513 514 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); 515 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 516 TargetLowering::MakeLibCallOptions CallOptions; 517 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 518 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 519 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 520 CallOptions, SDLoc(N), 521 Chain); 522 if (IsStrict) 523 ReplaceValueWith(SDValue(N, 1), Tmp.second); 524 return Tmp.first; 525 } 526 527 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 528 // nodes? 529 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { 530 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 531 SDValue Op = N->getOperand(0); 532 TargetLowering::MakeLibCallOptions CallOptions; 533 EVT OpsVT[1] = { N->getOperand(0).getValueType() }; 534 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 535 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op, 536 CallOptions, SDLoc(N)).first; 537 if (N->getValueType(0) == MVT::f32) 538 return Res32; 539 540 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 541 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0)); 542 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 543 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first; 544 } 545 546 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { 547 bool IsStrict = N->isStrictFPOpcode(); 548 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 549 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 550 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 551 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); 552 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 553 TargetLowering::MakeLibCallOptions CallOptions; 554 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 555 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 556 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 557 CallOptions, SDLoc(N), 558 Chain); 559 if (IsStrict) 560 ReplaceValueWith(SDValue(N, 1), Tmp.second); 561 return Tmp.first; 562 } 563 564 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { 565 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 566 RTLIB::POW_F32, 567 RTLIB::POW_F64, 568 RTLIB::POW_F80, 569 RTLIB::POW_F128, 570 RTLIB::POW_PPCF128)); 571 } 572 573 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) { 574 bool IsStrict = N->isStrictFPOpcode(); 575 unsigned Offset = IsStrict ? 1 : 0; 576 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 || 577 N->getOperand(1 + Offset).getValueType() == MVT::i32) && 578 "Unsupported power type!"); 579 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0)); 580 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 581 if (!TLI.getLibcallName(LC)) { 582 // Some targets don't have a powi libcall; use pow instead. 583 // FIXME: Implement this if some target needs it. 584 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow"); 585 return DAG.getUNDEF(N->getValueType(0)); 586 } 587 588 if (DAG.getLibInfo().getIntSize() != 589 N->getOperand(1 + Offset).getValueType().getSizeInBits()) { 590 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI 591 // would use the wrong type for the argument. 592 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); 593 return DAG.getUNDEF(N->getValueType(0)); 594 } 595 596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 597 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 598 N->getOperand(1 + Offset) }; 599 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 600 TargetLowering::MakeLibCallOptions CallOptions; 601 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 602 N->getOperand(1 + Offset).getValueType() }; 603 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 604 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 605 CallOptions, SDLoc(N), 606 Chain); 607 if (IsStrict) 608 ReplaceValueWith(SDValue(N, 1), Tmp.second); 609 return Tmp.first; 610 } 611 612 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { 613 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 614 RTLIB::REM_F32, 615 RTLIB::REM_F64, 616 RTLIB::REM_F80, 617 RTLIB::REM_F128, 618 RTLIB::REM_PPCF128)); 619 } 620 621 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { 622 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 623 RTLIB::RINT_F32, 624 RTLIB::RINT_F64, 625 RTLIB::RINT_F80, 626 RTLIB::RINT_F128, 627 RTLIB::RINT_PPCF128)); 628 } 629 630 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { 631 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 632 RTLIB::ROUND_F32, 633 RTLIB::ROUND_F64, 634 RTLIB::ROUND_F80, 635 RTLIB::ROUND_F128, 636 RTLIB::ROUND_PPCF128)); 637 } 638 639 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { 640 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 641 RTLIB::ROUNDEVEN_F32, 642 RTLIB::ROUNDEVEN_F64, 643 RTLIB::ROUNDEVEN_F80, 644 RTLIB::ROUNDEVEN_F128, 645 RTLIB::ROUNDEVEN_PPCF128)); 646 } 647 648 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { 649 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 650 RTLIB::SIN_F32, 651 RTLIB::SIN_F64, 652 RTLIB::SIN_F80, 653 RTLIB::SIN_F128, 654 RTLIB::SIN_PPCF128)); 655 } 656 657 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { 658 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 659 RTLIB::SQRT_F32, 660 RTLIB::SQRT_F64, 661 RTLIB::SQRT_F80, 662 RTLIB::SQRT_F128, 663 RTLIB::SQRT_PPCF128)); 664 } 665 666 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { 667 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 668 RTLIB::SUB_F32, 669 RTLIB::SUB_F64, 670 RTLIB::SUB_F80, 671 RTLIB::SUB_F128, 672 RTLIB::SUB_PPCF128)); 673 } 674 675 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { 676 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 677 RTLIB::TRUNC_F32, 678 RTLIB::TRUNC_F64, 679 RTLIB::TRUNC_F80, 680 RTLIB::TRUNC_F128, 681 RTLIB::TRUNC_PPCF128)); 682 } 683 684 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { 685 LoadSDNode *L = cast<LoadSDNode>(N); 686 EVT VT = N->getValueType(0); 687 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 688 SDLoc dl(N); 689 690 auto MMOFlags = 691 L->getMemOperand()->getFlags() & 692 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 693 SDValue NewL; 694 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 695 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, 696 L->getChain(), L->getBasePtr(), L->getOffset(), 697 L->getPointerInfo(), NVT, L->getOriginalAlign(), 698 MMOFlags, L->getAAInfo()); 699 // Legalized the chain result - switch anything that used the old chain to 700 // use the new one. 701 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 702 return NewL; 703 } 704 705 // Do a non-extending load followed by FP_EXTEND. 706 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(), 707 dl, L->getChain(), L->getBasePtr(), L->getOffset(), 708 L->getPointerInfo(), L->getMemoryVT(), 709 L->getOriginalAlign(), MMOFlags, L->getAAInfo()); 710 // Legalized the chain result - switch anything that used the old chain to 711 // use the new one. 712 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 713 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL); 714 return BitConvertToInteger(ExtendNode); 715 } 716 717 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { 718 SDValue LHS = GetSoftenedFloat(N->getOperand(1)); 719 SDValue RHS = GetSoftenedFloat(N->getOperand(2)); 720 return DAG.getSelect(SDLoc(N), 721 LHS.getValueType(), N->getOperand(0), LHS, RHS); 722 } 723 724 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { 725 SDValue LHS = GetSoftenedFloat(N->getOperand(2)); 726 SDValue RHS = GetSoftenedFloat(N->getOperand(3)); 727 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 728 LHS.getValueType(), N->getOperand(0), 729 N->getOperand(1), LHS, RHS, N->getOperand(4)); 730 } 731 732 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { 733 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 734 N->getValueType(0))); 735 } 736 737 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { 738 SDValue Chain = N->getOperand(0); // Get the chain. 739 SDValue Ptr = N->getOperand(1); // Get the pointer. 740 EVT VT = N->getValueType(0); 741 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 742 SDLoc dl(N); 743 744 SDValue NewVAARG; 745 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), 746 N->getConstantOperandVal(3)); 747 748 // Legalized the chain result - switch anything that used the old chain to 749 // use the new one. 750 if (N != NewVAARG.getValue(1).getNode()) 751 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1)); 752 return NewVAARG; 753 } 754 755 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { 756 bool IsStrict = N->isStrictFPOpcode(); 757 bool Signed = N->getOpcode() == ISD::SINT_TO_FP || 758 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 759 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 760 EVT RVT = N->getValueType(0); 761 EVT NVT = EVT(); 762 SDLoc dl(N); 763 764 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to 765 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly 766 // match. Look for an appropriate libcall. 767 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 768 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 769 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { 770 NVT = (MVT::SimpleValueType)t; 771 // The source needs to big enough to hold the operand. 772 if (NVT.bitsGE(SVT)) 773 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT); 774 } 775 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 776 777 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 778 // Sign/zero extend the argument if the libcall takes a larger type. 779 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 780 NVT, N->getOperand(IsStrict ? 1 : 0)); 781 TargetLowering::MakeLibCallOptions CallOptions; 782 CallOptions.setSExt(Signed); 783 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 784 std::pair<SDValue, SDValue> Tmp = 785 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), 786 Op, CallOptions, dl, Chain); 787 788 if (IsStrict) 789 ReplaceValueWith(SDValue(N, 1), Tmp.second); 790 return Tmp.first; 791 } 792 793 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) { 794 // Expand and soften recursively. 795 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 796 return SDValue(); 797 } 798 799 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) { 800 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 801 return SDValue(); 802 } 803 804 //===----------------------------------------------------------------------===// 805 // Convert Float Operand to Integer 806 //===----------------------------------------------------------------------===// 807 808 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { 809 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG); 810 dbgs() << "\n"); 811 SDValue Res = SDValue(); 812 813 switch (N->getOpcode()) { 814 default: 815 #ifndef NDEBUG 816 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": "; 817 N->dump(&DAG); dbgs() << "\n"; 818 #endif 819 llvm_unreachable("Do not know how to soften this operator's operand!"); 820 821 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; 822 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; 823 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes 824 case ISD::STRICT_FP_ROUND: 825 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; 826 case ISD::STRICT_FP_TO_SINT: 827 case ISD::STRICT_FP_TO_UINT: 828 case ISD::FP_TO_SINT: 829 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; 830 case ISD::FP_TO_SINT_SAT: 831 case ISD::FP_TO_UINT_SAT: 832 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; 833 case ISD::STRICT_LROUND: 834 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; 835 case ISD::STRICT_LLROUND: 836 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; 837 case ISD::STRICT_LRINT: 838 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; 839 case ISD::STRICT_LLRINT: 840 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; 841 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; 842 case ISD::STRICT_FSETCC: 843 case ISD::STRICT_FSETCCS: 844 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; 845 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; 846 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; 847 } 848 849 // If the result is null, the sub-method took care of registering results etc. 850 if (!Res.getNode()) return false; 851 852 // If the result is N, the sub-method updated N in place. Tell the legalizer 853 // core about this to re-analyze. 854 if (Res.getNode() == N) 855 return true; 856 857 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 858 "Invalid operand softening"); 859 860 ReplaceValueWith(SDValue(N, 0), Res); 861 return false; 862 } 863 864 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { 865 SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); 866 867 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 868 } 869 870 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { 871 // We actually deal with the partially-softened FP_TO_FP16 node too, which 872 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. 873 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || 874 N->getOpcode() == ISD::STRICT_FP_ROUND); 875 876 bool IsStrict = N->isStrictFPOpcode(); 877 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 878 EVT SVT = Op.getValueType(); 879 EVT RVT = N->getValueType(0); 880 EVT FloatRVT = N->getOpcode() == ISD::FP_TO_FP16 ? MVT::f16 : RVT; 881 882 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT); 883 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 884 885 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 886 Op = GetSoftenedFloat(Op); 887 TargetLowering::MakeLibCallOptions CallOptions; 888 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 889 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 890 CallOptions, SDLoc(N), 891 Chain); 892 if (IsStrict) { 893 ReplaceValueWith(SDValue(N, 1), Tmp.second); 894 ReplaceValueWith(SDValue(N, 0), Tmp.first); 895 return SDValue(); 896 } 897 return Tmp.first; 898 } 899 900 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 901 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 902 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 903 904 EVT VT = NewLHS.getValueType(); 905 NewLHS = GetSoftenedFloat(NewLHS); 906 NewRHS = GetSoftenedFloat(NewRHS); 907 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 908 N->getOperand(2), N->getOperand(3)); 909 910 // If softenSetCCOperands returned a scalar, we need to compare the result 911 // against zero to select between true and false values. 912 if (!NewRHS.getNode()) { 913 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 914 CCCode = ISD::SETNE; 915 } 916 917 // Update N to have the operands specified. 918 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 919 DAG.getCondCode(CCCode), NewLHS, NewRHS, 920 N->getOperand(4)), 921 0); 922 } 923 924 // Even if the result type is legal, no libcall may exactly match. (e.g. We 925 // don't have FP-i8 conversions) This helper method looks for an appropriate 926 // promoted libcall. 927 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, 928 bool Signed) { 929 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 930 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 931 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 932 ++IntVT) { 933 Promoted = (MVT::SimpleValueType)IntVT; 934 // The type needs to big enough to hold the result. 935 if (Promoted.bitsGE(RetVT)) 936 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted) 937 : RTLIB::getFPTOUINT(SrcVT, Promoted); 938 } 939 return LC; 940 } 941 942 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { 943 bool IsStrict = N->isStrictFPOpcode(); 944 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 945 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 946 947 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 948 EVT SVT = Op.getValueType(); 949 EVT RVT = N->getValueType(0); 950 EVT NVT = EVT(); 951 SDLoc dl(N); 952 953 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to 954 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly 955 // match, eg. we don't have fp -> i8 conversions. 956 // Look for an appropriate libcall. 957 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed); 958 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 959 "Unsupported FP_TO_XINT!"); 960 961 Op = GetSoftenedFloat(Op); 962 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 963 TargetLowering::MakeLibCallOptions CallOptions; 964 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 965 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 966 CallOptions, dl, Chain); 967 968 // Truncate the result if the libcall returns a larger type. 969 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first); 970 971 if (!IsStrict) 972 return Res; 973 974 ReplaceValueWith(SDValue(N, 1), Tmp.second); 975 ReplaceValueWith(SDValue(N, 0), Res); 976 return SDValue(); 977 } 978 979 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { 980 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 981 return Res; 982 } 983 984 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { 985 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 986 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 987 988 EVT VT = NewLHS.getValueType(); 989 NewLHS = GetSoftenedFloat(NewLHS); 990 NewRHS = GetSoftenedFloat(NewRHS); 991 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 992 N->getOperand(0), N->getOperand(1)); 993 994 // If softenSetCCOperands returned a scalar, we need to compare the result 995 // against zero to select between true and false values. 996 if (!NewRHS.getNode()) { 997 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 998 CCCode = ISD::SETNE; 999 } 1000 1001 // Update N to have the operands specified. 1002 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1003 N->getOperand(2), N->getOperand(3), 1004 DAG.getCondCode(CCCode)), 1005 0); 1006 } 1007 1008 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { 1009 bool IsStrict = N->isStrictFPOpcode(); 1010 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); 1011 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1); 1012 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1013 ISD::CondCode CCCode = 1014 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1015 1016 EVT VT = Op0.getValueType(); 1017 SDValue NewLHS = GetSoftenedFloat(Op0); 1018 SDValue NewRHS = GetSoftenedFloat(Op1); 1019 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1, 1020 Chain, N->getOpcode() == ISD::STRICT_FSETCCS); 1021 1022 // Update N to have the operands specified. 1023 if (NewRHS.getNode()) { 1024 if (IsStrict) 1025 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS, 1026 NewRHS, DAG.getCondCode(CCCode)); 1027 else 1028 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1029 DAG.getCondCode(CCCode)), 0); 1030 } 1031 1032 // Otherwise, softenSetCCOperands returned a scalar, use it. 1033 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && 1034 "Unexpected setcc expansion!"); 1035 1036 if (IsStrict) { 1037 ReplaceValueWith(SDValue(N, 0), NewLHS); 1038 ReplaceValueWith(SDValue(N, 1), Chain); 1039 return SDValue(); 1040 } 1041 return NewLHS; 1042 } 1043 1044 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { 1045 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1046 assert(OpNo == 1 && "Can only soften the stored value!"); 1047 StoreSDNode *ST = cast<StoreSDNode>(N); 1048 SDValue Val = ST->getValue(); 1049 SDLoc dl(N); 1050 1051 if (ST->isTruncatingStore()) 1052 // Do an FP_ROUND followed by a non-truncating store. 1053 Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), 1054 Val, DAG.getIntPtrConstant(0, dl))); 1055 else 1056 Val = GetSoftenedFloat(Val); 1057 1058 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(), 1059 ST->getMemOperand()); 1060 } 1061 1062 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { 1063 SDValue LHS = N->getOperand(0); 1064 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 1065 SDLoc dl(N); 1066 1067 EVT LVT = LHS.getValueType(); 1068 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits()); 1069 EVT RVT = RHS.getValueType(); 1070 1071 unsigned LSize = LVT.getSizeInBits(); 1072 unsigned RSize = RVT.getSizeInBits(); 1073 1074 // Shift right or sign-extend it if the two operands have different types. 1075 int SizeDiff = RSize - LSize; 1076 if (SizeDiff > 0) { 1077 RHS = 1078 DAG.getNode(ISD::SRL, dl, RVT, RHS, 1079 DAG.getConstant(SizeDiff, dl, 1080 TLI.getShiftAmountTy(RHS.getValueType(), 1081 DAG.getDataLayout()))); 1082 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS); 1083 } else if (SizeDiff < 0) { 1084 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS); 1085 RHS = 1086 DAG.getNode(ISD::SHL, dl, ILVT, RHS, 1087 DAG.getConstant(-SizeDiff, dl, 1088 TLI.getShiftAmountTy(RHS.getValueType(), 1089 DAG.getDataLayout()))); 1090 } 1091 1092 RHS = DAG.getBitcast(LVT, RHS); 1093 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS); 1094 } 1095 1096 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { 1097 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1098 bool IsStrict = N->isStrictFPOpcode(); 1099 unsigned Offset = IsStrict ? 1 : 0; 1100 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 1101 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1102 TargetLowering::MakeLibCallOptions CallOptions; 1103 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 1104 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 1105 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1106 CallOptions, SDLoc(N), 1107 Chain); 1108 if (IsStrict) { 1109 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1110 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1111 return SDValue(); 1112 } 1113 1114 return Tmp.first; 1115 } 1116 1117 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { 1118 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1119 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1120 RTLIB::LROUND_F32, 1121 RTLIB::LROUND_F64, 1122 RTLIB::LROUND_F80, 1123 RTLIB::LROUND_F128, 1124 RTLIB::LROUND_PPCF128)); 1125 } 1126 1127 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { 1128 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1129 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1130 RTLIB::LLROUND_F32, 1131 RTLIB::LLROUND_F64, 1132 RTLIB::LLROUND_F80, 1133 RTLIB::LLROUND_F128, 1134 RTLIB::LLROUND_PPCF128)); 1135 } 1136 1137 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { 1138 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1139 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1140 RTLIB::LRINT_F32, 1141 RTLIB::LRINT_F64, 1142 RTLIB::LRINT_F80, 1143 RTLIB::LRINT_F128, 1144 RTLIB::LRINT_PPCF128)); 1145 } 1146 1147 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { 1148 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1149 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1150 RTLIB::LLRINT_F32, 1151 RTLIB::LLRINT_F64, 1152 RTLIB::LLRINT_F80, 1153 RTLIB::LLRINT_F128, 1154 RTLIB::LLRINT_PPCF128)); 1155 } 1156 1157 //===----------------------------------------------------------------------===// 1158 // Float Result Expansion 1159 //===----------------------------------------------------------------------===// 1160 1161 /// ExpandFloatResult - This method is called when the specified result of the 1162 /// specified node is found to need expansion. At this point, the node may also 1163 /// have invalid operands or may have other results that need promotion, we just 1164 /// know that (at least) one result needs expansion. 1165 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { 1166 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n"); 1167 SDValue Lo, Hi; 1168 Lo = Hi = SDValue(); 1169 1170 // See if the target wants to custom expand this node. 1171 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1172 return; 1173 1174 switch (N->getOpcode()) { 1175 default: 1176 #ifndef NDEBUG 1177 dbgs() << "ExpandFloatResult #" << ResNo << ": "; 1178 N->dump(&DAG); dbgs() << "\n"; 1179 #endif 1180 llvm_unreachable("Do not know how to expand the result of this operator!"); 1181 1182 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1183 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1184 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1185 1186 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1187 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1188 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1189 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1190 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1191 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1192 1193 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; 1194 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; 1195 case ISD::STRICT_FMINNUM: 1196 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; 1197 case ISD::STRICT_FMAXNUM: 1198 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; 1199 case ISD::STRICT_FADD: 1200 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; 1201 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; 1202 case ISD::STRICT_FCEIL: 1203 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; 1204 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; 1205 case ISD::STRICT_FCOS: 1206 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; 1207 case ISD::STRICT_FDIV: 1208 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; 1209 case ISD::STRICT_FEXP: 1210 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; 1211 case ISD::STRICT_FEXP2: 1212 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; 1213 case ISD::STRICT_FFLOOR: 1214 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; 1215 case ISD::STRICT_FLOG: 1216 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; 1217 case ISD::STRICT_FLOG2: 1218 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; 1219 case ISD::STRICT_FLOG10: 1220 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; 1221 case ISD::STRICT_FMA: 1222 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; 1223 case ISD::STRICT_FMUL: 1224 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; 1225 case ISD::STRICT_FNEARBYINT: 1226 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; 1227 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; 1228 case ISD::STRICT_FP_EXTEND: 1229 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; 1230 case ISD::STRICT_FPOW: 1231 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; 1232 case ISD::STRICT_FPOWI: 1233 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; 1234 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; 1235 case ISD::STRICT_FRINT: 1236 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; 1237 case ISD::STRICT_FROUND: 1238 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; 1239 case ISD::STRICT_FROUNDEVEN: 1240 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; 1241 case ISD::STRICT_FSIN: 1242 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; 1243 case ISD::STRICT_FSQRT: 1244 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; 1245 case ISD::STRICT_FSUB: 1246 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; 1247 case ISD::STRICT_FTRUNC: 1248 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; 1249 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; 1250 case ISD::STRICT_SINT_TO_FP: 1251 case ISD::STRICT_UINT_TO_FP: 1252 case ISD::SINT_TO_FP: 1253 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; 1254 case ISD::STRICT_FREM: 1255 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; 1256 } 1257 1258 // If Lo/Hi is null, the sub-method took care of registering results etc. 1259 if (Lo.getNode()) 1260 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi); 1261 } 1262 1263 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, 1264 SDValue &Hi) { 1265 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1266 assert(NVT.getSizeInBits() == 64 && 1267 "Do not know how to expand this float constant!"); 1268 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); 1269 SDLoc dl(N); 1270 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1271 APInt(64, C.getRawData()[1])), 1272 dl, NVT); 1273 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1274 APInt(64, C.getRawData()[0])), 1275 dl, NVT); 1276 } 1277 1278 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, 1279 SDValue &Lo, SDValue &Hi) { 1280 bool IsStrict = N->isStrictFPOpcode(); 1281 unsigned Offset = IsStrict ? 1 : 0; 1282 SDValue Op = N->getOperand(0 + Offset); 1283 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1284 TargetLowering::MakeLibCallOptions CallOptions; 1285 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1286 Op, CallOptions, SDLoc(N), 1287 Chain); 1288 if (IsStrict) 1289 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1290 GetPairElements(Tmp.first, Lo, Hi); 1291 } 1292 1293 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, 1294 SDValue &Lo, SDValue &Hi) { 1295 bool IsStrict = N->isStrictFPOpcode(); 1296 unsigned Offset = IsStrict ? 1 : 0; 1297 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) }; 1298 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1299 TargetLowering::MakeLibCallOptions CallOptions; 1300 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1301 Ops, CallOptions, SDLoc(N), 1302 Chain); 1303 if (IsStrict) 1304 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1305 GetPairElements(Tmp.first, Lo, Hi); 1306 } 1307 1308 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, 1309 SDValue &Hi) { 1310 assert(N->getValueType(0) == MVT::ppcf128 && 1311 "Logic only correct for ppcf128!"); 1312 SDLoc dl(N); 1313 SDValue Tmp; 1314 GetExpandedFloat(N->getOperand(0), Lo, Tmp); 1315 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp); 1316 // Lo = Hi==fabs(Hi) ? Lo : -Lo; 1317 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo, 1318 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo), 1319 ISD::SETEQ); 1320 } 1321 1322 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, 1323 SDValue &Hi) { 1324 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1325 RTLIB::FMIN_F32, RTLIB::FMIN_F64, 1326 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 1327 RTLIB::FMIN_PPCF128), Lo, Hi); 1328 } 1329 1330 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, 1331 SDValue &Hi) { 1332 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1333 RTLIB::FMAX_F32, RTLIB::FMAX_F64, 1334 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 1335 RTLIB::FMAX_PPCF128), Lo, Hi); 1336 } 1337 1338 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, 1339 SDValue &Hi) { 1340 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1341 RTLIB::ADD_F32, RTLIB::ADD_F64, 1342 RTLIB::ADD_F80, RTLIB::ADD_F128, 1343 RTLIB::ADD_PPCF128), Lo, Hi); 1344 } 1345 1346 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, 1347 SDValue &Hi) { 1348 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32, 1349 RTLIB::CBRT_F64, RTLIB::CBRT_F80, 1350 RTLIB::CBRT_F128, 1351 RTLIB::CBRT_PPCF128), Lo, Hi); 1352 } 1353 1354 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, 1355 SDValue &Lo, SDValue &Hi) { 1356 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1357 RTLIB::CEIL_F32, RTLIB::CEIL_F64, 1358 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 1359 RTLIB::CEIL_PPCF128), Lo, Hi); 1360 } 1361 1362 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, 1363 SDValue &Lo, SDValue &Hi) { 1364 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1365 RTLIB::COPYSIGN_F32, 1366 RTLIB::COPYSIGN_F64, 1367 RTLIB::COPYSIGN_F80, 1368 RTLIB::COPYSIGN_F128, 1369 RTLIB::COPYSIGN_PPCF128), Lo, Hi); 1370 } 1371 1372 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, 1373 SDValue &Lo, SDValue &Hi) { 1374 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1375 RTLIB::COS_F32, RTLIB::COS_F64, 1376 RTLIB::COS_F80, RTLIB::COS_F128, 1377 RTLIB::COS_PPCF128), Lo, Hi); 1378 } 1379 1380 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, 1381 SDValue &Hi) { 1382 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1383 RTLIB::DIV_F32, 1384 RTLIB::DIV_F64, 1385 RTLIB::DIV_F80, 1386 RTLIB::DIV_F128, 1387 RTLIB::DIV_PPCF128), Lo, Hi); 1388 } 1389 1390 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, 1391 SDValue &Lo, SDValue &Hi) { 1392 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1393 RTLIB::EXP_F32, RTLIB::EXP_F64, 1394 RTLIB::EXP_F80, RTLIB::EXP_F128, 1395 RTLIB::EXP_PPCF128), Lo, Hi); 1396 } 1397 1398 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, 1399 SDValue &Lo, SDValue &Hi) { 1400 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1401 RTLIB::EXP2_F32, RTLIB::EXP2_F64, 1402 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 1403 RTLIB::EXP2_PPCF128), Lo, Hi); 1404 } 1405 1406 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, 1407 SDValue &Lo, SDValue &Hi) { 1408 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1409 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 1410 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 1411 RTLIB::FLOOR_PPCF128), Lo, Hi); 1412 } 1413 1414 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, 1415 SDValue &Lo, SDValue &Hi) { 1416 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1417 RTLIB::LOG_F32, RTLIB::LOG_F64, 1418 RTLIB::LOG_F80, RTLIB::LOG_F128, 1419 RTLIB::LOG_PPCF128), Lo, Hi); 1420 } 1421 1422 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, 1423 SDValue &Lo, SDValue &Hi) { 1424 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1425 RTLIB::LOG2_F32, RTLIB::LOG2_F64, 1426 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 1427 RTLIB::LOG2_PPCF128), Lo, Hi); 1428 } 1429 1430 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, 1431 SDValue &Lo, SDValue &Hi) { 1432 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1433 RTLIB::LOG10_F32, RTLIB::LOG10_F64, 1434 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 1435 RTLIB::LOG10_PPCF128), Lo, Hi); 1436 } 1437 1438 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, 1439 SDValue &Hi) { 1440 bool IsStrict = N->isStrictFPOpcode(); 1441 unsigned Offset = IsStrict ? 1 : 0; 1442 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset), 1443 N->getOperand(2 + Offset) }; 1444 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1445 TargetLowering::MakeLibCallOptions CallOptions; 1446 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), 1447 RTLIB::FMA_F32, 1448 RTLIB::FMA_F64, 1449 RTLIB::FMA_F80, 1450 RTLIB::FMA_F128, 1451 RTLIB::FMA_PPCF128), 1452 N->getValueType(0), Ops, CallOptions, 1453 SDLoc(N), Chain); 1454 if (IsStrict) 1455 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1456 GetPairElements(Tmp.first, Lo, Hi); 1457 } 1458 1459 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, 1460 SDValue &Hi) { 1461 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1462 RTLIB::MUL_F32, 1463 RTLIB::MUL_F64, 1464 RTLIB::MUL_F80, 1465 RTLIB::MUL_F128, 1466 RTLIB::MUL_PPCF128), Lo, Hi); 1467 } 1468 1469 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, 1470 SDValue &Lo, SDValue &Hi) { 1471 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1472 RTLIB::NEARBYINT_F32, 1473 RTLIB::NEARBYINT_F64, 1474 RTLIB::NEARBYINT_F80, 1475 RTLIB::NEARBYINT_F128, 1476 RTLIB::NEARBYINT_PPCF128), Lo, Hi); 1477 } 1478 1479 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, 1480 SDValue &Hi) { 1481 SDLoc dl(N); 1482 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1483 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo); 1484 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); 1485 } 1486 1487 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, 1488 SDValue &Hi) { 1489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1490 SDLoc dl(N); 1491 bool IsStrict = N->isStrictFPOpcode(); 1492 1493 SDValue Chain; 1494 if (IsStrict) { 1495 // If the expanded type is the same as the input type, just bypass the node. 1496 if (NVT == N->getOperand(1).getValueType()) { 1497 Hi = N->getOperand(1); 1498 Chain = N->getOperand(0); 1499 } else { 1500 // Other we need to extend. 1501 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, 1502 { N->getOperand(0), N->getOperand(1) }); 1503 Chain = Hi.getValue(1); 1504 } 1505 } else { 1506 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0)); 1507 } 1508 1509 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1510 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1511 1512 if (IsStrict) 1513 ReplaceValueWith(SDValue(N, 1), Chain); 1514 } 1515 1516 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, 1517 SDValue &Lo, SDValue &Hi) { 1518 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1519 RTLIB::POW_F32, RTLIB::POW_F64, 1520 RTLIB::POW_F80, RTLIB::POW_F128, 1521 RTLIB::POW_PPCF128), Lo, Hi); 1522 } 1523 1524 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, 1525 SDValue &Lo, SDValue &Hi) { 1526 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi); 1527 } 1528 1529 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, 1530 SDValue &Lo, SDValue &Hi) { 1531 assert(N->getValueType(0) == MVT::ppcf128 && 1532 "Logic only correct for ppcf128!"); 1533 1534 SDLoc dl(N); 1535 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1536 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo); 1537 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi); 1538 } 1539 1540 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, 1541 SDValue &Lo, SDValue &Hi) { 1542 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1543 RTLIB::REM_F32, RTLIB::REM_F64, 1544 RTLIB::REM_F80, RTLIB::REM_F128, 1545 RTLIB::REM_PPCF128), Lo, Hi); 1546 } 1547 1548 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, 1549 SDValue &Lo, SDValue &Hi) { 1550 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1551 RTLIB::RINT_F32, RTLIB::RINT_F64, 1552 RTLIB::RINT_F80, RTLIB::RINT_F128, 1553 RTLIB::RINT_PPCF128), Lo, Hi); 1554 } 1555 1556 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, 1557 SDValue &Lo, SDValue &Hi) { 1558 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1559 RTLIB::ROUND_F32, 1560 RTLIB::ROUND_F64, 1561 RTLIB::ROUND_F80, 1562 RTLIB::ROUND_F128, 1563 RTLIB::ROUND_PPCF128), Lo, Hi); 1564 } 1565 1566 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, 1567 SDValue &Lo, SDValue &Hi) { 1568 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1569 RTLIB::ROUNDEVEN_F32, 1570 RTLIB::ROUNDEVEN_F64, 1571 RTLIB::ROUNDEVEN_F80, 1572 RTLIB::ROUNDEVEN_F128, 1573 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); 1574 } 1575 1576 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, 1577 SDValue &Lo, SDValue &Hi) { 1578 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1579 RTLIB::SIN_F32, RTLIB::SIN_F64, 1580 RTLIB::SIN_F80, RTLIB::SIN_F128, 1581 RTLIB::SIN_PPCF128), Lo, Hi); 1582 } 1583 1584 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, 1585 SDValue &Lo, SDValue &Hi) { 1586 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1587 RTLIB::SQRT_F32, RTLIB::SQRT_F64, 1588 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 1589 RTLIB::SQRT_PPCF128), Lo, Hi); 1590 } 1591 1592 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, 1593 SDValue &Hi) { 1594 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1595 RTLIB::SUB_F32, 1596 RTLIB::SUB_F64, 1597 RTLIB::SUB_F80, 1598 RTLIB::SUB_F128, 1599 RTLIB::SUB_PPCF128), Lo, Hi); 1600 } 1601 1602 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, 1603 SDValue &Lo, SDValue &Hi) { 1604 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1605 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 1606 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 1607 RTLIB::TRUNC_PPCF128), Lo, Hi); 1608 } 1609 1610 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, 1611 SDValue &Hi) { 1612 if (ISD::isNormalLoad(N)) { 1613 ExpandRes_NormalLoad(N, Lo, Hi); 1614 return; 1615 } 1616 1617 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1618 LoadSDNode *LD = cast<LoadSDNode>(N); 1619 SDValue Chain = LD->getChain(); 1620 SDValue Ptr = LD->getBasePtr(); 1621 SDLoc dl(N); 1622 1623 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1624 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1625 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1626 1627 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr, 1628 LD->getMemoryVT(), LD->getMemOperand()); 1629 1630 // Remember the chain. 1631 Chain = Hi.getValue(1); 1632 1633 // The low part is zero. 1634 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1635 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1636 1637 // Modified the chain - switch anything that used the old chain to use the 1638 // new one. 1639 ReplaceValueWith(SDValue(LD, 1), Chain); 1640 } 1641 1642 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, 1643 SDValue &Hi) { 1644 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!"); 1645 EVT VT = N->getValueType(0); 1646 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1647 bool Strict = N->isStrictFPOpcode(); 1648 SDValue Src = N->getOperand(Strict ? 1 : 0); 1649 EVT SrcVT = Src.getValueType(); 1650 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP || 1651 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 1652 SDLoc dl(N); 1653 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode(); 1654 1655 // TODO: Any other flags to propagate? 1656 SDNodeFlags Flags; 1657 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept()); 1658 1659 // First do an SINT_TO_FP, whether the original was signed or unsigned. 1660 // When promoting partial word types to i32 we must honor the signedness, 1661 // though. 1662 if (SrcVT.bitsLE(MVT::i32)) { 1663 // The integer can be represented exactly in an f64. 1664 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1665 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1666 if (Strict) { 1667 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), 1668 {Chain, Src}, Flags); 1669 Chain = Hi.getValue(1); 1670 } else 1671 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src); 1672 } else { 1673 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1674 if (SrcVT.bitsLE(MVT::i64)) { 1675 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1676 MVT::i64, Src); 1677 LC = RTLIB::SINTTOFP_I64_PPCF128; 1678 } else if (SrcVT.bitsLE(MVT::i128)) { 1679 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); 1680 LC = RTLIB::SINTTOFP_I128_PPCF128; 1681 } 1682 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 1683 1684 TargetLowering::MakeLibCallOptions CallOptions; 1685 CallOptions.setSExt(true); 1686 std::pair<SDValue, SDValue> Tmp = 1687 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); 1688 if (Strict) 1689 Chain = Tmp.second; 1690 GetPairElements(Tmp.first, Lo, Hi); 1691 } 1692 1693 // No need to complement for unsigned 32-bit integers 1694 if (isSigned || SrcVT.bitsLE(MVT::i32)) { 1695 if (Strict) 1696 ReplaceValueWith(SDValue(N, 1), Chain); 1697 1698 return; 1699 } 1700 1701 // Unsigned - fix up the SINT_TO_FP value just calculated. 1702 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully 1703 // keep semantics correctness if the integer is not exactly representable 1704 // here. See ExpandLegalINT_TO_FP. 1705 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 1706 SrcVT = Src.getValueType(); 1707 1708 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. 1709 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 1710 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 1711 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; 1712 ArrayRef<uint64_t> Parts; 1713 1714 switch (SrcVT.getSimpleVT().SimpleTy) { 1715 default: 1716 llvm_unreachable("Unsupported UINT_TO_FP!"); 1717 case MVT::i32: 1718 Parts = TwoE32; 1719 break; 1720 case MVT::i64: 1721 Parts = TwoE64; 1722 break; 1723 case MVT::i128: 1724 Parts = TwoE128; 1725 break; 1726 } 1727 1728 // TODO: Are there other fast-math-flags to propagate to this FADD? 1729 SDValue NewLo = DAG.getConstantFP( 1730 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128); 1731 if (Strict) { 1732 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other), 1733 {Chain, Hi, NewLo}, Flags); 1734 Chain = Lo.getValue(1); 1735 ReplaceValueWith(SDValue(N, 1), Chain); 1736 } else 1737 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo); 1738 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT), 1739 Lo, Hi, ISD::SETLT); 1740 GetPairElements(Lo, Lo, Hi); 1741 } 1742 1743 1744 //===----------------------------------------------------------------------===// 1745 // Float Operand Expansion 1746 //===----------------------------------------------------------------------===// 1747 1748 /// ExpandFloatOperand - This method is called when the specified operand of the 1749 /// specified node is found to need expansion. At this point, all of the result 1750 /// types of the node are known to be legal, but other operands of the node may 1751 /// need promotion or expansion as well as the specified one. 1752 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { 1753 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n"); 1754 SDValue Res = SDValue(); 1755 1756 // See if the target wants to custom expand this node. 1757 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1758 return false; 1759 1760 switch (N->getOpcode()) { 1761 default: 1762 #ifndef NDEBUG 1763 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": "; 1764 N->dump(&DAG); dbgs() << "\n"; 1765 #endif 1766 llvm_unreachable("Do not know how to expand this operator's operand!"); 1767 1768 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 1769 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1770 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1771 1772 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; 1773 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; 1774 case ISD::STRICT_FP_ROUND: 1775 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; 1776 case ISD::STRICT_FP_TO_SINT: 1777 case ISD::STRICT_FP_TO_UINT: 1778 case ISD::FP_TO_SINT: 1779 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break; 1780 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; 1781 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; 1782 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; 1783 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; 1784 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; 1785 case ISD::STRICT_FSETCC: 1786 case ISD::STRICT_FSETCCS: 1787 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; 1788 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N), 1789 OpNo); break; 1790 } 1791 1792 // If the result is null, the sub-method took care of registering results etc. 1793 if (!Res.getNode()) return false; 1794 1795 // If the result is N, the sub-method updated N in place. Tell the legalizer 1796 // core about this. 1797 if (Res.getNode() == N) 1798 return true; 1799 1800 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1801 "Invalid operand expansion"); 1802 1803 ReplaceValueWith(SDValue(N, 0), Res); 1804 return false; 1805 } 1806 1807 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code 1808 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1809 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, 1810 SDValue &NewRHS, 1811 ISD::CondCode &CCCode, 1812 const SDLoc &dl, SDValue &Chain, 1813 bool IsSignaling) { 1814 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 1815 GetExpandedFloat(NewLHS, LHSLo, LHSHi); 1816 GetExpandedFloat(NewRHS, RHSLo, RHSHi); 1817 1818 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); 1819 1820 // FIXME: This generated code sucks. We want to generate 1821 // FCMPU crN, hi1, hi2 1822 // BNE crN, L: 1823 // FCMPU crN, lo1, lo2 1824 // The following can be improved, but not that much. 1825 SDValue Tmp1, Tmp2, Tmp3, OutputChain; 1826 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1827 RHSHi, ISD::SETOEQ, Chain, IsSignaling); 1828 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1829 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 1830 RHSLo, CCCode, OutputChain, IsSignaling); 1831 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1832 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1833 Tmp1 = 1834 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi, 1835 ISD::SETUNE, OutputChain, IsSignaling); 1836 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1837 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1838 RHSHi, CCCode, OutputChain, IsSignaling); 1839 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1840 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1841 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 1842 NewRHS = SDValue(); // LHS is the result, not a compare. 1843 Chain = OutputChain; 1844 } 1845 1846 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { 1847 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1848 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1849 SDValue Chain; 1850 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 1851 1852 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1853 // against zero to select between true and false values. 1854 if (!NewRHS.getNode()) { 1855 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1856 CCCode = ISD::SETNE; 1857 } 1858 1859 // Update N to have the operands specified. 1860 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1861 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1862 N->getOperand(4)), 0); 1863 } 1864 1865 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { 1866 assert(N->getOperand(1).getValueType() == MVT::ppcf128 && 1867 "Logic only correct for ppcf128!"); 1868 SDValue Lo, Hi; 1869 GetExpandedFloat(N->getOperand(1), Lo, Hi); 1870 // The ppcf128 value is providing only the sign; take it from the 1871 // higher-order double (which must have the larger magnitude). 1872 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), 1873 N->getValueType(0), N->getOperand(0), Hi); 1874 } 1875 1876 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { 1877 bool IsStrict = N->isStrictFPOpcode(); 1878 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && 1879 "Logic only correct for ppcf128!"); 1880 SDValue Lo, Hi; 1881 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi); 1882 1883 if (!IsStrict) 1884 // Round it the rest of the way (e.g. to f32) if needed. 1885 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 1886 N->getValueType(0), Hi, N->getOperand(1)); 1887 1888 // Eliminate the node if the input float type is the same as the output float 1889 // type. 1890 if (Hi.getValueType() == N->getValueType(0)) { 1891 // Connect the output chain to the input chain, unlinking the node. 1892 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 1893 ReplaceValueWith(SDValue(N, 0), Hi); 1894 return SDValue(); 1895 } 1896 1897 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 1898 {N->getValueType(0), MVT::Other}, 1899 {N->getOperand(0), Hi, N->getOperand(2)}); 1900 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1)); 1901 ReplaceValueWith(SDValue(N, 0), Expansion); 1902 return SDValue(); 1903 } 1904 1905 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) { 1906 EVT RVT = N->getValueType(0); 1907 SDLoc dl(N); 1908 1909 bool IsStrict = N->isStrictFPOpcode(); 1910 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 1911 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 1912 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1913 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1914 1915 EVT NVT; 1916 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed); 1917 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 1918 "Unsupported FP_TO_XINT!"); 1919 TargetLowering::MakeLibCallOptions CallOptions; 1920 std::pair<SDValue, SDValue> Tmp = 1921 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); 1922 if (!IsStrict) 1923 return Tmp.first; 1924 1925 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1926 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1927 return SDValue(); 1928 } 1929 1930 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { 1931 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1932 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1933 SDValue Chain; 1934 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 1935 1936 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1937 // against zero to select between true and false values. 1938 if (!NewRHS.getNode()) { 1939 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1940 CCCode = ISD::SETNE; 1941 } 1942 1943 // Update N to have the operands specified. 1944 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1945 N->getOperand(2), N->getOperand(3), 1946 DAG.getCondCode(CCCode)), 0); 1947 } 1948 1949 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { 1950 bool IsStrict = N->isStrictFPOpcode(); 1951 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0); 1952 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1); 1953 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1954 ISD::CondCode CCCode = 1955 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1956 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain, 1957 N->getOpcode() == ISD::STRICT_FSETCCS); 1958 1959 // FloatExpandSetCCOperands always returned a scalar. 1960 assert(!NewRHS.getNode() && "Expect to return scalar"); 1961 assert(NewLHS.getValueType() == N->getValueType(0) && 1962 "Unexpected setcc expansion!"); 1963 if (Chain) { 1964 ReplaceValueWith(SDValue(N, 0), NewLHS); 1965 ReplaceValueWith(SDValue(N, 1), Chain); 1966 return SDValue(); 1967 } 1968 return NewLHS; 1969 } 1970 1971 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { 1972 if (ISD::isNormalStore(N)) 1973 return ExpandOp_NormalStore(N, OpNo); 1974 1975 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1976 assert(OpNo == 1 && "Can only expand the stored value so far"); 1977 StoreSDNode *ST = cast<StoreSDNode>(N); 1978 1979 SDValue Chain = ST->getChain(); 1980 SDValue Ptr = ST->getBasePtr(); 1981 1982 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 1983 ST->getValue().getValueType()); 1984 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1985 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1986 (void)NVT; 1987 1988 SDValue Lo, Hi; 1989 GetExpandedOp(ST->getValue(), Lo, Hi); 1990 1991 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr, 1992 ST->getMemoryVT(), ST->getMemOperand()); 1993 } 1994 1995 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { 1996 EVT RVT = N->getValueType(0); 1997 EVT RetVT = N->getOperand(0).getValueType(); 1998 TargetLowering::MakeLibCallOptions CallOptions; 1999 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2000 RTLIB::LROUND_F32, 2001 RTLIB::LROUND_F64, 2002 RTLIB::LROUND_F80, 2003 RTLIB::LROUND_F128, 2004 RTLIB::LROUND_PPCF128), 2005 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2006 } 2007 2008 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { 2009 EVT RVT = N->getValueType(0); 2010 EVT RetVT = N->getOperand(0).getValueType(); 2011 TargetLowering::MakeLibCallOptions CallOptions; 2012 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2013 RTLIB::LLROUND_F32, 2014 RTLIB::LLROUND_F64, 2015 RTLIB::LLROUND_F80, 2016 RTLIB::LLROUND_F128, 2017 RTLIB::LLROUND_PPCF128), 2018 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2019 } 2020 2021 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { 2022 EVT RVT = N->getValueType(0); 2023 EVT RetVT = N->getOperand(0).getValueType(); 2024 TargetLowering::MakeLibCallOptions CallOptions; 2025 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2026 RTLIB::LRINT_F32, 2027 RTLIB::LRINT_F64, 2028 RTLIB::LRINT_F80, 2029 RTLIB::LRINT_F128, 2030 RTLIB::LRINT_PPCF128), 2031 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2032 } 2033 2034 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { 2035 EVT RVT = N->getValueType(0); 2036 EVT RetVT = N->getOperand(0).getValueType(); 2037 TargetLowering::MakeLibCallOptions CallOptions; 2038 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2039 RTLIB::LLRINT_F32, 2040 RTLIB::LLRINT_F64, 2041 RTLIB::LLRINT_F80, 2042 RTLIB::LLRINT_F128, 2043 RTLIB::LLRINT_PPCF128), 2044 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2045 } 2046 2047 //===----------------------------------------------------------------------===// 2048 // Float Operand Promotion 2049 //===----------------------------------------------------------------------===// 2050 // 2051 2052 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { 2053 if (OpVT == MVT::f16) { 2054 return ISD::FP16_TO_FP; 2055 } else if (RetVT == MVT::f16) { 2056 return ISD::FP_TO_FP16; 2057 } 2058 2059 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2060 } 2061 2062 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { 2063 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG); 2064 dbgs() << "\n"); 2065 SDValue R = SDValue(); 2066 2067 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2068 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2069 return false; 2070 } 2071 2072 // Nodes that use a promotion-requiring floating point operand, but doesn't 2073 // produce a promotion-requiring floating point result, need to be legalized 2074 // to use the promoted float operand. Nodes that produce at least one 2075 // promotion-requiring floating point result have their operands legalized as 2076 // a part of PromoteFloatResult. 2077 switch (N->getOpcode()) { 2078 default: 2079 #ifndef NDEBUG 2080 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; 2081 N->dump(&DAG); dbgs() << "\n"; 2082 #endif 2083 llvm_unreachable("Do not know how to promote this operator's operand!"); 2084 2085 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; 2086 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; 2087 case ISD::FP_TO_SINT: 2088 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break; 2089 case ISD::FP_TO_SINT_SAT: 2090 case ISD::FP_TO_UINT_SAT: 2091 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; 2092 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; 2093 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; 2094 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; 2095 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; 2096 } 2097 2098 if (R.getNode()) 2099 ReplaceValueWith(SDValue(N, 0), R); 2100 return false; 2101 } 2102 2103 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { 2104 SDValue Op = N->getOperand(0); 2105 EVT OpVT = Op->getValueType(0); 2106 2107 SDValue Promoted = GetPromotedFloat(N->getOperand(0)); 2108 EVT PromotedVT = Promoted->getValueType(0); 2109 2110 // Convert the promoted float value to the desired IVT. 2111 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); 2112 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), 2113 IVT, Promoted); 2114 // The final result type might not be an scalar so we need a bitcast. The 2115 // bitcast will be further legalized if needed. 2116 return DAG.getBitcast(N->getValueType(0), Convert); 2117 } 2118 2119 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by 2120 // PromoteFloatRes_FCOPYSIGN. 2121 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { 2122 assert (OpNo == 1 && "Only Operand 1 must need promotion here"); 2123 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2124 2125 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2126 N->getOperand(0), Op1); 2127 } 2128 2129 // Convert the promoted float value to the desired integer type 2130 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) { 2131 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2132 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); 2133 } 2134 2135 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, 2136 unsigned OpNo) { 2137 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2138 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, 2139 N->getOperand(1)); 2140 } 2141 2142 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { 2143 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2144 EVT VT = N->getValueType(0); 2145 2146 // Desired VT is same as promoted type. Use promoted float directly. 2147 if (VT == Op->getValueType(0)) 2148 return Op; 2149 2150 // Else, extend the promoted float value to the desired VT. 2151 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); 2152 } 2153 2154 // Promote the float operands used for comparison. The true- and false- 2155 // operands have the same type as the result and are promoted, if needed, by 2156 // PromoteFloatRes_SELECT_CC 2157 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2158 SDValue LHS = GetPromotedFloat(N->getOperand(0)); 2159 SDValue RHS = GetPromotedFloat(N->getOperand(1)); 2160 2161 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 2162 LHS, RHS, N->getOperand(2), N->getOperand(3), 2163 N->getOperand(4)); 2164 } 2165 2166 // Construct a SETCC that compares the promoted values and sets the conditional 2167 // code. 2168 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { 2169 EVT VT = N->getValueType(0); 2170 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2171 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2172 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2173 2174 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); 2175 2176 } 2177 2178 // Lower the promoted Float down to the integer value of same size and construct 2179 // a STORE of the integer value. 2180 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { 2181 StoreSDNode *ST = cast<StoreSDNode>(N); 2182 SDValue Val = ST->getValue(); 2183 SDLoc DL(N); 2184 2185 SDValue Promoted = GetPromotedFloat(Val); 2186 EVT VT = ST->getOperand(1).getValueType(); 2187 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2188 2189 SDValue NewVal; 2190 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, 2191 IVT, Promoted); 2192 2193 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), 2194 ST->getMemOperand()); 2195 } 2196 2197 //===----------------------------------------------------------------------===// 2198 // Float Result Promotion 2199 //===----------------------------------------------------------------------===// 2200 2201 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { 2202 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG); 2203 dbgs() << "\n"); 2204 SDValue R = SDValue(); 2205 2206 // See if the target wants to custom expand this node. 2207 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2208 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2209 return; 2210 } 2211 2212 switch (N->getOpcode()) { 2213 // These opcodes cannot appear if promotion of FP16 is done in the backend 2214 // instead of Clang 2215 case ISD::FP16_TO_FP: 2216 case ISD::FP_TO_FP16: 2217 default: 2218 #ifndef NDEBUG 2219 dbgs() << "PromoteFloatResult #" << ResNo << ": "; 2220 N->dump(&DAG); dbgs() << "\n"; 2221 #endif 2222 llvm_unreachable("Do not know how to promote this operator's result!"); 2223 2224 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; 2225 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; 2226 case ISD::EXTRACT_VECTOR_ELT: 2227 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; 2228 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; 2229 2230 // Unary FP Operations 2231 case ISD::FABS: 2232 case ISD::FCBRT: 2233 case ISD::FCEIL: 2234 case ISD::FCOS: 2235 case ISD::FEXP: 2236 case ISD::FEXP2: 2237 case ISD::FFLOOR: 2238 case ISD::FLOG: 2239 case ISD::FLOG2: 2240 case ISD::FLOG10: 2241 case ISD::FNEARBYINT: 2242 case ISD::FNEG: 2243 case ISD::FRINT: 2244 case ISD::FROUND: 2245 case ISD::FROUNDEVEN: 2246 case ISD::FSIN: 2247 case ISD::FSQRT: 2248 case ISD::FTRUNC: 2249 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; 2250 2251 // Binary FP Operations 2252 case ISD::FADD: 2253 case ISD::FDIV: 2254 case ISD::FMAXIMUM: 2255 case ISD::FMINIMUM: 2256 case ISD::FMAXNUM: 2257 case ISD::FMINNUM: 2258 case ISD::FMUL: 2259 case ISD::FPOW: 2260 case ISD::FREM: 2261 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; 2262 2263 case ISD::FMA: // FMA is same as FMAD 2264 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; 2265 2266 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break; 2267 2268 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; 2269 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; 2270 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; 2271 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; 2272 2273 case ISD::SINT_TO_FP: 2274 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; 2275 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; 2276 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2277 case ISD::VECREDUCE_FADD: 2278 case ISD::VECREDUCE_FMUL: 2279 case ISD::VECREDUCE_FMIN: 2280 case ISD::VECREDUCE_FMAX: 2281 R = PromoteFloatRes_VECREDUCE(N); 2282 break; 2283 case ISD::VECREDUCE_SEQ_FADD: 2284 case ISD::VECREDUCE_SEQ_FMUL: 2285 R = PromoteFloatRes_VECREDUCE_SEQ(N); 2286 break; 2287 } 2288 2289 if (R.getNode()) 2290 SetPromotedFloat(SDValue(N, ResNo), R); 2291 } 2292 2293 // Bitcast from i16 to f16: convert the i16 to a f32 value instead. 2294 // At this point, it is not possible to determine if the bitcast value is 2295 // eventually stored to memory or promoted to f32 or promoted to a floating 2296 // point at a higher precision. Some of these cases are handled by FP_EXTEND, 2297 // STORE promotion handlers. 2298 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { 2299 EVT VT = N->getValueType(0); 2300 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2301 // Input type isn't guaranteed to be a scalar int so bitcast if not. The 2302 // bitcast will be legalized further if necessary. 2303 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), 2304 N->getOperand(0).getValueType().getSizeInBits()); 2305 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); 2306 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); 2307 } 2308 2309 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { 2310 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); 2311 EVT VT = N->getValueType(0); 2312 SDLoc DL(N); 2313 2314 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2315 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2316 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, 2317 IVT); 2318 2319 // Convert the Constant to the desired FP type 2320 // FIXME We might be able to do the conversion during compilation and get rid 2321 // of it from the object code 2322 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2323 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); 2324 } 2325 2326 // If the Index operand is a constant, try to redirect the extract operation to 2327 // the correct legalized vector. If not, bit-convert the input vector to 2328 // equivalent integer vector. Extract the element as an (bit-cast) integer 2329 // value and convert it to the promoted type. 2330 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2331 SDLoc DL(N); 2332 2333 // If the index is constant, try to extract the value from the legalized 2334 // vector type. 2335 if (isa<ConstantSDNode>(N->getOperand(1))) { 2336 SDValue Vec = N->getOperand(0); 2337 SDValue Idx = N->getOperand(1); 2338 EVT VecVT = Vec->getValueType(0); 2339 EVT EltVT = VecVT.getVectorElementType(); 2340 2341 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2342 2343 switch (getTypeAction(VecVT)) { 2344 default: break; 2345 case TargetLowering::TypeScalarizeVector: { 2346 SDValue Res = GetScalarizedVector(N->getOperand(0)); 2347 ReplaceValueWith(SDValue(N, 0), Res); 2348 return SDValue(); 2349 } 2350 case TargetLowering::TypeWidenVector: { 2351 Vec = GetWidenedVector(Vec); 2352 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); 2353 ReplaceValueWith(SDValue(N, 0), Res); 2354 return SDValue(); 2355 } 2356 case TargetLowering::TypeSplitVector: { 2357 SDValue Lo, Hi; 2358 GetSplitVector(Vec, Lo, Hi); 2359 2360 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2361 SDValue Res; 2362 if (IdxVal < LoElts) 2363 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); 2364 else 2365 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, 2366 DAG.getConstant(IdxVal - LoElts, DL, 2367 Idx.getValueType())); 2368 ReplaceValueWith(SDValue(N, 0), Res); 2369 return SDValue(); 2370 } 2371 2372 } 2373 } 2374 2375 // Bit-convert the input vector to the equivalent integer vector 2376 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2377 EVT IVT = NewOp.getValueType().getVectorElementType(); 2378 2379 // Extract the element as an (bit-cast) integer value 2380 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, 2381 NewOp, N->getOperand(1)); 2382 2383 // Convert the element to the desired FP type 2384 EVT VT = N->getValueType(0); 2385 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2386 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); 2387 } 2388 2389 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result 2390 // needs promotion, so does the argument X. Note that Y, if needed, will be 2391 // handled during operand promotion. 2392 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { 2393 EVT VT = N->getValueType(0); 2394 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2395 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2396 2397 SDValue Op1 = N->getOperand(1); 2398 2399 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2400 } 2401 2402 // Unary operation where the result and the operand have PromoteFloat type 2403 // action. Construct a new SDNode with the promoted float value of the old 2404 // operand. 2405 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { 2406 EVT VT = N->getValueType(0); 2407 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2408 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2409 2410 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); 2411 } 2412 2413 // Binary operations where the result and both operands have PromoteFloat type 2414 // action. Construct a new SDNode with the promoted float values of the old 2415 // operands. 2416 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { 2417 EVT VT = N->getValueType(0); 2418 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2419 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2420 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2421 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); 2422 } 2423 2424 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { 2425 EVT VT = N->getValueType(0); 2426 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2427 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2428 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2429 SDValue Op2 = GetPromotedFloat(N->getOperand(2)); 2430 2431 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); 2432 } 2433 2434 // Promote the Float (first) operand and retain the Integer (second) operand 2435 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) { 2436 EVT VT = N->getValueType(0); 2437 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2438 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2439 SDValue Op1 = N->getOperand(1); 2440 2441 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2442 } 2443 2444 // Explicit operation to reduce precision. Reduce the value to half precision 2445 // and promote it back to the legal type. 2446 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { 2447 SDLoc DL(N); 2448 2449 SDValue Op = N->getOperand(0); 2450 EVT VT = N->getValueType(0); 2451 EVT OpVT = Op->getValueType(0); 2452 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2453 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2454 2455 // Round promoted float to desired precision 2456 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); 2457 // Promote it back to the legal output type 2458 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); 2459 } 2460 2461 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { 2462 LoadSDNode *L = cast<LoadSDNode>(N); 2463 EVT VT = N->getValueType(0); 2464 2465 // Load the value as an integer value with the same number of bits. 2466 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2467 SDValue newL = DAG.getLoad( 2468 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), 2469 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, 2470 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); 2471 // Legalize the chain result by replacing uses of the old value chain with the 2472 // new one 2473 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 2474 2475 // Convert the integer value to the desired FP type 2476 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2477 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); 2478 } 2479 2480 // Construct a new SELECT node with the promoted true- and false- values. 2481 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { 2482 SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); 2483 SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); 2484 2485 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), 2486 N->getOperand(0), TrueVal, FalseVal); 2487 } 2488 2489 // Construct a new SELECT_CC node with the promoted true- and false- values. 2490 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. 2491 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { 2492 SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); 2493 SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); 2494 2495 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2496 TrueVal.getNode()->getValueType(0), N->getOperand(0), 2497 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); 2498 } 2499 2500 // Construct a SDNode that transforms the SINT or UINT operand to the promoted 2501 // float type. 2502 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { 2503 SDLoc DL(N); 2504 EVT VT = N->getValueType(0); 2505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2506 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); 2507 // Round the value to the desired precision (that of the source type). 2508 return DAG.getNode( 2509 ISD::FP_EXTEND, DL, NVT, 2510 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL))); 2511 } 2512 2513 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { 2514 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 2515 N->getValueType(0))); 2516 } 2517 2518 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { 2519 // Expand and promote recursively. 2520 // TODO: This is non-optimal, but dealing with the concurrently happening 2521 // vector-legalization is non-trivial. We could do something similar to 2522 // PromoteFloatRes_EXTRACT_VECTOR_ELT here. 2523 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2524 return SDValue(); 2525 } 2526 2527 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { 2528 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2529 return SDValue(); 2530 } 2531 2532 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { 2533 EVT VT = N->getValueType(0); 2534 2535 AtomicSDNode *AM = cast<AtomicSDNode>(N); 2536 SDLoc SL(N); 2537 2538 SDValue CastVal = BitConvertToInteger(AM->getVal()); 2539 EVT CastVT = CastVal.getValueType(); 2540 2541 SDValue NewAtomic 2542 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, 2543 DAG.getVTList(CastVT, MVT::Other), 2544 { AM->getChain(), AM->getBasePtr(), CastVal }, 2545 AM->getMemOperand()); 2546 2547 SDValue Result = NewAtomic; 2548 2549 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { 2550 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2551 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, 2552 NewAtomic); 2553 } 2554 2555 // Legalize the chain result by replacing uses of the old value chain with the 2556 // new one 2557 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); 2558 2559 return Result; 2560 2561 } 2562 2563 //===----------------------------------------------------------------------===// 2564 // Half Result Soft Promotion 2565 //===----------------------------------------------------------------------===// 2566 2567 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { 2568 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": "; 2569 N->dump(&DAG); dbgs() << "\n"); 2570 SDValue R = SDValue(); 2571 2572 // See if the target wants to custom expand this node. 2573 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2574 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2575 return; 2576 } 2577 2578 switch (N->getOpcode()) { 2579 default: 2580 #ifndef NDEBUG 2581 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": "; 2582 N->dump(&DAG); dbgs() << "\n"; 2583 #endif 2584 llvm_unreachable("Do not know how to soft promote this operator's result!"); 2585 2586 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; 2587 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; 2588 case ISD::EXTRACT_VECTOR_ELT: 2589 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; 2590 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; 2591 case ISD::STRICT_FP_ROUND: 2592 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; 2593 2594 // Unary FP Operations 2595 case ISD::FABS: 2596 case ISD::FCBRT: 2597 case ISD::FCEIL: 2598 case ISD::FCOS: 2599 case ISD::FEXP: 2600 case ISD::FEXP2: 2601 case ISD::FFLOOR: 2602 case ISD::FLOG: 2603 case ISD::FLOG2: 2604 case ISD::FLOG10: 2605 case ISD::FNEARBYINT: 2606 case ISD::FNEG: 2607 case ISD::FREEZE: 2608 case ISD::FRINT: 2609 case ISD::FROUND: 2610 case ISD::FROUNDEVEN: 2611 case ISD::FSIN: 2612 case ISD::FSQRT: 2613 case ISD::FTRUNC: 2614 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; 2615 2616 // Binary FP Operations 2617 case ISD::FADD: 2618 case ISD::FDIV: 2619 case ISD::FMAXIMUM: 2620 case ISD::FMINIMUM: 2621 case ISD::FMAXNUM: 2622 case ISD::FMINNUM: 2623 case ISD::FMUL: 2624 case ISD::FPOW: 2625 case ISD::FREM: 2626 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; 2627 2628 case ISD::FMA: // FMA is same as FMAD 2629 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; 2630 2631 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break; 2632 2633 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; 2634 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; 2635 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; 2636 case ISD::SINT_TO_FP: 2637 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; 2638 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; 2639 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2640 case ISD::VECREDUCE_FADD: 2641 case ISD::VECREDUCE_FMUL: 2642 case ISD::VECREDUCE_FMIN: 2643 case ISD::VECREDUCE_FMAX: 2644 R = SoftPromoteHalfRes_VECREDUCE(N); 2645 break; 2646 case ISD::VECREDUCE_SEQ_FADD: 2647 case ISD::VECREDUCE_SEQ_FMUL: 2648 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N); 2649 break; 2650 } 2651 2652 if (R.getNode()) 2653 SetSoftPromotedHalf(SDValue(N, ResNo), R); 2654 } 2655 2656 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { 2657 return BitConvertToInteger(N->getOperand(0)); 2658 } 2659 2660 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { 2661 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 2662 2663 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2664 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 2665 MVT::i16); 2666 } 2667 2668 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2669 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2670 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2671 NewOp.getValueType().getVectorElementType(), NewOp, 2672 N->getOperand(1)); 2673 } 2674 2675 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { 2676 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0)); 2677 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 2678 SDLoc dl(N); 2679 2680 EVT LVT = LHS.getValueType(); 2681 EVT RVT = RHS.getValueType(); 2682 2683 unsigned LSize = LVT.getSizeInBits(); 2684 unsigned RSize = RVT.getSizeInBits(); 2685 2686 // First get the sign bit of second operand. 2687 SDValue SignBit = DAG.getNode( 2688 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 2689 DAG.getConstant(RSize - 1, dl, 2690 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 2691 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 2692 2693 // Shift right or sign-extend it if the two operands have different types. 2694 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 2695 if (SizeDiff > 0) { 2696 SignBit = 2697 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 2698 DAG.getConstant(SizeDiff, dl, 2699 TLI.getShiftAmountTy(SignBit.getValueType(), 2700 DAG.getDataLayout()))); 2701 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 2702 } else if (SizeDiff < 0) { 2703 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 2704 SignBit = 2705 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 2706 DAG.getConstant(-SizeDiff, dl, 2707 TLI.getShiftAmountTy(SignBit.getValueType(), 2708 DAG.getDataLayout()))); 2709 } 2710 2711 // Clear the sign bit of the first operand. 2712 SDValue Mask = DAG.getNode( 2713 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 2714 DAG.getConstant(LSize - 1, dl, 2715 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 2716 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 2717 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 2718 2719 // Or the value with the sign bit. 2720 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 2721 } 2722 2723 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { 2724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2725 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2726 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2727 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2728 SDLoc dl(N); 2729 2730 // Promote to the larger FP type. 2731 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2732 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2733 Op2 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op2); 2734 2735 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2); 2736 2737 // Convert back to FP16 as an integer. 2738 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2739 } 2740 2741 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) { 2742 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2743 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2744 SDValue Op1 = N->getOperand(1); 2745 SDLoc dl(N); 2746 2747 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2748 2749 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2750 2751 // Convert back to FP16 as an integer. 2752 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2753 } 2754 2755 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { 2756 if (N->isStrictFPOpcode()) { 2757 SDValue Res = 2758 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other}, 2759 {N->getOperand(0), N->getOperand(1)}); 2760 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2761 return Res; 2762 } 2763 2764 return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), MVT::i16, N->getOperand(0)); 2765 } 2766 2767 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { 2768 LoadSDNode *L = cast<LoadSDNode>(N); 2769 2770 // Load the value as an integer value with the same number of bits. 2771 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!"); 2772 SDValue NewL = 2773 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, 2774 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), 2775 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), 2776 L->getMemOperand()->getFlags(), L->getAAInfo()); 2777 // Legalize the chain result by replacing uses of the old value chain with the 2778 // new one 2779 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 2780 return NewL; 2781 } 2782 2783 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { 2784 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2785 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2786 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1, 2787 Op2); 2788 } 2789 2790 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { 2791 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2792 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3)); 2793 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(), 2794 N->getOperand(0), N->getOperand(1), Op2, Op3, 2795 N->getOperand(4)); 2796 } 2797 2798 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { 2799 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2800 SDLoc dl(N); 2801 2802 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 2803 2804 // Round the value to the softened type. 2805 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2806 } 2807 2808 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { 2809 return DAG.getUNDEF(MVT::i16); 2810 } 2811 2812 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { 2813 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2814 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 2815 SDLoc dl(N); 2816 2817 // Promote to the larger FP type. 2818 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); 2819 2820 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op); 2821 2822 // Convert back to FP16 as an integer. 2823 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2824 } 2825 2826 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { 2827 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2828 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2829 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2830 SDLoc dl(N); 2831 2832 // Promote to the larger FP type. 2833 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2834 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2835 2836 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2837 2838 // Convert back to FP16 as an integer. 2839 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res); 2840 } 2841 2842 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) { 2843 // Expand and soften recursively. 2844 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2845 return SDValue(); 2846 } 2847 2848 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) { 2849 // Expand and soften. 2850 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2851 return SDValue(); 2852 } 2853 2854 //===----------------------------------------------------------------------===// 2855 // Half Operand Soft Promotion 2856 //===----------------------------------------------------------------------===// 2857 2858 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { 2859 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": "; 2860 N->dump(&DAG); dbgs() << "\n"); 2861 SDValue Res = SDValue(); 2862 2863 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2864 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2865 return false; 2866 } 2867 2868 // Nodes that use a promotion-requiring floating point operand, but doesn't 2869 // produce a soft promotion-requiring floating point result, need to be 2870 // legalized to use the soft promoted float operand. Nodes that produce at 2871 // least one soft promotion-requiring floating point result have their 2872 // operands legalized as a part of PromoteFloatResult. 2873 switch (N->getOpcode()) { 2874 default: 2875 #ifndef NDEBUG 2876 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": "; 2877 N->dump(&DAG); dbgs() << "\n"; 2878 #endif 2879 llvm_unreachable("Do not know how to soft promote this operator's operand!"); 2880 2881 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; 2882 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; 2883 case ISD::FP_TO_SINT: 2884 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; 2885 case ISD::FP_TO_SINT_SAT: 2886 case ISD::FP_TO_UINT_SAT: 2887 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; 2888 case ISD::STRICT_FP_EXTEND: 2889 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; 2890 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; 2891 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; 2892 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; 2893 } 2894 2895 if (!Res.getNode()) 2896 return false; 2897 2898 assert(Res.getNode() != N && "Expected a new node!"); 2899 2900 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2901 "Invalid operand expansion"); 2902 2903 ReplaceValueWith(SDValue(N, 0), Res); 2904 return false; 2905 } 2906 2907 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { 2908 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2909 2910 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 2911 } 2912 2913 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, 2914 unsigned OpNo) { 2915 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 2916 SDValue Op1 = N->getOperand(1); 2917 SDLoc dl(N); 2918 2919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType()); 2920 2921 Op1 = GetSoftPromotedHalf(Op1); 2922 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2923 2924 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0), 2925 Op1); 2926 } 2927 2928 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { 2929 bool IsStrict = N->isStrictFPOpcode(); 2930 SDValue Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0)); 2931 2932 if (IsStrict) { 2933 SDValue Res = 2934 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N), 2935 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op}); 2936 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2937 ReplaceValueWith(SDValue(N, 0), Res); 2938 return SDValue(); 2939 } 2940 2941 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), Op); 2942 } 2943 2944 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { 2945 SDValue Op = N->getOperand(0); 2946 SDLoc dl(N); 2947 2948 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 2949 2950 Op = GetSoftPromotedHalf(Op); 2951 2952 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); 2953 2954 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res); 2955 } 2956 2957 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { 2958 SDValue Op = N->getOperand(0); 2959 SDLoc dl(N); 2960 2961 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 2962 2963 Op = GetSoftPromotedHalf(Op); 2964 2965 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op); 2966 2967 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res, 2968 N->getOperand(1)); 2969 } 2970 2971 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, 2972 unsigned OpNo) { 2973 assert(OpNo == 0 && "Can only soften the comparison values"); 2974 SDValue Op0 = N->getOperand(0); 2975 SDValue Op1 = N->getOperand(1); 2976 SDLoc dl(N); 2977 2978 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 2979 2980 Op0 = GetSoftPromotedHalf(Op0); 2981 Op1 = GetSoftPromotedHalf(Op1); 2982 2983 // Promote to the larger FP type. 2984 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 2985 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 2986 2987 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1, 2988 N->getOperand(2), N->getOperand(3), N->getOperand(4)); 2989 } 2990 2991 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { 2992 SDValue Op0 = N->getOperand(0); 2993 SDValue Op1 = N->getOperand(1); 2994 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2995 SDLoc dl(N); 2996 2997 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 2998 2999 Op0 = GetSoftPromotedHalf(Op0); 3000 Op1 = GetSoftPromotedHalf(Op1); 3001 3002 // Promote to the larger FP type. 3003 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0); 3004 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1); 3005 3006 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode); 3007 } 3008 3009 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { 3010 assert(OpNo == 1 && "Can only soften the stored value!"); 3011 StoreSDNode *ST = cast<StoreSDNode>(N); 3012 SDValue Val = ST->getValue(); 3013 SDLoc dl(N); 3014 3015 assert(!ST->isTruncatingStore() && "Unexpected truncating store."); 3016 SDValue Promoted = GetSoftPromotedHalf(Val); 3017 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), 3018 ST->getMemOperand()); 3019 } 3020