1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements float type expansion and softening for LegalizeTypes. 10 // Softening is the act of turning a computation in an illegal floating point 11 // type into a computation in an integer type of the same size; also known as 12 // "soft float". For example, turning f32 arithmetic into operations using i32. 13 // The resulting integer value is the same as what you would get by performing 14 // the floating point operation and bitcasting the result to the integer type. 15 // Expansion is the act of changing a computation in an illegal type to be a 16 // computation in two identical registers of a smaller type. For example, 17 // implementing ppcf128 arithmetic in two f64 registers. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/Analysis/TargetLibraryInfo.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 /// GetFPLibCall - Return the right libcall for the given floating point type. 30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be 31 /// refactored away (see RTLIB::getPOWI for an example). 32 static RTLIB::Libcall GetFPLibCall(EVT VT, 33 RTLIB::Libcall Call_F32, 34 RTLIB::Libcall Call_F64, 35 RTLIB::Libcall Call_F80, 36 RTLIB::Libcall Call_F128, 37 RTLIB::Libcall Call_PPCF128) { 38 return 39 VT == MVT::f32 ? Call_F32 : 40 VT == MVT::f64 ? Call_F64 : 41 VT == MVT::f80 ? Call_F80 : 42 VT == MVT::f128 ? Call_F128 : 43 VT == MVT::ppcf128 ? Call_PPCF128 : 44 RTLIB::UNKNOWN_LIBCALL; 45 } 46 47 //===----------------------------------------------------------------------===// 48 // Convert Float Results to Integer 49 //===----------------------------------------------------------------------===// 50 51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { 52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG); 53 dbgs() << "\n"); 54 SDValue R = SDValue(); 55 56 switch (N->getOpcode()) { 57 default: 58 #ifndef NDEBUG 59 dbgs() << "SoftenFloatResult #" << ResNo << ": "; 60 N->dump(&DAG); dbgs() << "\n"; 61 #endif 62 llvm_unreachable("Do not know how to soften the result of this operator!"); 63 64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break; 65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; 66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; 67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; 68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; 69 case ISD::EXTRACT_VECTOR_ELT: 70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; 71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break; 72 case ISD::STRICT_FMINNUM: 73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; 74 case ISD::STRICT_FMAXNUM: 75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; 76 case ISD::STRICT_FADD: 77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break; 78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; 79 case ISD::STRICT_FCEIL: 80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; 81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; 82 case ISD::STRICT_FCOS: 83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; 84 case ISD::STRICT_FDIV: 85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; 86 case ISD::STRICT_FEXP: 87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; 88 case ISD::STRICT_FEXP2: 89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; 90 case ISD::STRICT_FFLOOR: 91 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; 92 case ISD::STRICT_FLOG: 93 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; 94 case ISD::STRICT_FLOG2: 95 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; 96 case ISD::STRICT_FLOG10: 97 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; 98 case ISD::STRICT_FMA: 99 case ISD::FMA: R = SoftenFloatRes_FMA(N); break; 100 case ISD::STRICT_FMUL: 101 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; 102 case ISD::STRICT_FNEARBYINT: 103 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; 104 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; 105 case ISD::STRICT_FP_EXTEND: 106 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; 107 case ISD::STRICT_FP_ROUND: 108 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; 109 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; 110 case ISD::STRICT_FPOW: 111 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; 112 case ISD::STRICT_FPOWI: 113 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break; 114 case ISD::STRICT_FREM: 115 case ISD::FREM: R = SoftenFloatRes_FREM(N); break; 116 case ISD::STRICT_FRINT: 117 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; 118 case ISD::STRICT_FROUND: 119 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; 120 case ISD::STRICT_FROUNDEVEN: 121 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; 122 case ISD::STRICT_FSIN: 123 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; 124 case ISD::STRICT_FSQRT: 125 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; 126 case ISD::STRICT_FSUB: 127 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; 128 case ISD::STRICT_FTRUNC: 129 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; 130 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; 131 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 132 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; 133 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; 134 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; 135 case ISD::STRICT_SINT_TO_FP: 136 case ISD::STRICT_UINT_TO_FP: 137 case ISD::SINT_TO_FP: 138 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; 139 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; 140 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; 141 case ISD::VECREDUCE_FADD: 142 case ISD::VECREDUCE_FMUL: 143 case ISD::VECREDUCE_FMIN: 144 case ISD::VECREDUCE_FMAX: 145 R = SoftenFloatRes_VECREDUCE(N); 146 break; 147 case ISD::VECREDUCE_SEQ_FADD: 148 case ISD::VECREDUCE_SEQ_FMUL: 149 R = SoftenFloatRes_VECREDUCE_SEQ(N); 150 break; 151 } 152 153 // If R is null, the sub-method took care of registering the result. 154 if (R.getNode()) { 155 assert(R.getNode() != N); 156 SetSoftenedFloat(SDValue(N, ResNo), R); 157 } 158 } 159 160 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { 161 bool IsStrict = N->isStrictFPOpcode(); 162 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 163 unsigned Offset = IsStrict ? 1 : 0; 164 assert(N->getNumOperands() == (1 + Offset) && 165 "Unexpected number of operands!"); 166 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 167 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 168 TargetLowering::MakeLibCallOptions CallOptions; 169 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 170 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 171 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 172 CallOptions, SDLoc(N), 173 Chain); 174 if (IsStrict) 175 ReplaceValueWith(SDValue(N, 1), Tmp.second); 176 return Tmp.first; 177 } 178 179 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { 180 bool IsStrict = N->isStrictFPOpcode(); 181 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 182 unsigned Offset = IsStrict ? 1 : 0; 183 assert(N->getNumOperands() == (2 + Offset) && 184 "Unexpected number of operands!"); 185 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 186 GetSoftenedFloat(N->getOperand(1 + Offset)) }; 187 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 188 TargetLowering::MakeLibCallOptions CallOptions; 189 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 190 N->getOperand(1 + Offset).getValueType() }; 191 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 192 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 193 CallOptions, SDLoc(N), 194 Chain); 195 if (IsStrict) 196 ReplaceValueWith(SDValue(N, 1), Tmp.second); 197 return Tmp.first; 198 } 199 200 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { 201 return BitConvertToInteger(N->getOperand(0)); 202 } 203 204 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { 205 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 206 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty, 207 GetSoftenedFloat(N->getOperand(0))); 208 } 209 210 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) { 211 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 212 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty, 213 GetSoftenedFloat(N->getOperand(0))); 214 return NewFence; 215 } 216 217 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, 218 unsigned ResNo) { 219 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 220 return BitConvertToInteger(Op); 221 } 222 223 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { 224 // Convert the inputs to integers, and build a new pair out of them. 225 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), 226 TLI.getTypeToTransformTo(*DAG.getContext(), 227 N->getValueType(0)), 228 BitConvertToInteger(N->getOperand(0)), 229 BitConvertToInteger(N->getOperand(1))); 230 } 231 232 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { 233 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 234 // In ppcf128, the high 64 bits are always first in memory regardless 235 // of Endianness. LLVM's APFloat representation is not Endian sensitive, 236 // and so always converts into a 128-bit APInt in a non-Endian-sensitive 237 // way. However, APInt's are serialized in an Endian-sensitive fashion, 238 // so on big-Endian targets, the two doubles are output in the wrong 239 // order. Fix this by manually flipping the order of the high 64 bits 240 // and the low 64 bits here. 241 if (DAG.getDataLayout().isBigEndian() && 242 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) { 243 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], 244 CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; 245 APInt Val(128, words); 246 return DAG.getConstant(Val, SDLoc(CN), 247 TLI.getTypeToTransformTo(*DAG.getContext(), 248 CN->getValueType(0))); 249 } else { 250 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 251 TLI.getTypeToTransformTo(*DAG.getContext(), 252 CN->getValueType(0))); 253 } 254 } 255 256 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) { 257 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 258 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 259 NewOp.getValueType().getVectorElementType(), 260 NewOp, N->getOperand(1)); 261 } 262 263 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { 264 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 265 unsigned Size = NVT.getSizeInBits(); 266 267 // Mask = ~(1 << (Size-1)) 268 APInt API = APInt::getAllOnes(Size); 269 API.clearBit(Size - 1); 270 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT); 271 SDValue Op = GetSoftenedFloat(N->getOperand(0)); 272 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask); 273 } 274 275 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { 276 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 277 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 278 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 279 RTLIB::FMIN_F32, 280 RTLIB::FMIN_F64, 281 RTLIB::FMIN_F80, 282 RTLIB::FMIN_F128, 283 RTLIB::FMIN_PPCF128)); 284 } 285 286 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { 287 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 288 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 289 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 290 RTLIB::FMAX_F32, 291 RTLIB::FMAX_F64, 292 RTLIB::FMAX_F80, 293 RTLIB::FMAX_F128, 294 RTLIB::FMAX_PPCF128)); 295 } 296 297 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { 298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 299 RTLIB::ADD_F32, 300 RTLIB::ADD_F64, 301 RTLIB::ADD_F80, 302 RTLIB::ADD_F128, 303 RTLIB::ADD_PPCF128)); 304 } 305 306 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { 307 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 308 RTLIB::CBRT_F32, 309 RTLIB::CBRT_F64, 310 RTLIB::CBRT_F80, 311 RTLIB::CBRT_F128, 312 RTLIB::CBRT_PPCF128)); 313 } 314 315 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { 316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 317 RTLIB::CEIL_F32, 318 RTLIB::CEIL_F64, 319 RTLIB::CEIL_F80, 320 RTLIB::CEIL_F128, 321 RTLIB::CEIL_PPCF128)); 322 } 323 324 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { 325 SDValue LHS = GetSoftenedFloat(N->getOperand(0)); 326 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 327 SDLoc dl(N); 328 329 EVT LVT = LHS.getValueType(); 330 EVT RVT = RHS.getValueType(); 331 332 unsigned LSize = LVT.getSizeInBits(); 333 unsigned RSize = RVT.getSizeInBits(); 334 335 // First get the sign bit of second operand. 336 SDValue SignBit = DAG.getNode( 337 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 338 DAG.getConstant(RSize - 1, dl, 339 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 340 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 341 342 // Shift right or sign-extend it if the two operands have different types. 343 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 344 if (SizeDiff > 0) { 345 SignBit = 346 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 347 DAG.getConstant(SizeDiff, dl, 348 TLI.getShiftAmountTy(SignBit.getValueType(), 349 DAG.getDataLayout()))); 350 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 351 } else if (SizeDiff < 0) { 352 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 353 SignBit = 354 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 355 DAG.getConstant(-SizeDiff, dl, 356 TLI.getShiftAmountTy(SignBit.getValueType(), 357 DAG.getDataLayout()))); 358 } 359 360 // Clear the sign bit of the first operand. 361 SDValue Mask = DAG.getNode( 362 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 363 DAG.getConstant(LSize - 1, dl, 364 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 365 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 366 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 367 368 // Or the value with the sign bit. 369 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 370 } 371 372 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { 373 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 374 RTLIB::COS_F32, 375 RTLIB::COS_F64, 376 RTLIB::COS_F80, 377 RTLIB::COS_F128, 378 RTLIB::COS_PPCF128)); 379 } 380 381 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { 382 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 383 RTLIB::DIV_F32, 384 RTLIB::DIV_F64, 385 RTLIB::DIV_F80, 386 RTLIB::DIV_F128, 387 RTLIB::DIV_PPCF128)); 388 } 389 390 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { 391 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 392 RTLIB::EXP_F32, 393 RTLIB::EXP_F64, 394 RTLIB::EXP_F80, 395 RTLIB::EXP_F128, 396 RTLIB::EXP_PPCF128)); 397 } 398 399 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { 400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 401 RTLIB::EXP2_F32, 402 RTLIB::EXP2_F64, 403 RTLIB::EXP2_F80, 404 RTLIB::EXP2_F128, 405 RTLIB::EXP2_PPCF128)); 406 } 407 408 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { 409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 410 RTLIB::FLOOR_F32, 411 RTLIB::FLOOR_F64, 412 RTLIB::FLOOR_F80, 413 RTLIB::FLOOR_F128, 414 RTLIB::FLOOR_PPCF128)); 415 } 416 417 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { 418 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 419 RTLIB::LOG_F32, 420 RTLIB::LOG_F64, 421 RTLIB::LOG_F80, 422 RTLIB::LOG_F128, 423 RTLIB::LOG_PPCF128)); 424 } 425 426 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { 427 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 428 RTLIB::LOG2_F32, 429 RTLIB::LOG2_F64, 430 RTLIB::LOG2_F80, 431 RTLIB::LOG2_F128, 432 RTLIB::LOG2_PPCF128)); 433 } 434 435 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { 436 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 437 RTLIB::LOG10_F32, 438 RTLIB::LOG10_F64, 439 RTLIB::LOG10_F80, 440 RTLIB::LOG10_F128, 441 RTLIB::LOG10_PPCF128)); 442 } 443 444 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { 445 bool IsStrict = N->isStrictFPOpcode(); 446 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 447 unsigned Offset = IsStrict ? 1 : 0; 448 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 449 GetSoftenedFloat(N->getOperand(1 + Offset)), 450 GetSoftenedFloat(N->getOperand(2 + Offset)) }; 451 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 452 TargetLowering::MakeLibCallOptions CallOptions; 453 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(), 454 N->getOperand(1 + Offset).getValueType(), 455 N->getOperand(2 + Offset).getValueType() }; 456 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 457 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, 458 GetFPLibCall(N->getValueType(0), 459 RTLIB::FMA_F32, 460 RTLIB::FMA_F64, 461 RTLIB::FMA_F80, 462 RTLIB::FMA_F128, 463 RTLIB::FMA_PPCF128), 464 NVT, Ops, CallOptions, SDLoc(N), Chain); 465 if (IsStrict) 466 ReplaceValueWith(SDValue(N, 1), Tmp.second); 467 return Tmp.first; 468 } 469 470 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { 471 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 472 RTLIB::MUL_F32, 473 RTLIB::MUL_F64, 474 RTLIB::MUL_F80, 475 RTLIB::MUL_F128, 476 RTLIB::MUL_PPCF128)); 477 } 478 479 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { 480 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 481 RTLIB::NEARBYINT_F32, 482 RTLIB::NEARBYINT_F64, 483 RTLIB::NEARBYINT_F80, 484 RTLIB::NEARBYINT_F128, 485 RTLIB::NEARBYINT_PPCF128)); 486 } 487 488 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { 489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 490 SDLoc dl(N); 491 492 // Expand Y = FNEG(X) -> Y = X ^ sign mask 493 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); 494 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), 495 DAG.getConstant(SignMask, dl, NVT)); 496 } 497 498 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { 499 bool IsStrict = N->isStrictFPOpcode(); 500 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 501 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 502 503 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 504 505 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { 506 Op = GetPromotedFloat(Op); 507 // If the promotion did the FP_EXTEND to the destination type for us, 508 // there's nothing left to do here. 509 if (Op.getValueType() == N->getValueType(0)) 510 return BitConvertToInteger(Op); 511 } 512 513 // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's 514 // entirely possible for both f16 and f32 to be legal, so use the fully 515 // hard-float FP_EXTEND rather than FP16_TO_FP. 516 if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) { 517 if (IsStrict) { 518 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 519 { MVT::f32, MVT::Other }, { Chain, Op }); 520 Chain = Op.getValue(1); 521 } else { 522 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); 523 } 524 } 525 526 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); 527 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 528 TargetLowering::MakeLibCallOptions CallOptions; 529 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 530 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 531 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 532 CallOptions, SDLoc(N), 533 Chain); 534 if (IsStrict) 535 ReplaceValueWith(SDValue(N, 1), Tmp.second); 536 return Tmp.first; 537 } 538 539 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 540 // nodes? 541 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { 542 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 543 SDValue Op = N->getOperand(0); 544 TargetLowering::MakeLibCallOptions CallOptions; 545 EVT OpsVT[1] = { N->getOperand(0).getValueType() }; 546 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 547 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op, 548 CallOptions, SDLoc(N)).first; 549 if (N->getValueType(0) == MVT::f32) 550 return Res32; 551 552 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 553 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0)); 554 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 555 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first; 556 } 557 558 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { 559 bool IsStrict = N->isStrictFPOpcode(); 560 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 561 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 562 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 563 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); 564 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 565 TargetLowering::MakeLibCallOptions CallOptions; 566 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 567 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 568 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 569 CallOptions, SDLoc(N), 570 Chain); 571 if (IsStrict) 572 ReplaceValueWith(SDValue(N, 1), Tmp.second); 573 return Tmp.first; 574 } 575 576 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { 577 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 578 RTLIB::POW_F32, 579 RTLIB::POW_F64, 580 RTLIB::POW_F80, 581 RTLIB::POW_F128, 582 RTLIB::POW_PPCF128)); 583 } 584 585 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) { 586 bool IsStrict = N->isStrictFPOpcode(); 587 unsigned Offset = IsStrict ? 1 : 0; 588 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 || 589 N->getOperand(1 + Offset).getValueType() == MVT::i32) && 590 "Unsupported power type!"); 591 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0)); 592 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 593 if (!TLI.getLibcallName(LC)) { 594 // Some targets don't have a powi libcall; use pow instead. 595 // FIXME: Implement this if some target needs it. 596 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow"); 597 return DAG.getUNDEF(N->getValueType(0)); 598 } 599 600 if (DAG.getLibInfo().getIntSize() != 601 N->getOperand(1 + Offset).getValueType().getSizeInBits()) { 602 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI 603 // would use the wrong type for the argument. 604 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); 605 return DAG.getUNDEF(N->getValueType(0)); 606 } 607 608 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 609 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 610 N->getOperand(1 + Offset) }; 611 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 612 TargetLowering::MakeLibCallOptions CallOptions; 613 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 614 N->getOperand(1 + Offset).getValueType() }; 615 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 616 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 617 CallOptions, SDLoc(N), 618 Chain); 619 if (IsStrict) 620 ReplaceValueWith(SDValue(N, 1), Tmp.second); 621 return Tmp.first; 622 } 623 624 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { 625 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 626 RTLIB::REM_F32, 627 RTLIB::REM_F64, 628 RTLIB::REM_F80, 629 RTLIB::REM_F128, 630 RTLIB::REM_PPCF128)); 631 } 632 633 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { 634 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 635 RTLIB::RINT_F32, 636 RTLIB::RINT_F64, 637 RTLIB::RINT_F80, 638 RTLIB::RINT_F128, 639 RTLIB::RINT_PPCF128)); 640 } 641 642 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { 643 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 644 RTLIB::ROUND_F32, 645 RTLIB::ROUND_F64, 646 RTLIB::ROUND_F80, 647 RTLIB::ROUND_F128, 648 RTLIB::ROUND_PPCF128)); 649 } 650 651 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { 652 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 653 RTLIB::ROUNDEVEN_F32, 654 RTLIB::ROUNDEVEN_F64, 655 RTLIB::ROUNDEVEN_F80, 656 RTLIB::ROUNDEVEN_F128, 657 RTLIB::ROUNDEVEN_PPCF128)); 658 } 659 660 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { 661 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 662 RTLIB::SIN_F32, 663 RTLIB::SIN_F64, 664 RTLIB::SIN_F80, 665 RTLIB::SIN_F128, 666 RTLIB::SIN_PPCF128)); 667 } 668 669 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { 670 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 671 RTLIB::SQRT_F32, 672 RTLIB::SQRT_F64, 673 RTLIB::SQRT_F80, 674 RTLIB::SQRT_F128, 675 RTLIB::SQRT_PPCF128)); 676 } 677 678 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { 679 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 680 RTLIB::SUB_F32, 681 RTLIB::SUB_F64, 682 RTLIB::SUB_F80, 683 RTLIB::SUB_F128, 684 RTLIB::SUB_PPCF128)); 685 } 686 687 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { 688 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 689 RTLIB::TRUNC_F32, 690 RTLIB::TRUNC_F64, 691 RTLIB::TRUNC_F80, 692 RTLIB::TRUNC_F128, 693 RTLIB::TRUNC_PPCF128)); 694 } 695 696 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { 697 LoadSDNode *L = cast<LoadSDNode>(N); 698 EVT VT = N->getValueType(0); 699 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 700 SDLoc dl(N); 701 702 auto MMOFlags = 703 L->getMemOperand()->getFlags() & 704 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 705 SDValue NewL; 706 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 707 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, 708 L->getChain(), L->getBasePtr(), L->getOffset(), 709 L->getPointerInfo(), NVT, L->getOriginalAlign(), 710 MMOFlags, L->getAAInfo()); 711 // Legalized the chain result - switch anything that used the old chain to 712 // use the new one. 713 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 714 return NewL; 715 } 716 717 // Do a non-extending load followed by FP_EXTEND. 718 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(), 719 dl, L->getChain(), L->getBasePtr(), L->getOffset(), 720 L->getPointerInfo(), L->getMemoryVT(), 721 L->getOriginalAlign(), MMOFlags, L->getAAInfo()); 722 // Legalized the chain result - switch anything that used the old chain to 723 // use the new one. 724 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 725 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL); 726 return BitConvertToInteger(ExtendNode); 727 } 728 729 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { 730 SDValue LHS = GetSoftenedFloat(N->getOperand(1)); 731 SDValue RHS = GetSoftenedFloat(N->getOperand(2)); 732 return DAG.getSelect(SDLoc(N), 733 LHS.getValueType(), N->getOperand(0), LHS, RHS); 734 } 735 736 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { 737 SDValue LHS = GetSoftenedFloat(N->getOperand(2)); 738 SDValue RHS = GetSoftenedFloat(N->getOperand(3)); 739 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 740 LHS.getValueType(), N->getOperand(0), 741 N->getOperand(1), LHS, RHS, N->getOperand(4)); 742 } 743 744 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { 745 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 746 N->getValueType(0))); 747 } 748 749 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { 750 SDValue Chain = N->getOperand(0); // Get the chain. 751 SDValue Ptr = N->getOperand(1); // Get the pointer. 752 EVT VT = N->getValueType(0); 753 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 754 SDLoc dl(N); 755 756 SDValue NewVAARG; 757 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), 758 N->getConstantOperandVal(3)); 759 760 // Legalized the chain result - switch anything that used the old chain to 761 // use the new one. 762 if (N != NewVAARG.getValue(1).getNode()) 763 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1)); 764 return NewVAARG; 765 } 766 767 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { 768 bool IsStrict = N->isStrictFPOpcode(); 769 bool Signed = N->getOpcode() == ISD::SINT_TO_FP || 770 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 771 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 772 EVT RVT = N->getValueType(0); 773 EVT NVT = EVT(); 774 SDLoc dl(N); 775 776 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to 777 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly 778 // match. Look for an appropriate libcall. 779 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 780 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 781 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { 782 NVT = (MVT::SimpleValueType)t; 783 // The source needs to big enough to hold the operand. 784 if (NVT.bitsGE(SVT)) 785 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT); 786 } 787 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 788 789 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 790 // Sign/zero extend the argument if the libcall takes a larger type. 791 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 792 NVT, N->getOperand(IsStrict ? 1 : 0)); 793 TargetLowering::MakeLibCallOptions CallOptions; 794 CallOptions.setSExt(Signed); 795 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 796 std::pair<SDValue, SDValue> Tmp = 797 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), 798 Op, CallOptions, dl, Chain); 799 800 if (IsStrict) 801 ReplaceValueWith(SDValue(N, 1), Tmp.second); 802 return Tmp.first; 803 } 804 805 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) { 806 // Expand and soften recursively. 807 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 808 return SDValue(); 809 } 810 811 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) { 812 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 813 return SDValue(); 814 } 815 816 //===----------------------------------------------------------------------===// 817 // Convert Float Operand to Integer 818 //===----------------------------------------------------------------------===// 819 820 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { 821 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG); 822 dbgs() << "\n"); 823 SDValue Res = SDValue(); 824 825 switch (N->getOpcode()) { 826 default: 827 #ifndef NDEBUG 828 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": "; 829 N->dump(&DAG); dbgs() << "\n"; 830 #endif 831 llvm_unreachable("Do not know how to soften this operator's operand!"); 832 833 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; 834 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; 835 case ISD::STRICT_FP_TO_FP16: 836 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes 837 case ISD::FP_TO_BF16: 838 case ISD::STRICT_FP_ROUND: 839 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; 840 case ISD::STRICT_FP_TO_SINT: 841 case ISD::STRICT_FP_TO_UINT: 842 case ISD::FP_TO_SINT: 843 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; 844 case ISD::FP_TO_SINT_SAT: 845 case ISD::FP_TO_UINT_SAT: 846 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; 847 case ISD::STRICT_LROUND: 848 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; 849 case ISD::STRICT_LLROUND: 850 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; 851 case ISD::STRICT_LRINT: 852 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; 853 case ISD::STRICT_LLRINT: 854 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; 855 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; 856 case ISD::STRICT_FSETCC: 857 case ISD::STRICT_FSETCCS: 858 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; 859 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; 860 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; 861 } 862 863 // If the result is null, the sub-method took care of registering results etc. 864 if (!Res.getNode()) return false; 865 866 // If the result is N, the sub-method updated N in place. Tell the legalizer 867 // core about this to re-analyze. 868 if (Res.getNode() == N) 869 return true; 870 871 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 872 "Invalid operand softening"); 873 874 ReplaceValueWith(SDValue(N, 0), Res); 875 return false; 876 } 877 878 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { 879 SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); 880 881 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 882 } 883 884 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { 885 // We actually deal with the partially-softened FP_TO_FP16 node too, which 886 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. 887 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || 888 N->getOpcode() == ISD::STRICT_FP_TO_FP16 || 889 N->getOpcode() == ISD::FP_TO_BF16 || 890 N->getOpcode() == ISD::STRICT_FP_ROUND); 891 892 bool IsStrict = N->isStrictFPOpcode(); 893 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 894 EVT SVT = Op.getValueType(); 895 EVT RVT = N->getValueType(0); 896 EVT FloatRVT = RVT; 897 if (N->getOpcode() == ISD::FP_TO_FP16 || 898 N->getOpcode() == ISD::STRICT_FP_TO_FP16) 899 FloatRVT = MVT::f16; 900 else if (N->getOpcode() == ISD::FP_TO_BF16) 901 FloatRVT = MVT::bf16; 902 903 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT); 904 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 905 906 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 907 Op = GetSoftenedFloat(Op); 908 TargetLowering::MakeLibCallOptions CallOptions; 909 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 910 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 911 CallOptions, SDLoc(N), 912 Chain); 913 if (IsStrict) { 914 ReplaceValueWith(SDValue(N, 1), Tmp.second); 915 ReplaceValueWith(SDValue(N, 0), Tmp.first); 916 return SDValue(); 917 } 918 return Tmp.first; 919 } 920 921 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 922 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 923 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 924 925 EVT VT = NewLHS.getValueType(); 926 NewLHS = GetSoftenedFloat(NewLHS); 927 NewRHS = GetSoftenedFloat(NewRHS); 928 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 929 N->getOperand(2), N->getOperand(3)); 930 931 // If softenSetCCOperands returned a scalar, we need to compare the result 932 // against zero to select between true and false values. 933 if (!NewRHS.getNode()) { 934 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 935 CCCode = ISD::SETNE; 936 } 937 938 // Update N to have the operands specified. 939 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 940 DAG.getCondCode(CCCode), NewLHS, NewRHS, 941 N->getOperand(4)), 942 0); 943 } 944 945 // Even if the result type is legal, no libcall may exactly match. (e.g. We 946 // don't have FP-i8 conversions) This helper method looks for an appropriate 947 // promoted libcall. 948 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, 949 bool Signed) { 950 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 951 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 952 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 953 ++IntVT) { 954 Promoted = (MVT::SimpleValueType)IntVT; 955 // The type needs to big enough to hold the result. 956 if (Promoted.bitsGE(RetVT)) 957 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted) 958 : RTLIB::getFPTOUINT(SrcVT, Promoted); 959 } 960 return LC; 961 } 962 963 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { 964 bool IsStrict = N->isStrictFPOpcode(); 965 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 966 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 967 968 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 969 EVT SVT = Op.getValueType(); 970 EVT RVT = N->getValueType(0); 971 EVT NVT = EVT(); 972 SDLoc dl(N); 973 974 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to 975 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly 976 // match, eg. we don't have fp -> i8 conversions. 977 // Look for an appropriate libcall. 978 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed); 979 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 980 "Unsupported FP_TO_XINT!"); 981 982 Op = GetSoftenedFloat(Op); 983 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 984 TargetLowering::MakeLibCallOptions CallOptions; 985 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 986 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 987 CallOptions, dl, Chain); 988 989 // Truncate the result if the libcall returns a larger type. 990 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first); 991 992 if (!IsStrict) 993 return Res; 994 995 ReplaceValueWith(SDValue(N, 1), Tmp.second); 996 ReplaceValueWith(SDValue(N, 0), Res); 997 return SDValue(); 998 } 999 1000 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { 1001 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 1002 return Res; 1003 } 1004 1005 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { 1006 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1007 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1008 1009 EVT VT = NewLHS.getValueType(); 1010 NewLHS = GetSoftenedFloat(NewLHS); 1011 NewRHS = GetSoftenedFloat(NewRHS); 1012 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1013 N->getOperand(0), N->getOperand(1)); 1014 1015 // If softenSetCCOperands returned a scalar, we need to compare the result 1016 // against zero to select between true and false values. 1017 if (!NewRHS.getNode()) { 1018 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1019 CCCode = ISD::SETNE; 1020 } 1021 1022 // Update N to have the operands specified. 1023 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1024 N->getOperand(2), N->getOperand(3), 1025 DAG.getCondCode(CCCode)), 1026 0); 1027 } 1028 1029 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { 1030 bool IsStrict = N->isStrictFPOpcode(); 1031 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); 1032 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1); 1033 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1034 ISD::CondCode CCCode = 1035 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1036 1037 EVT VT = Op0.getValueType(); 1038 SDValue NewLHS = GetSoftenedFloat(Op0); 1039 SDValue NewRHS = GetSoftenedFloat(Op1); 1040 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1, 1041 Chain, N->getOpcode() == ISD::STRICT_FSETCCS); 1042 1043 // Update N to have the operands specified. 1044 if (NewRHS.getNode()) { 1045 if (IsStrict) 1046 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS, 1047 NewRHS, DAG.getCondCode(CCCode)); 1048 else 1049 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1050 DAG.getCondCode(CCCode)), 0); 1051 } 1052 1053 // Otherwise, softenSetCCOperands returned a scalar, use it. 1054 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && 1055 "Unexpected setcc expansion!"); 1056 1057 if (IsStrict) { 1058 ReplaceValueWith(SDValue(N, 0), NewLHS); 1059 ReplaceValueWith(SDValue(N, 1), Chain); 1060 return SDValue(); 1061 } 1062 return NewLHS; 1063 } 1064 1065 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { 1066 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1067 assert(OpNo == 1 && "Can only soften the stored value!"); 1068 StoreSDNode *ST = cast<StoreSDNode>(N); 1069 SDValue Val = ST->getValue(); 1070 SDLoc dl(N); 1071 1072 if (ST->isTruncatingStore()) 1073 // Do an FP_ROUND followed by a non-truncating store. 1074 Val = BitConvertToInteger( 1075 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val, 1076 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); 1077 else 1078 Val = GetSoftenedFloat(Val); 1079 1080 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(), 1081 ST->getMemOperand()); 1082 } 1083 1084 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { 1085 SDValue LHS = N->getOperand(0); 1086 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 1087 SDLoc dl(N); 1088 1089 EVT LVT = LHS.getValueType(); 1090 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits()); 1091 EVT RVT = RHS.getValueType(); 1092 1093 unsigned LSize = LVT.getSizeInBits(); 1094 unsigned RSize = RVT.getSizeInBits(); 1095 1096 // Shift right or sign-extend it if the two operands have different types. 1097 int SizeDiff = RSize - LSize; 1098 if (SizeDiff > 0) { 1099 RHS = 1100 DAG.getNode(ISD::SRL, dl, RVT, RHS, 1101 DAG.getConstant(SizeDiff, dl, 1102 TLI.getShiftAmountTy(RHS.getValueType(), 1103 DAG.getDataLayout()))); 1104 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS); 1105 } else if (SizeDiff < 0) { 1106 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS); 1107 RHS = 1108 DAG.getNode(ISD::SHL, dl, ILVT, RHS, 1109 DAG.getConstant(-SizeDiff, dl, 1110 TLI.getShiftAmountTy(RHS.getValueType(), 1111 DAG.getDataLayout()))); 1112 } 1113 1114 RHS = DAG.getBitcast(LVT, RHS); 1115 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS); 1116 } 1117 1118 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { 1119 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1120 bool IsStrict = N->isStrictFPOpcode(); 1121 unsigned Offset = IsStrict ? 1 : 0; 1122 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 1123 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1124 TargetLowering::MakeLibCallOptions CallOptions; 1125 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 1126 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 1127 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1128 CallOptions, SDLoc(N), 1129 Chain); 1130 if (IsStrict) { 1131 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1132 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1133 return SDValue(); 1134 } 1135 1136 return Tmp.first; 1137 } 1138 1139 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { 1140 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1141 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1142 RTLIB::LROUND_F32, 1143 RTLIB::LROUND_F64, 1144 RTLIB::LROUND_F80, 1145 RTLIB::LROUND_F128, 1146 RTLIB::LROUND_PPCF128)); 1147 } 1148 1149 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { 1150 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1151 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1152 RTLIB::LLROUND_F32, 1153 RTLIB::LLROUND_F64, 1154 RTLIB::LLROUND_F80, 1155 RTLIB::LLROUND_F128, 1156 RTLIB::LLROUND_PPCF128)); 1157 } 1158 1159 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { 1160 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1161 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1162 RTLIB::LRINT_F32, 1163 RTLIB::LRINT_F64, 1164 RTLIB::LRINT_F80, 1165 RTLIB::LRINT_F128, 1166 RTLIB::LRINT_PPCF128)); 1167 } 1168 1169 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { 1170 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1171 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1172 RTLIB::LLRINT_F32, 1173 RTLIB::LLRINT_F64, 1174 RTLIB::LLRINT_F80, 1175 RTLIB::LLRINT_F128, 1176 RTLIB::LLRINT_PPCF128)); 1177 } 1178 1179 //===----------------------------------------------------------------------===// 1180 // Float Result Expansion 1181 //===----------------------------------------------------------------------===// 1182 1183 /// ExpandFloatResult - This method is called when the specified result of the 1184 /// specified node is found to need expansion. At this point, the node may also 1185 /// have invalid operands or may have other results that need promotion, we just 1186 /// know that (at least) one result needs expansion. 1187 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { 1188 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n"); 1189 SDValue Lo, Hi; 1190 Lo = Hi = SDValue(); 1191 1192 // See if the target wants to custom expand this node. 1193 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1194 return; 1195 1196 switch (N->getOpcode()) { 1197 default: 1198 #ifndef NDEBUG 1199 dbgs() << "ExpandFloatResult #" << ResNo << ": "; 1200 N->dump(&DAG); dbgs() << "\n"; 1201 #endif 1202 llvm_unreachable("Do not know how to expand the result of this operator!"); 1203 1204 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1205 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 1206 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1207 1208 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1209 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1210 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1211 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1212 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1213 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1214 1215 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; 1216 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; 1217 case ISD::STRICT_FMINNUM: 1218 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; 1219 case ISD::STRICT_FMAXNUM: 1220 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; 1221 case ISD::STRICT_FADD: 1222 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; 1223 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; 1224 case ISD::STRICT_FCEIL: 1225 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; 1226 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; 1227 case ISD::STRICT_FCOS: 1228 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; 1229 case ISD::STRICT_FDIV: 1230 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; 1231 case ISD::STRICT_FEXP: 1232 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; 1233 case ISD::STRICT_FEXP2: 1234 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; 1235 case ISD::STRICT_FFLOOR: 1236 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; 1237 case ISD::STRICT_FLOG: 1238 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; 1239 case ISD::STRICT_FLOG2: 1240 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; 1241 case ISD::STRICT_FLOG10: 1242 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; 1243 case ISD::STRICT_FMA: 1244 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; 1245 case ISD::STRICT_FMUL: 1246 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; 1247 case ISD::STRICT_FNEARBYINT: 1248 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; 1249 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; 1250 case ISD::STRICT_FP_EXTEND: 1251 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; 1252 case ISD::STRICT_FPOW: 1253 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; 1254 case ISD::STRICT_FPOWI: 1255 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; 1256 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; 1257 case ISD::STRICT_FRINT: 1258 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; 1259 case ISD::STRICT_FROUND: 1260 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; 1261 case ISD::STRICT_FROUNDEVEN: 1262 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; 1263 case ISD::STRICT_FSIN: 1264 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; 1265 case ISD::STRICT_FSQRT: 1266 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; 1267 case ISD::STRICT_FSUB: 1268 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; 1269 case ISD::STRICT_FTRUNC: 1270 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; 1271 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; 1272 case ISD::STRICT_SINT_TO_FP: 1273 case ISD::STRICT_UINT_TO_FP: 1274 case ISD::SINT_TO_FP: 1275 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; 1276 case ISD::STRICT_FREM: 1277 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; 1278 } 1279 1280 // If Lo/Hi is null, the sub-method took care of registering results etc. 1281 if (Lo.getNode()) 1282 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi); 1283 } 1284 1285 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, 1286 SDValue &Hi) { 1287 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1288 assert(NVT.getSizeInBits() == 64 && 1289 "Do not know how to expand this float constant!"); 1290 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); 1291 SDLoc dl(N); 1292 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1293 APInt(64, C.getRawData()[1])), 1294 dl, NVT); 1295 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1296 APInt(64, C.getRawData()[0])), 1297 dl, NVT); 1298 } 1299 1300 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, 1301 SDValue &Lo, SDValue &Hi) { 1302 bool IsStrict = N->isStrictFPOpcode(); 1303 unsigned Offset = IsStrict ? 1 : 0; 1304 SDValue Op = N->getOperand(0 + Offset); 1305 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1306 TargetLowering::MakeLibCallOptions CallOptions; 1307 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1308 Op, CallOptions, SDLoc(N), 1309 Chain); 1310 if (IsStrict) 1311 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1312 GetPairElements(Tmp.first, Lo, Hi); 1313 } 1314 1315 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, 1316 SDValue &Lo, SDValue &Hi) { 1317 bool IsStrict = N->isStrictFPOpcode(); 1318 unsigned Offset = IsStrict ? 1 : 0; 1319 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) }; 1320 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1321 TargetLowering::MakeLibCallOptions CallOptions; 1322 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1323 Ops, CallOptions, SDLoc(N), 1324 Chain); 1325 if (IsStrict) 1326 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1327 GetPairElements(Tmp.first, Lo, Hi); 1328 } 1329 1330 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, 1331 SDValue &Hi) { 1332 assert(N->getValueType(0) == MVT::ppcf128 && 1333 "Logic only correct for ppcf128!"); 1334 SDLoc dl(N); 1335 SDValue Tmp; 1336 GetExpandedFloat(N->getOperand(0), Lo, Tmp); 1337 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp); 1338 // Lo = Hi==fabs(Hi) ? Lo : -Lo; 1339 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo, 1340 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo), 1341 ISD::SETEQ); 1342 } 1343 1344 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, 1345 SDValue &Hi) { 1346 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1347 RTLIB::FMIN_F32, RTLIB::FMIN_F64, 1348 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 1349 RTLIB::FMIN_PPCF128), Lo, Hi); 1350 } 1351 1352 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, 1353 SDValue &Hi) { 1354 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1355 RTLIB::FMAX_F32, RTLIB::FMAX_F64, 1356 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 1357 RTLIB::FMAX_PPCF128), Lo, Hi); 1358 } 1359 1360 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, 1361 SDValue &Hi) { 1362 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1363 RTLIB::ADD_F32, RTLIB::ADD_F64, 1364 RTLIB::ADD_F80, RTLIB::ADD_F128, 1365 RTLIB::ADD_PPCF128), Lo, Hi); 1366 } 1367 1368 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, 1369 SDValue &Hi) { 1370 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32, 1371 RTLIB::CBRT_F64, RTLIB::CBRT_F80, 1372 RTLIB::CBRT_F128, 1373 RTLIB::CBRT_PPCF128), Lo, Hi); 1374 } 1375 1376 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, 1377 SDValue &Lo, SDValue &Hi) { 1378 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1379 RTLIB::CEIL_F32, RTLIB::CEIL_F64, 1380 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 1381 RTLIB::CEIL_PPCF128), Lo, Hi); 1382 } 1383 1384 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, 1385 SDValue &Lo, SDValue &Hi) { 1386 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1387 RTLIB::COPYSIGN_F32, 1388 RTLIB::COPYSIGN_F64, 1389 RTLIB::COPYSIGN_F80, 1390 RTLIB::COPYSIGN_F128, 1391 RTLIB::COPYSIGN_PPCF128), Lo, Hi); 1392 } 1393 1394 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, 1395 SDValue &Lo, SDValue &Hi) { 1396 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1397 RTLIB::COS_F32, RTLIB::COS_F64, 1398 RTLIB::COS_F80, RTLIB::COS_F128, 1399 RTLIB::COS_PPCF128), Lo, Hi); 1400 } 1401 1402 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, 1403 SDValue &Hi) { 1404 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1405 RTLIB::DIV_F32, 1406 RTLIB::DIV_F64, 1407 RTLIB::DIV_F80, 1408 RTLIB::DIV_F128, 1409 RTLIB::DIV_PPCF128), Lo, Hi); 1410 } 1411 1412 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, 1413 SDValue &Lo, SDValue &Hi) { 1414 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1415 RTLIB::EXP_F32, RTLIB::EXP_F64, 1416 RTLIB::EXP_F80, RTLIB::EXP_F128, 1417 RTLIB::EXP_PPCF128), Lo, Hi); 1418 } 1419 1420 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, 1421 SDValue &Lo, SDValue &Hi) { 1422 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1423 RTLIB::EXP2_F32, RTLIB::EXP2_F64, 1424 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 1425 RTLIB::EXP2_PPCF128), Lo, Hi); 1426 } 1427 1428 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, 1429 SDValue &Lo, SDValue &Hi) { 1430 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1431 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 1432 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 1433 RTLIB::FLOOR_PPCF128), Lo, Hi); 1434 } 1435 1436 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, 1437 SDValue &Lo, SDValue &Hi) { 1438 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1439 RTLIB::LOG_F32, RTLIB::LOG_F64, 1440 RTLIB::LOG_F80, RTLIB::LOG_F128, 1441 RTLIB::LOG_PPCF128), Lo, Hi); 1442 } 1443 1444 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, 1445 SDValue &Lo, SDValue &Hi) { 1446 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1447 RTLIB::LOG2_F32, RTLIB::LOG2_F64, 1448 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 1449 RTLIB::LOG2_PPCF128), Lo, Hi); 1450 } 1451 1452 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, 1453 SDValue &Lo, SDValue &Hi) { 1454 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1455 RTLIB::LOG10_F32, RTLIB::LOG10_F64, 1456 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 1457 RTLIB::LOG10_PPCF128), Lo, Hi); 1458 } 1459 1460 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, 1461 SDValue &Hi) { 1462 bool IsStrict = N->isStrictFPOpcode(); 1463 unsigned Offset = IsStrict ? 1 : 0; 1464 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset), 1465 N->getOperand(2 + Offset) }; 1466 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1467 TargetLowering::MakeLibCallOptions CallOptions; 1468 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), 1469 RTLIB::FMA_F32, 1470 RTLIB::FMA_F64, 1471 RTLIB::FMA_F80, 1472 RTLIB::FMA_F128, 1473 RTLIB::FMA_PPCF128), 1474 N->getValueType(0), Ops, CallOptions, 1475 SDLoc(N), Chain); 1476 if (IsStrict) 1477 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1478 GetPairElements(Tmp.first, Lo, Hi); 1479 } 1480 1481 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, 1482 SDValue &Hi) { 1483 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1484 RTLIB::MUL_F32, 1485 RTLIB::MUL_F64, 1486 RTLIB::MUL_F80, 1487 RTLIB::MUL_F128, 1488 RTLIB::MUL_PPCF128), Lo, Hi); 1489 } 1490 1491 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, 1492 SDValue &Lo, SDValue &Hi) { 1493 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1494 RTLIB::NEARBYINT_F32, 1495 RTLIB::NEARBYINT_F64, 1496 RTLIB::NEARBYINT_F80, 1497 RTLIB::NEARBYINT_F128, 1498 RTLIB::NEARBYINT_PPCF128), Lo, Hi); 1499 } 1500 1501 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, 1502 SDValue &Hi) { 1503 SDLoc dl(N); 1504 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1505 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo); 1506 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); 1507 } 1508 1509 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, 1510 SDValue &Hi) { 1511 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1512 SDLoc dl(N); 1513 bool IsStrict = N->isStrictFPOpcode(); 1514 1515 SDValue Chain; 1516 if (IsStrict) { 1517 // If the expanded type is the same as the input type, just bypass the node. 1518 if (NVT == N->getOperand(1).getValueType()) { 1519 Hi = N->getOperand(1); 1520 Chain = N->getOperand(0); 1521 } else { 1522 // Other we need to extend. 1523 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, 1524 { N->getOperand(0), N->getOperand(1) }); 1525 Chain = Hi.getValue(1); 1526 } 1527 } else { 1528 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0)); 1529 } 1530 1531 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1532 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1533 1534 if (IsStrict) 1535 ReplaceValueWith(SDValue(N, 1), Chain); 1536 } 1537 1538 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, 1539 SDValue &Lo, SDValue &Hi) { 1540 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1541 RTLIB::POW_F32, RTLIB::POW_F64, 1542 RTLIB::POW_F80, RTLIB::POW_F128, 1543 RTLIB::POW_PPCF128), Lo, Hi); 1544 } 1545 1546 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, 1547 SDValue &Lo, SDValue &Hi) { 1548 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi); 1549 } 1550 1551 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, 1552 SDValue &Lo, SDValue &Hi) { 1553 assert(N->getValueType(0) == MVT::ppcf128 && 1554 "Logic only correct for ppcf128!"); 1555 1556 SDLoc dl(N); 1557 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1558 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo); 1559 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi); 1560 } 1561 1562 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, 1563 SDValue &Lo, SDValue &Hi) { 1564 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1565 RTLIB::REM_F32, RTLIB::REM_F64, 1566 RTLIB::REM_F80, RTLIB::REM_F128, 1567 RTLIB::REM_PPCF128), Lo, Hi); 1568 } 1569 1570 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, 1571 SDValue &Lo, SDValue &Hi) { 1572 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1573 RTLIB::RINT_F32, RTLIB::RINT_F64, 1574 RTLIB::RINT_F80, RTLIB::RINT_F128, 1575 RTLIB::RINT_PPCF128), Lo, Hi); 1576 } 1577 1578 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, 1579 SDValue &Lo, SDValue &Hi) { 1580 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1581 RTLIB::ROUND_F32, 1582 RTLIB::ROUND_F64, 1583 RTLIB::ROUND_F80, 1584 RTLIB::ROUND_F128, 1585 RTLIB::ROUND_PPCF128), Lo, Hi); 1586 } 1587 1588 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, 1589 SDValue &Lo, SDValue &Hi) { 1590 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1591 RTLIB::ROUNDEVEN_F32, 1592 RTLIB::ROUNDEVEN_F64, 1593 RTLIB::ROUNDEVEN_F80, 1594 RTLIB::ROUNDEVEN_F128, 1595 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); 1596 } 1597 1598 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, 1599 SDValue &Lo, SDValue &Hi) { 1600 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1601 RTLIB::SIN_F32, RTLIB::SIN_F64, 1602 RTLIB::SIN_F80, RTLIB::SIN_F128, 1603 RTLIB::SIN_PPCF128), Lo, Hi); 1604 } 1605 1606 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, 1607 SDValue &Lo, SDValue &Hi) { 1608 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1609 RTLIB::SQRT_F32, RTLIB::SQRT_F64, 1610 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 1611 RTLIB::SQRT_PPCF128), Lo, Hi); 1612 } 1613 1614 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, 1615 SDValue &Hi) { 1616 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1617 RTLIB::SUB_F32, 1618 RTLIB::SUB_F64, 1619 RTLIB::SUB_F80, 1620 RTLIB::SUB_F128, 1621 RTLIB::SUB_PPCF128), Lo, Hi); 1622 } 1623 1624 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, 1625 SDValue &Lo, SDValue &Hi) { 1626 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1627 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 1628 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 1629 RTLIB::TRUNC_PPCF128), Lo, Hi); 1630 } 1631 1632 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, 1633 SDValue &Hi) { 1634 if (ISD::isNormalLoad(N)) { 1635 ExpandRes_NormalLoad(N, Lo, Hi); 1636 return; 1637 } 1638 1639 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1640 LoadSDNode *LD = cast<LoadSDNode>(N); 1641 SDValue Chain = LD->getChain(); 1642 SDValue Ptr = LD->getBasePtr(); 1643 SDLoc dl(N); 1644 1645 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1646 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1647 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1648 1649 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr, 1650 LD->getMemoryVT(), LD->getMemOperand()); 1651 1652 // Remember the chain. 1653 Chain = Hi.getValue(1); 1654 1655 // The low part is zero. 1656 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1657 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1658 1659 // Modified the chain - switch anything that used the old chain to use the 1660 // new one. 1661 ReplaceValueWith(SDValue(LD, 1), Chain); 1662 } 1663 1664 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, 1665 SDValue &Hi) { 1666 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!"); 1667 EVT VT = N->getValueType(0); 1668 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1669 bool Strict = N->isStrictFPOpcode(); 1670 SDValue Src = N->getOperand(Strict ? 1 : 0); 1671 EVT SrcVT = Src.getValueType(); 1672 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP || 1673 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 1674 SDLoc dl(N); 1675 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode(); 1676 1677 // TODO: Any other flags to propagate? 1678 SDNodeFlags Flags; 1679 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept()); 1680 1681 // First do an SINT_TO_FP, whether the original was signed or unsigned. 1682 // When promoting partial word types to i32 we must honor the signedness, 1683 // though. 1684 if (SrcVT.bitsLE(MVT::i32)) { 1685 // The integer can be represented exactly in an f64. 1686 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1687 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1688 if (Strict) { 1689 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), 1690 {Chain, Src}, Flags); 1691 Chain = Hi.getValue(1); 1692 } else 1693 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src); 1694 } else { 1695 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1696 if (SrcVT.bitsLE(MVT::i64)) { 1697 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1698 MVT::i64, Src); 1699 LC = RTLIB::SINTTOFP_I64_PPCF128; 1700 } else if (SrcVT.bitsLE(MVT::i128)) { 1701 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); 1702 LC = RTLIB::SINTTOFP_I128_PPCF128; 1703 } 1704 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 1705 1706 TargetLowering::MakeLibCallOptions CallOptions; 1707 CallOptions.setSExt(true); 1708 std::pair<SDValue, SDValue> Tmp = 1709 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); 1710 if (Strict) 1711 Chain = Tmp.second; 1712 GetPairElements(Tmp.first, Lo, Hi); 1713 } 1714 1715 // No need to complement for unsigned 32-bit integers 1716 if (isSigned || SrcVT.bitsLE(MVT::i32)) { 1717 if (Strict) 1718 ReplaceValueWith(SDValue(N, 1), Chain); 1719 1720 return; 1721 } 1722 1723 // Unsigned - fix up the SINT_TO_FP value just calculated. 1724 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully 1725 // keep semantics correctness if the integer is not exactly representable 1726 // here. See ExpandLegalINT_TO_FP. 1727 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 1728 SrcVT = Src.getValueType(); 1729 1730 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. 1731 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 1732 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 1733 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; 1734 ArrayRef<uint64_t> Parts; 1735 1736 switch (SrcVT.getSimpleVT().SimpleTy) { 1737 default: 1738 llvm_unreachable("Unsupported UINT_TO_FP!"); 1739 case MVT::i32: 1740 Parts = TwoE32; 1741 break; 1742 case MVT::i64: 1743 Parts = TwoE64; 1744 break; 1745 case MVT::i128: 1746 Parts = TwoE128; 1747 break; 1748 } 1749 1750 // TODO: Are there other fast-math-flags to propagate to this FADD? 1751 SDValue NewLo = DAG.getConstantFP( 1752 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128); 1753 if (Strict) { 1754 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other), 1755 {Chain, Hi, NewLo}, Flags); 1756 Chain = Lo.getValue(1); 1757 ReplaceValueWith(SDValue(N, 1), Chain); 1758 } else 1759 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo); 1760 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT), 1761 Lo, Hi, ISD::SETLT); 1762 GetPairElements(Lo, Lo, Hi); 1763 } 1764 1765 1766 //===----------------------------------------------------------------------===// 1767 // Float Operand Expansion 1768 //===----------------------------------------------------------------------===// 1769 1770 /// ExpandFloatOperand - This method is called when the specified operand of the 1771 /// specified node is found to need expansion. At this point, all of the result 1772 /// types of the node are known to be legal, but other operands of the node may 1773 /// need promotion or expansion as well as the specified one. 1774 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { 1775 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n"); 1776 SDValue Res = SDValue(); 1777 1778 // See if the target wants to custom expand this node. 1779 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1780 return false; 1781 1782 switch (N->getOpcode()) { 1783 default: 1784 #ifndef NDEBUG 1785 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": "; 1786 N->dump(&DAG); dbgs() << "\n"; 1787 #endif 1788 llvm_unreachable("Do not know how to expand this operator's operand!"); 1789 1790 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 1791 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 1792 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 1793 1794 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; 1795 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; 1796 case ISD::STRICT_FP_ROUND: 1797 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; 1798 case ISD::STRICT_FP_TO_SINT: 1799 case ISD::STRICT_FP_TO_UINT: 1800 case ISD::FP_TO_SINT: 1801 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break; 1802 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; 1803 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; 1804 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; 1805 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; 1806 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; 1807 case ISD::STRICT_FSETCC: 1808 case ISD::STRICT_FSETCCS: 1809 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; 1810 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N), 1811 OpNo); break; 1812 } 1813 1814 // If the result is null, the sub-method took care of registering results etc. 1815 if (!Res.getNode()) return false; 1816 1817 // If the result is N, the sub-method updated N in place. Tell the legalizer 1818 // core about this. 1819 if (Res.getNode() == N) 1820 return true; 1821 1822 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1823 "Invalid operand expansion"); 1824 1825 ReplaceValueWith(SDValue(N, 0), Res); 1826 return false; 1827 } 1828 1829 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code 1830 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 1831 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, 1832 SDValue &NewRHS, 1833 ISD::CondCode &CCCode, 1834 const SDLoc &dl, SDValue &Chain, 1835 bool IsSignaling) { 1836 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 1837 GetExpandedFloat(NewLHS, LHSLo, LHSHi); 1838 GetExpandedFloat(NewRHS, RHSLo, RHSHi); 1839 1840 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); 1841 1842 // FIXME: This generated code sucks. We want to generate 1843 // FCMPU crN, hi1, hi2 1844 // BNE crN, L: 1845 // FCMPU crN, lo1, lo2 1846 // The following can be improved, but not that much. 1847 SDValue Tmp1, Tmp2, Tmp3, OutputChain; 1848 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1849 RHSHi, ISD::SETOEQ, Chain, IsSignaling); 1850 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1851 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 1852 RHSLo, CCCode, OutputChain, IsSignaling); 1853 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1854 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1855 Tmp1 = 1856 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi, 1857 ISD::SETUNE, OutputChain, IsSignaling); 1858 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 1859 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 1860 RHSHi, CCCode, OutputChain, IsSignaling); 1861 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 1862 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 1863 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 1864 NewRHS = SDValue(); // LHS is the result, not a compare. 1865 Chain = OutputChain; 1866 } 1867 1868 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { 1869 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1870 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1871 SDValue Chain; 1872 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 1873 1874 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1875 // against zero to select between true and false values. 1876 if (!NewRHS.getNode()) { 1877 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1878 CCCode = ISD::SETNE; 1879 } 1880 1881 // Update N to have the operands specified. 1882 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1883 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1884 N->getOperand(4)), 0); 1885 } 1886 1887 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { 1888 assert(N->getOperand(1).getValueType() == MVT::ppcf128 && 1889 "Logic only correct for ppcf128!"); 1890 SDValue Lo, Hi; 1891 GetExpandedFloat(N->getOperand(1), Lo, Hi); 1892 // The ppcf128 value is providing only the sign; take it from the 1893 // higher-order double (which must have the larger magnitude). 1894 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), 1895 N->getValueType(0), N->getOperand(0), Hi); 1896 } 1897 1898 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { 1899 bool IsStrict = N->isStrictFPOpcode(); 1900 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && 1901 "Logic only correct for ppcf128!"); 1902 SDValue Lo, Hi; 1903 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi); 1904 1905 if (!IsStrict) 1906 // Round it the rest of the way (e.g. to f32) if needed. 1907 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 1908 N->getValueType(0), Hi, N->getOperand(1)); 1909 1910 // Eliminate the node if the input float type is the same as the output float 1911 // type. 1912 if (Hi.getValueType() == N->getValueType(0)) { 1913 // Connect the output chain to the input chain, unlinking the node. 1914 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 1915 ReplaceValueWith(SDValue(N, 0), Hi); 1916 return SDValue(); 1917 } 1918 1919 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 1920 {N->getValueType(0), MVT::Other}, 1921 {N->getOperand(0), Hi, N->getOperand(2)}); 1922 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1)); 1923 ReplaceValueWith(SDValue(N, 0), Expansion); 1924 return SDValue(); 1925 } 1926 1927 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) { 1928 EVT RVT = N->getValueType(0); 1929 SDLoc dl(N); 1930 1931 bool IsStrict = N->isStrictFPOpcode(); 1932 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 1933 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 1934 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1935 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1936 1937 EVT NVT; 1938 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed); 1939 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 1940 "Unsupported FP_TO_XINT!"); 1941 TargetLowering::MakeLibCallOptions CallOptions; 1942 std::pair<SDValue, SDValue> Tmp = 1943 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); 1944 if (!IsStrict) 1945 return Tmp.first; 1946 1947 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1948 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1949 return SDValue(); 1950 } 1951 1952 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { 1953 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1954 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1955 SDValue Chain; 1956 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 1957 1958 // If ExpandSetCCOperands returned a scalar, we need to compare the result 1959 // against zero to select between true and false values. 1960 if (!NewRHS.getNode()) { 1961 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1962 CCCode = ISD::SETNE; 1963 } 1964 1965 // Update N to have the operands specified. 1966 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1967 N->getOperand(2), N->getOperand(3), 1968 DAG.getCondCode(CCCode)), 0); 1969 } 1970 1971 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { 1972 bool IsStrict = N->isStrictFPOpcode(); 1973 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0); 1974 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1); 1975 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1976 ISD::CondCode CCCode = 1977 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1978 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain, 1979 N->getOpcode() == ISD::STRICT_FSETCCS); 1980 1981 // FloatExpandSetCCOperands always returned a scalar. 1982 assert(!NewRHS.getNode() && "Expect to return scalar"); 1983 assert(NewLHS.getValueType() == N->getValueType(0) && 1984 "Unexpected setcc expansion!"); 1985 if (Chain) { 1986 ReplaceValueWith(SDValue(N, 0), NewLHS); 1987 ReplaceValueWith(SDValue(N, 1), Chain); 1988 return SDValue(); 1989 } 1990 return NewLHS; 1991 } 1992 1993 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { 1994 if (ISD::isNormalStore(N)) 1995 return ExpandOp_NormalStore(N, OpNo); 1996 1997 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1998 assert(OpNo == 1 && "Can only expand the stored value so far"); 1999 StoreSDNode *ST = cast<StoreSDNode>(N); 2000 2001 SDValue Chain = ST->getChain(); 2002 SDValue Ptr = ST->getBasePtr(); 2003 2004 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 2005 ST->getValue().getValueType()); 2006 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2007 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 2008 (void)NVT; 2009 2010 SDValue Lo, Hi; 2011 GetExpandedOp(ST->getValue(), Lo, Hi); 2012 2013 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr, 2014 ST->getMemoryVT(), ST->getMemOperand()); 2015 } 2016 2017 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { 2018 EVT RVT = N->getValueType(0); 2019 EVT RetVT = N->getOperand(0).getValueType(); 2020 TargetLowering::MakeLibCallOptions CallOptions; 2021 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2022 RTLIB::LROUND_F32, 2023 RTLIB::LROUND_F64, 2024 RTLIB::LROUND_F80, 2025 RTLIB::LROUND_F128, 2026 RTLIB::LROUND_PPCF128), 2027 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2028 } 2029 2030 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { 2031 EVT RVT = N->getValueType(0); 2032 EVT RetVT = N->getOperand(0).getValueType(); 2033 TargetLowering::MakeLibCallOptions CallOptions; 2034 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2035 RTLIB::LLROUND_F32, 2036 RTLIB::LLROUND_F64, 2037 RTLIB::LLROUND_F80, 2038 RTLIB::LLROUND_F128, 2039 RTLIB::LLROUND_PPCF128), 2040 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2041 } 2042 2043 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { 2044 EVT RVT = N->getValueType(0); 2045 EVT RetVT = N->getOperand(0).getValueType(); 2046 TargetLowering::MakeLibCallOptions CallOptions; 2047 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2048 RTLIB::LRINT_F32, 2049 RTLIB::LRINT_F64, 2050 RTLIB::LRINT_F80, 2051 RTLIB::LRINT_F128, 2052 RTLIB::LRINT_PPCF128), 2053 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2054 } 2055 2056 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { 2057 EVT RVT = N->getValueType(0); 2058 EVT RetVT = N->getOperand(0).getValueType(); 2059 TargetLowering::MakeLibCallOptions CallOptions; 2060 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2061 RTLIB::LLRINT_F32, 2062 RTLIB::LLRINT_F64, 2063 RTLIB::LLRINT_F80, 2064 RTLIB::LLRINT_F128, 2065 RTLIB::LLRINT_PPCF128), 2066 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2067 } 2068 2069 //===----------------------------------------------------------------------===// 2070 // Float Operand Promotion 2071 //===----------------------------------------------------------------------===// 2072 // 2073 2074 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { 2075 if (OpVT == MVT::f16) { 2076 return ISD::FP16_TO_FP; 2077 } else if (RetVT == MVT::f16) { 2078 return ISD::FP_TO_FP16; 2079 } else if (OpVT == MVT::bf16) { 2080 return ISD::BF16_TO_FP; 2081 } else if (RetVT == MVT::bf16) { 2082 return ISD::FP_TO_BF16; 2083 } 2084 2085 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2086 } 2087 2088 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { 2089 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG); 2090 dbgs() << "\n"); 2091 SDValue R = SDValue(); 2092 2093 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2094 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2095 return false; 2096 } 2097 2098 // Nodes that use a promotion-requiring floating point operand, but doesn't 2099 // produce a promotion-requiring floating point result, need to be legalized 2100 // to use the promoted float operand. Nodes that produce at least one 2101 // promotion-requiring floating point result have their operands legalized as 2102 // a part of PromoteFloatResult. 2103 switch (N->getOpcode()) { 2104 default: 2105 #ifndef NDEBUG 2106 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; 2107 N->dump(&DAG); dbgs() << "\n"; 2108 #endif 2109 llvm_unreachable("Do not know how to promote this operator's operand!"); 2110 2111 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; 2112 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; 2113 case ISD::FP_TO_SINT: 2114 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break; 2115 case ISD::FP_TO_SINT_SAT: 2116 case ISD::FP_TO_UINT_SAT: 2117 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; 2118 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; 2119 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; 2120 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; 2121 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; 2122 } 2123 2124 if (R.getNode()) 2125 ReplaceValueWith(SDValue(N, 0), R); 2126 return false; 2127 } 2128 2129 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { 2130 SDValue Op = N->getOperand(0); 2131 EVT OpVT = Op->getValueType(0); 2132 2133 SDValue Promoted = GetPromotedFloat(N->getOperand(0)); 2134 EVT PromotedVT = Promoted->getValueType(0); 2135 2136 // Convert the promoted float value to the desired IVT. 2137 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); 2138 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), 2139 IVT, Promoted); 2140 // The final result type might not be an scalar so we need a bitcast. The 2141 // bitcast will be further legalized if needed. 2142 return DAG.getBitcast(N->getValueType(0), Convert); 2143 } 2144 2145 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by 2146 // PromoteFloatRes_FCOPYSIGN. 2147 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { 2148 assert (OpNo == 1 && "Only Operand 1 must need promotion here"); 2149 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2150 2151 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2152 N->getOperand(0), Op1); 2153 } 2154 2155 // Convert the promoted float value to the desired integer type 2156 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) { 2157 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2158 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); 2159 } 2160 2161 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, 2162 unsigned OpNo) { 2163 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2164 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, 2165 N->getOperand(1)); 2166 } 2167 2168 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { 2169 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2170 EVT VT = N->getValueType(0); 2171 2172 // Desired VT is same as promoted type. Use promoted float directly. 2173 if (VT == Op->getValueType(0)) 2174 return Op; 2175 2176 // Else, extend the promoted float value to the desired VT. 2177 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); 2178 } 2179 2180 // Promote the float operands used for comparison. The true- and false- 2181 // operands have the same type as the result and are promoted, if needed, by 2182 // PromoteFloatRes_SELECT_CC 2183 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2184 SDValue LHS = GetPromotedFloat(N->getOperand(0)); 2185 SDValue RHS = GetPromotedFloat(N->getOperand(1)); 2186 2187 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 2188 LHS, RHS, N->getOperand(2), N->getOperand(3), 2189 N->getOperand(4)); 2190 } 2191 2192 // Construct a SETCC that compares the promoted values and sets the conditional 2193 // code. 2194 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { 2195 EVT VT = N->getValueType(0); 2196 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2197 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2198 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2199 2200 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); 2201 2202 } 2203 2204 // Lower the promoted Float down to the integer value of same size and construct 2205 // a STORE of the integer value. 2206 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { 2207 StoreSDNode *ST = cast<StoreSDNode>(N); 2208 SDValue Val = ST->getValue(); 2209 SDLoc DL(N); 2210 2211 SDValue Promoted = GetPromotedFloat(Val); 2212 EVT VT = ST->getOperand(1).getValueType(); 2213 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2214 2215 SDValue NewVal; 2216 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, 2217 IVT, Promoted); 2218 2219 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), 2220 ST->getMemOperand()); 2221 } 2222 2223 //===----------------------------------------------------------------------===// 2224 // Float Result Promotion 2225 //===----------------------------------------------------------------------===// 2226 2227 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { 2228 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG); 2229 dbgs() << "\n"); 2230 SDValue R = SDValue(); 2231 2232 // See if the target wants to custom expand this node. 2233 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2234 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2235 return; 2236 } 2237 2238 switch (N->getOpcode()) { 2239 // These opcodes cannot appear if promotion of FP16 is done in the backend 2240 // instead of Clang 2241 case ISD::FP16_TO_FP: 2242 case ISD::FP_TO_FP16: 2243 default: 2244 #ifndef NDEBUG 2245 dbgs() << "PromoteFloatResult #" << ResNo << ": "; 2246 N->dump(&DAG); dbgs() << "\n"; 2247 #endif 2248 llvm_unreachable("Do not know how to promote this operator's result!"); 2249 2250 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; 2251 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; 2252 case ISD::EXTRACT_VECTOR_ELT: 2253 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; 2254 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; 2255 2256 // Unary FP Operations 2257 case ISD::FABS: 2258 case ISD::FCBRT: 2259 case ISD::FCEIL: 2260 case ISD::FCOS: 2261 case ISD::FEXP: 2262 case ISD::FEXP2: 2263 case ISD::FFLOOR: 2264 case ISD::FLOG: 2265 case ISD::FLOG2: 2266 case ISD::FLOG10: 2267 case ISD::FNEARBYINT: 2268 case ISD::FNEG: 2269 case ISD::FRINT: 2270 case ISD::FROUND: 2271 case ISD::FROUNDEVEN: 2272 case ISD::FSIN: 2273 case ISD::FSQRT: 2274 case ISD::FTRUNC: 2275 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; 2276 2277 // Binary FP Operations 2278 case ISD::FADD: 2279 case ISD::FDIV: 2280 case ISD::FMAXIMUM: 2281 case ISD::FMINIMUM: 2282 case ISD::FMAXNUM: 2283 case ISD::FMINNUM: 2284 case ISD::FMUL: 2285 case ISD::FPOW: 2286 case ISD::FREM: 2287 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; 2288 2289 case ISD::FMA: // FMA is same as FMAD 2290 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; 2291 2292 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break; 2293 2294 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; 2295 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; 2296 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; 2297 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; 2298 2299 case ISD::SINT_TO_FP: 2300 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; 2301 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; 2302 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2303 case ISD::VECREDUCE_FADD: 2304 case ISD::VECREDUCE_FMUL: 2305 case ISD::VECREDUCE_FMIN: 2306 case ISD::VECREDUCE_FMAX: 2307 R = PromoteFloatRes_VECREDUCE(N); 2308 break; 2309 case ISD::VECREDUCE_SEQ_FADD: 2310 case ISD::VECREDUCE_SEQ_FMUL: 2311 R = PromoteFloatRes_VECREDUCE_SEQ(N); 2312 break; 2313 } 2314 2315 if (R.getNode()) 2316 SetPromotedFloat(SDValue(N, ResNo), R); 2317 } 2318 2319 // Bitcast from i16 to f16: convert the i16 to a f32 value instead. 2320 // At this point, it is not possible to determine if the bitcast value is 2321 // eventually stored to memory or promoted to f32 or promoted to a floating 2322 // point at a higher precision. Some of these cases are handled by FP_EXTEND, 2323 // STORE promotion handlers. 2324 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { 2325 EVT VT = N->getValueType(0); 2326 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2327 // Input type isn't guaranteed to be a scalar int so bitcast if not. The 2328 // bitcast will be legalized further if necessary. 2329 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), 2330 N->getOperand(0).getValueType().getSizeInBits()); 2331 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); 2332 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); 2333 } 2334 2335 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { 2336 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); 2337 EVT VT = N->getValueType(0); 2338 SDLoc DL(N); 2339 2340 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2341 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2342 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, 2343 IVT); 2344 2345 // Convert the Constant to the desired FP type 2346 // FIXME We might be able to do the conversion during compilation and get rid 2347 // of it from the object code 2348 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2349 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); 2350 } 2351 2352 // If the Index operand is a constant, try to redirect the extract operation to 2353 // the correct legalized vector. If not, bit-convert the input vector to 2354 // equivalent integer vector. Extract the element as an (bit-cast) integer 2355 // value and convert it to the promoted type. 2356 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2357 SDLoc DL(N); 2358 2359 // If the index is constant, try to extract the value from the legalized 2360 // vector type. 2361 if (isa<ConstantSDNode>(N->getOperand(1))) { 2362 SDValue Vec = N->getOperand(0); 2363 SDValue Idx = N->getOperand(1); 2364 EVT VecVT = Vec->getValueType(0); 2365 EVT EltVT = VecVT.getVectorElementType(); 2366 2367 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2368 2369 switch (getTypeAction(VecVT)) { 2370 default: break; 2371 case TargetLowering::TypeScalarizeVector: { 2372 SDValue Res = GetScalarizedVector(N->getOperand(0)); 2373 ReplaceValueWith(SDValue(N, 0), Res); 2374 return SDValue(); 2375 } 2376 case TargetLowering::TypeWidenVector: { 2377 Vec = GetWidenedVector(Vec); 2378 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); 2379 ReplaceValueWith(SDValue(N, 0), Res); 2380 return SDValue(); 2381 } 2382 case TargetLowering::TypeSplitVector: { 2383 SDValue Lo, Hi; 2384 GetSplitVector(Vec, Lo, Hi); 2385 2386 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2387 SDValue Res; 2388 if (IdxVal < LoElts) 2389 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); 2390 else 2391 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, 2392 DAG.getConstant(IdxVal - LoElts, DL, 2393 Idx.getValueType())); 2394 ReplaceValueWith(SDValue(N, 0), Res); 2395 return SDValue(); 2396 } 2397 2398 } 2399 } 2400 2401 // Bit-convert the input vector to the equivalent integer vector 2402 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2403 EVT IVT = NewOp.getValueType().getVectorElementType(); 2404 2405 // Extract the element as an (bit-cast) integer value 2406 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, 2407 NewOp, N->getOperand(1)); 2408 2409 // Convert the element to the desired FP type 2410 EVT VT = N->getValueType(0); 2411 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2412 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); 2413 } 2414 2415 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result 2416 // needs promotion, so does the argument X. Note that Y, if needed, will be 2417 // handled during operand promotion. 2418 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { 2419 EVT VT = N->getValueType(0); 2420 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2421 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2422 2423 SDValue Op1 = N->getOperand(1); 2424 2425 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2426 } 2427 2428 // Unary operation where the result and the operand have PromoteFloat type 2429 // action. Construct a new SDNode with the promoted float value of the old 2430 // operand. 2431 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { 2432 EVT VT = N->getValueType(0); 2433 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2434 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2435 2436 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); 2437 } 2438 2439 // Binary operations where the result and both operands have PromoteFloat type 2440 // action. Construct a new SDNode with the promoted float values of the old 2441 // operands. 2442 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { 2443 EVT VT = N->getValueType(0); 2444 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2445 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2446 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2447 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); 2448 } 2449 2450 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { 2451 EVT VT = N->getValueType(0); 2452 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2453 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2454 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2455 SDValue Op2 = GetPromotedFloat(N->getOperand(2)); 2456 2457 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); 2458 } 2459 2460 // Promote the Float (first) operand and retain the Integer (second) operand 2461 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) { 2462 EVT VT = N->getValueType(0); 2463 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2464 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2465 SDValue Op1 = N->getOperand(1); 2466 2467 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2468 } 2469 2470 // Explicit operation to reduce precision. Reduce the value to half precision 2471 // and promote it back to the legal type. 2472 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { 2473 SDLoc DL(N); 2474 2475 SDValue Op = N->getOperand(0); 2476 EVT VT = N->getValueType(0); 2477 EVT OpVT = Op->getValueType(0); 2478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2479 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2480 2481 // Round promoted float to desired precision 2482 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); 2483 // Promote it back to the legal output type 2484 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); 2485 } 2486 2487 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { 2488 LoadSDNode *L = cast<LoadSDNode>(N); 2489 EVT VT = N->getValueType(0); 2490 2491 // Load the value as an integer value with the same number of bits. 2492 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2493 SDValue newL = DAG.getLoad( 2494 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), 2495 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, 2496 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); 2497 // Legalize the chain result by replacing uses of the old value chain with the 2498 // new one 2499 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 2500 2501 // Convert the integer value to the desired FP type 2502 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2503 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); 2504 } 2505 2506 // Construct a new SELECT node with the promoted true- and false- values. 2507 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { 2508 SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); 2509 SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); 2510 2511 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), 2512 N->getOperand(0), TrueVal, FalseVal); 2513 } 2514 2515 // Construct a new SELECT_CC node with the promoted true- and false- values. 2516 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. 2517 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { 2518 SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); 2519 SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); 2520 2521 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2522 TrueVal.getNode()->getValueType(0), N->getOperand(0), 2523 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); 2524 } 2525 2526 // Construct a SDNode that transforms the SINT or UINT operand to the promoted 2527 // float type. 2528 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { 2529 SDLoc DL(N); 2530 EVT VT = N->getValueType(0); 2531 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2532 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); 2533 // Round the value to the desired precision (that of the source type). 2534 return DAG.getNode( 2535 ISD::FP_EXTEND, DL, NVT, 2536 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, 2537 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true))); 2538 } 2539 2540 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { 2541 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 2542 N->getValueType(0))); 2543 } 2544 2545 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { 2546 // Expand and promote recursively. 2547 // TODO: This is non-optimal, but dealing with the concurrently happening 2548 // vector-legalization is non-trivial. We could do something similar to 2549 // PromoteFloatRes_EXTRACT_VECTOR_ELT here. 2550 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2551 return SDValue(); 2552 } 2553 2554 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { 2555 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2556 return SDValue(); 2557 } 2558 2559 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { 2560 EVT VT = N->getValueType(0); 2561 2562 AtomicSDNode *AM = cast<AtomicSDNode>(N); 2563 SDLoc SL(N); 2564 2565 SDValue CastVal = BitConvertToInteger(AM->getVal()); 2566 EVT CastVT = CastVal.getValueType(); 2567 2568 SDValue NewAtomic 2569 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, 2570 DAG.getVTList(CastVT, MVT::Other), 2571 { AM->getChain(), AM->getBasePtr(), CastVal }, 2572 AM->getMemOperand()); 2573 2574 SDValue Result = NewAtomic; 2575 2576 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { 2577 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2578 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, 2579 NewAtomic); 2580 } 2581 2582 // Legalize the chain result by replacing uses of the old value chain with the 2583 // new one 2584 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); 2585 2586 return Result; 2587 2588 } 2589 2590 //===----------------------------------------------------------------------===// 2591 // Half Result Soft Promotion 2592 //===----------------------------------------------------------------------===// 2593 2594 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { 2595 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": "; 2596 N->dump(&DAG); dbgs() << "\n"); 2597 SDValue R = SDValue(); 2598 2599 // See if the target wants to custom expand this node. 2600 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2601 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2602 return; 2603 } 2604 2605 switch (N->getOpcode()) { 2606 default: 2607 #ifndef NDEBUG 2608 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": "; 2609 N->dump(&DAG); dbgs() << "\n"; 2610 #endif 2611 llvm_unreachable("Do not know how to soft promote this operator's result!"); 2612 2613 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; 2614 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; 2615 case ISD::EXTRACT_VECTOR_ELT: 2616 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; 2617 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; 2618 case ISD::STRICT_FP_ROUND: 2619 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; 2620 2621 // Unary FP Operations 2622 case ISD::FABS: 2623 case ISD::FCBRT: 2624 case ISD::FCEIL: 2625 case ISD::FCOS: 2626 case ISD::FEXP: 2627 case ISD::FEXP2: 2628 case ISD::FFLOOR: 2629 case ISD::FLOG: 2630 case ISD::FLOG2: 2631 case ISD::FLOG10: 2632 case ISD::FNEARBYINT: 2633 case ISD::FNEG: 2634 case ISD::FREEZE: 2635 case ISD::FRINT: 2636 case ISD::FROUND: 2637 case ISD::FROUNDEVEN: 2638 case ISD::FSIN: 2639 case ISD::FSQRT: 2640 case ISD::FTRUNC: 2641 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; 2642 2643 // Binary FP Operations 2644 case ISD::FADD: 2645 case ISD::FDIV: 2646 case ISD::FMAXIMUM: 2647 case ISD::FMINIMUM: 2648 case ISD::FMAXNUM: 2649 case ISD::FMINNUM: 2650 case ISD::FMUL: 2651 case ISD::FPOW: 2652 case ISD::FREM: 2653 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; 2654 2655 case ISD::FMA: // FMA is same as FMAD 2656 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; 2657 2658 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break; 2659 2660 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; 2661 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; 2662 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; 2663 case ISD::SINT_TO_FP: 2664 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; 2665 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; 2666 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2667 case ISD::VECREDUCE_FADD: 2668 case ISD::VECREDUCE_FMUL: 2669 case ISD::VECREDUCE_FMIN: 2670 case ISD::VECREDUCE_FMAX: 2671 R = SoftPromoteHalfRes_VECREDUCE(N); 2672 break; 2673 case ISD::VECREDUCE_SEQ_FADD: 2674 case ISD::VECREDUCE_SEQ_FMUL: 2675 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N); 2676 break; 2677 } 2678 2679 if (R.getNode()) 2680 SetSoftPromotedHalf(SDValue(N, ResNo), R); 2681 } 2682 2683 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { 2684 return BitConvertToInteger(N->getOperand(0)); 2685 } 2686 2687 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { 2688 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 2689 2690 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2691 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 2692 MVT::i16); 2693 } 2694 2695 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2696 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2697 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2698 NewOp.getValueType().getVectorElementType(), NewOp, 2699 N->getOperand(1)); 2700 } 2701 2702 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { 2703 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0)); 2704 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 2705 SDLoc dl(N); 2706 2707 EVT LVT = LHS.getValueType(); 2708 EVT RVT = RHS.getValueType(); 2709 2710 unsigned LSize = LVT.getSizeInBits(); 2711 unsigned RSize = RVT.getSizeInBits(); 2712 2713 // First get the sign bit of second operand. 2714 SDValue SignBit = DAG.getNode( 2715 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 2716 DAG.getConstant(RSize - 1, dl, 2717 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 2718 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 2719 2720 // Shift right or sign-extend it if the two operands have different types. 2721 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 2722 if (SizeDiff > 0) { 2723 SignBit = 2724 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 2725 DAG.getConstant(SizeDiff, dl, 2726 TLI.getShiftAmountTy(SignBit.getValueType(), 2727 DAG.getDataLayout()))); 2728 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 2729 } else if (SizeDiff < 0) { 2730 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 2731 SignBit = 2732 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 2733 DAG.getConstant(-SizeDiff, dl, 2734 TLI.getShiftAmountTy(SignBit.getValueType(), 2735 DAG.getDataLayout()))); 2736 } 2737 2738 // Clear the sign bit of the first operand. 2739 SDValue Mask = DAG.getNode( 2740 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 2741 DAG.getConstant(LSize - 1, dl, 2742 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 2743 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 2744 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 2745 2746 // Or the value with the sign bit. 2747 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 2748 } 2749 2750 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { 2751 EVT OVT = N->getValueType(0); 2752 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2753 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2754 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2755 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2756 SDLoc dl(N); 2757 2758 // Promote to the larger FP type. 2759 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 2760 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 2761 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 2762 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2); 2763 2764 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2); 2765 2766 // Convert back to FP16 as an integer. 2767 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2768 } 2769 2770 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) { 2771 EVT OVT = N->getValueType(0); 2772 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2773 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2774 SDValue Op1 = N->getOperand(1); 2775 SDLoc dl(N); 2776 2777 // Promote to the larger FP type. 2778 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0); 2779 2780 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2781 2782 // Convert back to FP16 as an integer. 2783 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2784 } 2785 2786 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { 2787 EVT RVT = N->getValueType(0); 2788 EVT SVT = N->getOperand(0).getValueType(); 2789 2790 if (N->isStrictFPOpcode()) { 2791 assert(RVT == MVT::f16); 2792 SDValue Res = 2793 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other}, 2794 {N->getOperand(0), N->getOperand(1)}); 2795 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2796 return Res; 2797 } 2798 2799 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16, 2800 N->getOperand(0)); 2801 } 2802 2803 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { 2804 LoadSDNode *L = cast<LoadSDNode>(N); 2805 2806 // Load the value as an integer value with the same number of bits. 2807 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!"); 2808 SDValue NewL = 2809 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, 2810 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), 2811 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), 2812 L->getMemOperand()->getFlags(), L->getAAInfo()); 2813 // Legalize the chain result by replacing uses of the old value chain with the 2814 // new one 2815 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 2816 return NewL; 2817 } 2818 2819 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { 2820 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2821 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2822 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1, 2823 Op2); 2824 } 2825 2826 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { 2827 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 2828 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3)); 2829 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(), 2830 N->getOperand(0), N->getOperand(1), Op2, Op3, 2831 N->getOperand(4)); 2832 } 2833 2834 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { 2835 EVT OVT = N->getValueType(0); 2836 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2837 SDLoc dl(N); 2838 2839 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 2840 2841 // Round the value to the softened type. 2842 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2843 } 2844 2845 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { 2846 return DAG.getUNDEF(MVT::i16); 2847 } 2848 2849 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { 2850 EVT OVT = N->getValueType(0); 2851 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2852 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 2853 SDLoc dl(N); 2854 2855 // Promote to the larger FP type. 2856 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 2857 2858 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op); 2859 2860 // Convert back to FP16 as an integer. 2861 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2862 } 2863 2864 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { 2865 EVT OVT = N->getValueType(0); 2866 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 2867 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2868 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 2869 SDLoc dl(N); 2870 2871 // Promote to the larger FP type. 2872 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 2873 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 2874 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 2875 2876 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 2877 2878 // Convert back to FP16 as an integer. 2879 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 2880 } 2881 2882 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) { 2883 // Expand and soften recursively. 2884 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2885 return SDValue(); 2886 } 2887 2888 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) { 2889 // Expand and soften. 2890 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2891 return SDValue(); 2892 } 2893 2894 //===----------------------------------------------------------------------===// 2895 // Half Operand Soft Promotion 2896 //===----------------------------------------------------------------------===// 2897 2898 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { 2899 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": "; 2900 N->dump(&DAG); dbgs() << "\n"); 2901 SDValue Res = SDValue(); 2902 2903 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2904 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2905 return false; 2906 } 2907 2908 // Nodes that use a promotion-requiring floating point operand, but doesn't 2909 // produce a soft promotion-requiring floating point result, need to be 2910 // legalized to use the soft promoted float operand. Nodes that produce at 2911 // least one soft promotion-requiring floating point result have their 2912 // operands legalized as a part of PromoteFloatResult. 2913 switch (N->getOpcode()) { 2914 default: 2915 #ifndef NDEBUG 2916 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": "; 2917 N->dump(&DAG); dbgs() << "\n"; 2918 #endif 2919 llvm_unreachable("Do not know how to soft promote this operator's operand!"); 2920 2921 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; 2922 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; 2923 case ISD::FP_TO_SINT: 2924 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; 2925 case ISD::FP_TO_SINT_SAT: 2926 case ISD::FP_TO_UINT_SAT: 2927 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; 2928 case ISD::STRICT_FP_EXTEND: 2929 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; 2930 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; 2931 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; 2932 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; 2933 case ISD::STACKMAP: 2934 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); 2935 break; 2936 case ISD::PATCHPOINT: 2937 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo); 2938 break; 2939 } 2940 2941 if (!Res.getNode()) 2942 return false; 2943 2944 assert(Res.getNode() != N && "Expected a new node!"); 2945 2946 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2947 "Invalid operand expansion"); 2948 2949 ReplaceValueWith(SDValue(N, 0), Res); 2950 return false; 2951 } 2952 2953 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { 2954 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 2955 2956 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 2957 } 2958 2959 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, 2960 unsigned OpNo) { 2961 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 2962 SDValue Op1 = N->getOperand(1); 2963 EVT RVT = Op1.getValueType(); 2964 SDLoc dl(N); 2965 2966 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType()); 2967 2968 Op1 = GetSoftPromotedHalf(Op1); 2969 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1); 2970 2971 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0), 2972 Op1); 2973 } 2974 2975 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { 2976 EVT RVT = N->getValueType(0); 2977 bool IsStrict = N->isStrictFPOpcode(); 2978 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 2979 EVT SVT = Op.getValueType(); 2980 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0)); 2981 2982 if (IsStrict) { 2983 assert(SVT == MVT::f16); 2984 SDValue Res = 2985 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N), 2986 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op}); 2987 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2988 ReplaceValueWith(SDValue(N, 0), Res); 2989 return SDValue(); 2990 } 2991 2992 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op); 2993 } 2994 2995 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { 2996 EVT RVT = N->getValueType(0); 2997 SDValue Op = N->getOperand(0); 2998 EVT SVT = Op.getValueType(); 2999 SDLoc dl(N); 3000 3001 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3002 3003 Op = GetSoftPromotedHalf(Op); 3004 3005 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3006 3007 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res); 3008 } 3009 3010 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { 3011 EVT RVT = N->getValueType(0); 3012 SDValue Op = N->getOperand(0); 3013 EVT SVT = Op.getValueType(); 3014 SDLoc dl(N); 3015 3016 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3017 3018 Op = GetSoftPromotedHalf(Op); 3019 3020 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3021 3022 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res, 3023 N->getOperand(1)); 3024 } 3025 3026 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, 3027 unsigned OpNo) { 3028 assert(OpNo == 0 && "Can only soften the comparison values"); 3029 SDValue Op0 = N->getOperand(0); 3030 SDValue Op1 = N->getOperand(1); 3031 SDLoc dl(N); 3032 3033 EVT SVT = Op0.getValueType(); 3034 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT); 3035 3036 Op0 = GetSoftPromotedHalf(Op0); 3037 Op1 = GetSoftPromotedHalf(Op1); 3038 3039 // Promote to the larger FP type. 3040 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3041 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3042 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3043 3044 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1, 3045 N->getOperand(2), N->getOperand(3), N->getOperand(4)); 3046 } 3047 3048 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { 3049 SDValue Op0 = N->getOperand(0); 3050 SDValue Op1 = N->getOperand(1); 3051 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3052 SDLoc dl(N); 3053 3054 EVT SVT = Op0.getValueType(); 3055 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 3056 3057 Op0 = GetSoftPromotedHalf(Op0); 3058 Op1 = GetSoftPromotedHalf(Op1); 3059 3060 // Promote to the larger FP type. 3061 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3062 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3063 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3064 3065 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode); 3066 } 3067 3068 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { 3069 assert(OpNo == 1 && "Can only soften the stored value!"); 3070 StoreSDNode *ST = cast<StoreSDNode>(N); 3071 SDValue Val = ST->getValue(); 3072 SDLoc dl(N); 3073 3074 assert(!ST->isTruncatingStore() && "Unexpected truncating store."); 3075 SDValue Promoted = GetSoftPromotedHalf(Val); 3076 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), 3077 ST->getMemOperand()); 3078 } 3079 3080 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { 3081 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 3082 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3083 SDValue Op = N->getOperand(OpNo); 3084 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3085 SDValue NewNode = 3086 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3087 3088 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3089 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3090 3091 return SDValue(); // Signal that we replaced the node ourselves. 3092 } 3093 3094 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N, 3095 unsigned OpNo) { 3096 assert(OpNo >= 7); 3097 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3098 SDValue Op = N->getOperand(OpNo); 3099 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3100 SDValue NewNode = 3101 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3102 3103 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3104 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3105 3106 return SDValue(); // Signal that we replaced the node ourselves. 3107 } 3108