1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2005-2011 David Schultz <das@FreeBSD.ORG> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <fenv.h> 33 #include <float.h> 34 #include <math.h> 35 36 #include "math_private.h" 37 38 #ifdef USE_BUILTIN_FMA 39 double 40 fma(double x, double y, double z) 41 { 42 return (__builtin_fma(x, y, z)); 43 } 44 #else 45 /* 46 * A struct dd represents a floating-point number with twice the precision 47 * of a double. We maintain the invariant that "hi" stores the 53 high-order 48 * bits of the result. 49 */ 50 struct dd { 51 double hi; 52 double lo; 53 }; 54 55 /* 56 * Compute a+b exactly, returning the exact result in a struct dd. We assume 57 * that both a and b are finite, but make no assumptions about their relative 58 * magnitudes. 59 */ 60 static inline struct dd 61 dd_add(double a, double b) 62 { 63 struct dd ret; 64 double s; 65 66 ret.hi = a + b; 67 s = ret.hi - a; 68 ret.lo = (a - (ret.hi - s)) + (b - s); 69 return (ret); 70 } 71 72 /* 73 * Compute a+b, with a small tweak: The least significant bit of the 74 * result is adjusted into a sticky bit summarizing all the bits that 75 * were lost to rounding. This adjustment negates the effects of double 76 * rounding when the result is added to another number with a higher 77 * exponent. For an explanation of round and sticky bits, see any reference 78 * on FPU design, e.g., 79 * 80 * J. Coonen. An Implementation Guide to a Proposed Standard for 81 * Floating-Point Arithmetic. Computer, vol. 13, no. 1, Jan 1980. 82 */ 83 static inline double 84 add_adjusted(double a, double b) 85 { 86 struct dd sum; 87 uint64_t hibits, lobits; 88 89 sum = dd_add(a, b); 90 if (sum.lo != 0) { 91 EXTRACT_WORD64(hibits, sum.hi); 92 if ((hibits & 1) == 0) { 93 /* hibits += (int)copysign(1.0, sum.hi * sum.lo) */ 94 EXTRACT_WORD64(lobits, sum.lo); 95 hibits += 1 - ((hibits ^ lobits) >> 62); 96 INSERT_WORD64(sum.hi, hibits); 97 } 98 } 99 return (sum.hi); 100 } 101 102 /* 103 * Compute ldexp(a+b, scale) with a single rounding error. It is assumed 104 * that the result will be subnormal, and care is taken to ensure that 105 * double rounding does not occur. 106 */ 107 static inline double 108 add_and_denormalize(double a, double b, int scale) 109 { 110 struct dd sum; 111 uint64_t hibits, lobits; 112 int bits_lost; 113 114 sum = dd_add(a, b); 115 116 /* 117 * If we are losing at least two bits of accuracy to denormalization, 118 * then the first lost bit becomes a round bit, and we adjust the 119 * lowest bit of sum.hi to make it a sticky bit summarizing all the 120 * bits in sum.lo. With the sticky bit adjusted, the hardware will 121 * break any ties in the correct direction. 122 * 123 * If we are losing only one bit to denormalization, however, we must 124 * break the ties manually. 125 */ 126 if (sum.lo != 0) { 127 EXTRACT_WORD64(hibits, sum.hi); 128 bits_lost = -((int)(hibits >> 52) & 0x7ff) - scale + 1; 129 if ((bits_lost != 1) ^ (int)(hibits & 1)) { 130 /* hibits += (int)copysign(1.0, sum.hi * sum.lo) */ 131 EXTRACT_WORD64(lobits, sum.lo); 132 hibits += 1 - (((hibits ^ lobits) >> 62) & 2); 133 INSERT_WORD64(sum.hi, hibits); 134 } 135 } 136 return (ldexp(sum.hi, scale)); 137 } 138 139 /* 140 * Compute a*b exactly, returning the exact result in a struct dd. We assume 141 * that both a and b are normalized, so no underflow or overflow will occur. 142 * The current rounding mode must be round-to-nearest. 143 */ 144 static inline struct dd 145 dd_mul(double a, double b) 146 { 147 static const double split = 0x1p27 + 1.0; 148 struct dd ret; 149 double ha, hb, la, lb, p, q; 150 151 p = a * split; 152 ha = a - p; 153 ha += p; 154 la = a - ha; 155 156 p = b * split; 157 hb = b - p; 158 hb += p; 159 lb = b - hb; 160 161 p = ha * hb; 162 q = ha * lb + la * hb; 163 164 ret.hi = p + q; 165 ret.lo = p - ret.hi + q + la * lb; 166 return (ret); 167 } 168 169 /* 170 * Fused multiply-add: Compute x * y + z with a single rounding error. 171 * 172 * We use scaling to avoid overflow/underflow, along with the 173 * canonical precision-doubling technique adapted from: 174 * 175 * Dekker, T. A Floating-Point Technique for Extending the 176 * Available Precision. Numer. Math. 18, 224-242 (1971). 177 * 178 * This algorithm is sensitive to the rounding precision. FPUs such 179 * as the i387 must be set in double-precision mode if variables are 180 * to be stored in FP registers in order to avoid incorrect results. 181 * This is the default on FreeBSD, but not on many other systems. 182 * 183 * Hardware instructions should be used on architectures that support it, 184 * since this implementation will likely be several times slower. 185 */ 186 double 187 fma(double x, double y, double z) 188 { 189 double xs, ys, zs, adj; 190 struct dd xy, r; 191 int oround; 192 int ex, ey, ez; 193 int spread; 194 195 /* 196 * Handle special cases. The order of operations and the particular 197 * return values here are crucial in handling special cases involving 198 * infinities, NaNs, overflows, and signed zeroes correctly. 199 */ 200 if (x == 0.0 || y == 0.0) 201 return (x * y + z); 202 if (z == 0.0) 203 return (x * y); 204 if (!isfinite(x) || !isfinite(y)) 205 return (x * y + z); 206 if (!isfinite(z)) 207 return (z); 208 209 xs = frexp(x, &ex); 210 ys = frexp(y, &ey); 211 zs = frexp(z, &ez); 212 oround = fegetround(); 213 spread = ex + ey - ez; 214 215 /* 216 * If x * y and z are many orders of magnitude apart, the scaling 217 * will overflow, so we handle these cases specially. Rounding 218 * modes other than FE_TONEAREST are painful. 219 */ 220 if (spread < -DBL_MANT_DIG) { 221 feraiseexcept(FE_INEXACT); 222 if (!isnormal(z)) 223 feraiseexcept(FE_UNDERFLOW); 224 switch (oround) { 225 case FE_TONEAREST: 226 return (z); 227 case FE_TOWARDZERO: 228 if (x > 0.0 ^ y < 0.0 ^ z < 0.0) 229 return (z); 230 else 231 return (nextafter(z, 0)); 232 case FE_DOWNWARD: 233 if (x > 0.0 ^ y < 0.0) 234 return (z); 235 else 236 return (nextafter(z, -INFINITY)); 237 default: /* FE_UPWARD */ 238 if (x > 0.0 ^ y < 0.0) 239 return (nextafter(z, INFINITY)); 240 else 241 return (z); 242 } 243 } 244 if (spread <= DBL_MANT_DIG * 2) 245 zs = ldexp(zs, -spread); 246 else 247 zs = copysign(DBL_MIN, zs); 248 249 fesetround(FE_TONEAREST); 250 /* work around clang bug 8100 */ 251 volatile double vxs = xs; 252 253 /* 254 * Basic approach for round-to-nearest: 255 * 256 * (xy.hi, xy.lo) = x * y (exact) 257 * (r.hi, r.lo) = xy.hi + z (exact) 258 * adj = xy.lo + r.lo (inexact; low bit is sticky) 259 * result = r.hi + adj (correctly rounded) 260 */ 261 xy = dd_mul(vxs, ys); 262 r = dd_add(xy.hi, zs); 263 264 spread = ex + ey; 265 266 if (r.hi == 0.0) { 267 /* 268 * When the addends cancel to 0, ensure that the result has 269 * the correct sign. 270 */ 271 fesetround(oround); 272 volatile double vzs = zs; /* XXX gcc CSE bug workaround */ 273 return (xy.hi + vzs + ldexp(xy.lo, spread)); 274 } 275 276 if (oround != FE_TONEAREST) { 277 /* 278 * There is no need to worry about double rounding in directed 279 * rounding modes. 280 */ 281 fesetround(oround); 282 /* work around clang bug 8100 */ 283 volatile double vrlo = r.lo; 284 adj = vrlo + xy.lo; 285 return (ldexp(r.hi + adj, spread)); 286 } 287 288 adj = add_adjusted(r.lo, xy.lo); 289 if (spread + ilogb(r.hi) > -1023) 290 return (ldexp(r.hi + adj, spread)); 291 else 292 return (add_and_denormalize(r.hi, adj, spread)); 293 } 294 #endif /* !USE_BUILTIN_FMA */ 295 296 #if (LDBL_MANT_DIG == 53) 297 __weak_reference(fma, fmal); 298 #endif 299