1 //===-- lib/fp_lib.h - Floating-point utilities -------------------*- C -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a configuration header for soft-float routines in compiler-rt.
10 // This file does not provide any part of the compiler-rt interface, but defines
11 // many useful constants and utility routines that are used in the
12 // implementation of the soft-float routines in compiler-rt.
13 //
14 // Assumes that float, double and long double correspond to the IEEE-754
15 // binary32, binary64 and binary 128 types, respectively, and that integer
16 // endianness matches floating point endianness on the target platform.
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #ifndef FP_LIB_HEADER
21 #define FP_LIB_HEADER
22 
23 #include "int_lib.h"
24 #include "int_math.h"
25 #include <limits.h>
26 #include <stdbool.h>
27 #include <stdint.h>
28 
29 // x86_64 FreeBSD prior v9.3 define fixed-width types incorrectly in
30 // 32-bit mode.
31 #if defined(__FreeBSD__) && defined(__i386__)
32 #include <sys/param.h>
33 #if __FreeBSD_version < 903000 // v9.3
34 #define uint64_t unsigned long long
35 #define int64_t long long
36 #undef UINT64_C
37 #define UINT64_C(c) (c##ULL)
38 #endif
39 #endif
40 
41 #if defined SINGLE_PRECISION
42 
43 typedef uint32_t rep_t;
44 typedef int32_t srep_t;
45 typedef float fp_t;
46 #define REP_C UINT32_C
47 #define significandBits 23
48 
49 static __inline int rep_clz(rep_t a) { return __builtin_clz(a); }
50 
51 // 32x32 --> 64 bit multiply
52 static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {
53   const uint64_t product = (uint64_t)a * b;
54   *hi = product >> 32;
55   *lo = product;
56 }
57 COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b);
58 
59 #elif defined DOUBLE_PRECISION
60 
61 typedef uint64_t rep_t;
62 typedef int64_t srep_t;
63 typedef double fp_t;
64 #define REP_C UINT64_C
65 #define significandBits 52
66 
67 static __inline int rep_clz(rep_t a) {
68 #if defined __LP64__
69   return __builtin_clzl(a);
70 #else
71   if (a & REP_C(0xffffffff00000000))
72     return __builtin_clz(a >> 32);
73   else
74     return 32 + __builtin_clz(a & REP_C(0xffffffff));
75 #endif
76 }
77 
78 #define loWord(a) (a & 0xffffffffU)
79 #define hiWord(a) (a >> 32)
80 
81 // 64x64 -> 128 wide multiply for platforms that don't have such an operation;
82 // many 64-bit platforms have this operation, but they tend to have hardware
83 // floating-point, so we don't bother with a special case for them here.
84 static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {
85   // Each of the component 32x32 -> 64 products
86   const uint64_t plolo = loWord(a) * loWord(b);
87   const uint64_t plohi = loWord(a) * hiWord(b);
88   const uint64_t philo = hiWord(a) * loWord(b);
89   const uint64_t phihi = hiWord(a) * hiWord(b);
90   // Sum terms that contribute to lo in a way that allows us to get the carry
91   const uint64_t r0 = loWord(plolo);
92   const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo);
93   *lo = r0 + (r1 << 32);
94   // Sum terms contributing to hi with the carry from lo
95   *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi;
96 }
97 #undef loWord
98 #undef hiWord
99 
100 COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b);
101 
102 #elif defined QUAD_PRECISION
103 #if __LDBL_MANT_DIG__ == 113 && defined(__SIZEOF_INT128__)
104 #define CRT_LDBL_128BIT
105 typedef __uint128_t rep_t;
106 typedef __int128_t srep_t;
107 typedef long double fp_t;
108 #define REP_C (__uint128_t)
109 // Note: Since there is no explicit way to tell compiler the constant is a
110 // 128-bit integer, we let the constant be casted to 128-bit integer
111 #define significandBits 112
112 
113 static __inline int rep_clz(rep_t a) {
114   const union {
115     __uint128_t ll;
116 #if _YUGA_BIG_ENDIAN
117     struct {
118       uint64_t high, low;
119     } s;
120 #else
121     struct {
122       uint64_t low, high;
123     } s;
124 #endif
125   } uu = {.ll = a};
126 
127   uint64_t word;
128   uint64_t add;
129 
130   if (uu.s.high) {
131     word = uu.s.high;
132     add = 0;
133   } else {
134     word = uu.s.low;
135     add = 64;
136   }
137   return __builtin_clzll(word) + add;
138 }
139 
140 #define Word_LoMask UINT64_C(0x00000000ffffffff)
141 #define Word_HiMask UINT64_C(0xffffffff00000000)
142 #define Word_FullMask UINT64_C(0xffffffffffffffff)
143 #define Word_1(a) (uint64_t)((a >> 96) & Word_LoMask)
144 #define Word_2(a) (uint64_t)((a >> 64) & Word_LoMask)
145 #define Word_3(a) (uint64_t)((a >> 32) & Word_LoMask)
146 #define Word_4(a) (uint64_t)(a & Word_LoMask)
147 
148 // 128x128 -> 256 wide multiply for platforms that don't have such an operation;
149 // many 64-bit platforms have this operation, but they tend to have hardware
150 // floating-point, so we don't bother with a special case for them here.
151 static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {
152 
153   const uint64_t product11 = Word_1(a) * Word_1(b);
154   const uint64_t product12 = Word_1(a) * Word_2(b);
155   const uint64_t product13 = Word_1(a) * Word_3(b);
156   const uint64_t product14 = Word_1(a) * Word_4(b);
157   const uint64_t product21 = Word_2(a) * Word_1(b);
158   const uint64_t product22 = Word_2(a) * Word_2(b);
159   const uint64_t product23 = Word_2(a) * Word_3(b);
160   const uint64_t product24 = Word_2(a) * Word_4(b);
161   const uint64_t product31 = Word_3(a) * Word_1(b);
162   const uint64_t product32 = Word_3(a) * Word_2(b);
163   const uint64_t product33 = Word_3(a) * Word_3(b);
164   const uint64_t product34 = Word_3(a) * Word_4(b);
165   const uint64_t product41 = Word_4(a) * Word_1(b);
166   const uint64_t product42 = Word_4(a) * Word_2(b);
167   const uint64_t product43 = Word_4(a) * Word_3(b);
168   const uint64_t product44 = Word_4(a) * Word_4(b);
169 
170   const __uint128_t sum0 = (__uint128_t)product44;
171   const __uint128_t sum1 = (__uint128_t)product34 + (__uint128_t)product43;
172   const __uint128_t sum2 =
173       (__uint128_t)product24 + (__uint128_t)product33 + (__uint128_t)product42;
174   const __uint128_t sum3 = (__uint128_t)product14 + (__uint128_t)product23 +
175                            (__uint128_t)product32 + (__uint128_t)product41;
176   const __uint128_t sum4 =
177       (__uint128_t)product13 + (__uint128_t)product22 + (__uint128_t)product31;
178   const __uint128_t sum5 = (__uint128_t)product12 + (__uint128_t)product21;
179   const __uint128_t sum6 = (__uint128_t)product11;
180 
181   const __uint128_t r0 = (sum0 & Word_FullMask) + ((sum1 & Word_LoMask) << 32);
182   const __uint128_t r1 = (sum0 >> 64) + ((sum1 >> 32) & Word_FullMask) +
183                          (sum2 & Word_FullMask) + ((sum3 << 32) & Word_HiMask);
184 
185   *lo = r0 + (r1 << 64);
186   *hi = (r1 >> 64) + (sum1 >> 96) + (sum2 >> 64) + (sum3 >> 32) + sum4 +
187         (sum5 << 32) + (sum6 << 64);
188 }
189 #undef Word_1
190 #undef Word_2
191 #undef Word_3
192 #undef Word_4
193 #undef Word_HiMask
194 #undef Word_LoMask
195 #undef Word_FullMask
196 #endif // __LDBL_MANT_DIG__ == 113 && __SIZEOF_INT128__
197 #else
198 #error SINGLE_PRECISION, DOUBLE_PRECISION or QUAD_PRECISION must be defined.
199 #endif
200 
201 #if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) ||                  \
202     defined(CRT_LDBL_128BIT)
203 #define typeWidth (sizeof(rep_t) * CHAR_BIT)
204 #define exponentBits (typeWidth - significandBits - 1)
205 #define maxExponent ((1 << exponentBits) - 1)
206 #define exponentBias (maxExponent >> 1)
207 
208 #define implicitBit (REP_C(1) << significandBits)
209 #define significandMask (implicitBit - 1U)
210 #define signBit (REP_C(1) << (significandBits + exponentBits))
211 #define absMask (signBit - 1U)
212 #define exponentMask (absMask ^ significandMask)
213 #define oneRep ((rep_t)exponentBias << significandBits)
214 #define infRep exponentMask
215 #define quietBit (implicitBit >> 1)
216 #define qnanRep (exponentMask | quietBit)
217 
218 static __inline rep_t toRep(fp_t x) {
219   const union {
220     fp_t f;
221     rep_t i;
222   } rep = {.f = x};
223   return rep.i;
224 }
225 
226 static __inline fp_t fromRep(rep_t x) {
227   const union {
228     fp_t f;
229     rep_t i;
230   } rep = {.i = x};
231   return rep.f;
232 }
233 
234 static __inline int normalize(rep_t *significand) {
235   const int shift = rep_clz(*significand) - rep_clz(implicitBit);
236   *significand <<= shift;
237   return 1 - shift;
238 }
239 
240 static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) {
241   *hi = *hi << count | *lo >> (typeWidth - count);
242   *lo = *lo << count;
243 }
244 
245 static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo,
246                                               unsigned int count) {
247   if (count < typeWidth) {
248     const bool sticky = (*lo << (typeWidth - count)) != 0;
249     *lo = *hi << (typeWidth - count) | *lo >> count | sticky;
250     *hi = *hi >> count;
251   } else if (count < 2 * typeWidth) {
252     const bool sticky = *hi << (2 * typeWidth - count) | *lo;
253     *lo = *hi >> (count - typeWidth) | sticky;
254     *hi = 0;
255   } else {
256     const bool sticky = *hi | *lo;
257     *lo = sticky;
258     *hi = 0;
259   }
260 }
261 
262 // Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids
263 // pulling in a libm dependency from compiler-rt, but is not meant to replace
264 // it (i.e. code calling logb() should get the one from libm, not this), hence
265 // the __compiler_rt prefix.
266 static __inline fp_t __compiler_rt_logbX(fp_t x) {
267   rep_t rep = toRep(x);
268   int exp = (rep & exponentMask) >> significandBits;
269 
270   // Abnormal cases:
271   // 1) +/- inf returns +inf; NaN returns NaN
272   // 2) 0.0 returns -inf
273   if (exp == maxExponent) {
274     if (((rep & signBit) == 0) || (x != x)) {
275       return x; // NaN or +inf: return x
276     } else {
277       return -x; // -inf: return -x
278     }
279   } else if (x == 0.0) {
280     // 0.0: return -inf
281     return fromRep(infRep | signBit);
282   }
283 
284   if (exp != 0) {
285     // Normal number
286     return exp - exponentBias; // Unbias exponent
287   } else {
288     // Subnormal number; normalize and repeat
289     rep &= absMask;
290     const int shift = 1 - normalize(&rep);
291     exp = (rep & exponentMask) >> significandBits;
292     return exp - exponentBias - shift; // Unbias exponent
293   }
294 }
295 #endif
296 
297 #if defined(SINGLE_PRECISION)
298 static __inline fp_t __compiler_rt_logbf(fp_t x) {
299   return __compiler_rt_logbX(x);
300 }
301 #elif defined(DOUBLE_PRECISION)
302 static __inline fp_t __compiler_rt_logb(fp_t x) {
303   return __compiler_rt_logbX(x);
304 }
305 #elif defined(QUAD_PRECISION)
306 #if defined(CRT_LDBL_128BIT)
307 static __inline fp_t __compiler_rt_logbl(fp_t x) {
308   return __compiler_rt_logbX(x);
309 }
310 #else
311 // The generic implementation only works for ieee754 floating point. For other
312 // floating point types, continue to rely on the libm implementation for now.
313 static __inline long double __compiler_rt_logbl(long double x) {
314   return crt_logbl(x);
315 }
316 #endif
317 #endif
318 
319 #endif // FP_LIB_HEADER
320