195 static inline int32_t  libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom);
196 static inline uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom);
197 static inline int64_t  libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom);
198 static inline uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom);
200 …nt32_t  libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom);
201 …nt32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom);
202 …nt64_t  libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom);
203 …nt64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom);
205 static inline int32_t  libdivide_s32_recover(const struct libdivide_s32_t *denom);
206 static inline uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom);
207 static inline int64_t  libdivide_s64_recover(const struct libdivide_s64_t *denom);
208 static inline uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom);
210 …c inline int32_t  libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom);
211 …c inline uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom);
212 …c inline int64_t  libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom);
213 …c inline uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom);
610 uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) {  in libdivide_u32_do()  argument
611     uint8_t more = denom->more;  in libdivide_u32_do()
612     if (!denom->magic) {  in libdivide_u32_do()
616         uint32_t q = libdivide_mullhi_u32(denom->magic, numer);  in libdivide_u32_do()
629 uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom…  in libdivide_u32_branchfree_do()  argument
630     uint32_t q = libdivide_mullhi_u32(denom->magic, numer);  in libdivide_u32_branchfree_do()
632     return t >> denom->more;  in libdivide_u32_branchfree_do()
635 uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) {  in libdivide_u32_recover()  argument
636     uint8_t more = denom->more;  in libdivide_u32_recover()
639     if (!denom->magic) {  in libdivide_u32_recover()
649         return 1 + libdivide_64_div_32_to_32(hi_dividend, 0, denom->magic, &rem_ignored);  in libdivide_u32_recover()
657         uint64_t d = (1ULL << 32) | denom->magic;  in libdivide_u32_recover()
673 uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom) {  in libdivide_u32_branchfree_recover()  argument
674     uint8_t more = denom->more;  in libdivide_u32_branchfree_recover()
677     if (!denom->magic) {  in libdivide_u32_branchfree_recover()
686         uint64_t d = (1ULL << 32) | denom->magic;  in libdivide_u32_branchfree_recover()
769 uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) {  in libdivide_u64_do()  argument
770     uint8_t more = denom->more;  in libdivide_u64_do()
771     if (!denom->magic) {  in libdivide_u64_do()
775         uint64_t q = libdivide_mullhi_u64(denom->magic, numer);  in libdivide_u64_do()
788 uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom…  in libdivide_u64_branchfree_do()  argument
789     uint64_t q = libdivide_mullhi_u64(denom->magic, numer);  in libdivide_u64_branchfree_do()
791     return t >> denom->more;  in libdivide_u64_branchfree_do()
794 uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) {  in libdivide_u64_recover()  argument
795     uint8_t more = denom->more;  in libdivide_u64_recover()
798     if (!denom->magic) {  in libdivide_u64_recover()
808         return 1 + libdivide_128_div_64_to_64(hi_dividend, 0, denom->magic, &rem_ignored);  in libdivide_u64_recover()
820         const uint64_t d_hi = 1, d_lo = denom->magic;  in libdivide_u64_recover()
837 uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom) {  in libdivide_u64_branchfree_recover()  argument
838     uint8_t more = denom->more;  in libdivide_u64_branchfree_recover()
841     if (!denom->magic) {  in libdivide_u64_branchfree_recover()
854         const uint64_t d_hi = 1, d_lo = denom->magic;  in libdivide_u64_branchfree_recover()
948 int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) {  in libdivide_s32_do()  argument
949     uint8_t more = denom->more;  in libdivide_s32_do()
952     if (!denom->magic) {  in libdivide_s32_do()
961         uint32_t uq = (uint32_t)libdivide_mullhi_s32(denom->magic, numer);  in libdivide_s32_do()
976 int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_do()  argument
977     uint8_t more = denom->more;  in libdivide_s32_branchfree_do()
981     int32_t magic = denom->magic;  in libdivide_s32_branchfree_do()
1000 int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) {  in libdivide_s32_recover()  argument
1001     uint8_t more = denom->more;  in libdivide_s32_recover()
1003     if (!denom->magic) {  in libdivide_s32_recover()
1019             ? denom->magic > 0 : denom->magic < 0;  in libdivide_s32_recover()
1022         if (denom->magic == 0) {  in libdivide_s32_recover()
1027         uint32_t d = (uint32_t)(magic_was_negated ? -denom->magic : denom->magic);  in libdivide_s32_recover()
1036 int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_recover()  argument
1037     return libdivide_s32_recover((const struct libdivide_s32_t *)denom);  in libdivide_s32_branchfree_recover()
1117 int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) {  in libdivide_s64_do()  argument
1118     uint8_t more = denom->more;  in libdivide_s64_do()
1121     if (!denom->magic) { // shift path  in libdivide_s64_do()
1131         uint64_t uq = (uint64_t)libdivide_mullhi_s64(denom->magic, numer);  in libdivide_s64_do()
1146 int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_do()  argument
1147     uint8_t more = denom->more;  in libdivide_s64_branchfree_do()
1151     int64_t magic = denom->magic;  in libdivide_s64_branchfree_do()
1170 int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) {  in libdivide_s64_recover()  argument
1171     uint8_t more = denom->more;  in libdivide_s64_recover()
1173     if (denom->magic == 0) { // shift path  in libdivide_s64_recover()
1183             ? denom->magic > 0 : denom->magic < 0;  in libdivide_s64_recover()
1185         uint64_t d = (uint64_t)(magic_was_negated ? -denom->magic : denom->magic);  in libdivide_s64_recover()
1197 int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_recover()  argument
1198     return libdivide_s64_recover((const struct libdivide_s64_t *)denom);  in libdivide_s64_branchfree_recover()
1203 static inline __m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom);
1204 static inline __m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom);
1205 static inline __m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom);
1206 static inline __m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom);
1208 …libdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom);
1209 …libdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom);
1210 …libdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom);
1211 …libdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom);
1275 __m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom) {  in libdivide_u32_do_vector()  argument
1276     uint8_t more = denom->more;  in libdivide_u32_do_vector()
1277     if (!denom->magic) {  in libdivide_u32_do_vector()
1281         __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic));  in libdivide_u32_do_vector()
1295 …ibdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom) {  in libdivide_u32_branchfree_do_vector()  argument
1296     __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic));  in libdivide_u32_branchfree_do_vector()
1298     return _mm512_srli_epi32(t, denom->more);  in libdivide_u32_branchfree_do_vector()
1303 __m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom) {  in libdivide_u64_do_vector()  argument
1304     uint8_t more = denom->more;  in libdivide_u64_do_vector()
1305     if (!denom->magic) {  in libdivide_u64_do_vector()
1309         __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic));  in libdivide_u64_do_vector()
1323 …ibdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom) {  in libdivide_u64_branchfree_do_vector()  argument
1324     __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic));  in libdivide_u64_branchfree_do_vector()
1326     return _mm512_srli_epi64(t, denom->more);  in libdivide_u64_branchfree_do_vector()
1331 __m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom) {  in libdivide_s32_do_vector()  argument
1332     uint8_t more = denom->more;  in libdivide_s32_do_vector()
1333     if (!denom->magic) {  in libdivide_s32_do_vector()
1346         __m512i q = libdivide_mullhi_s32_vector(numers, _mm512_set1_epi32(denom->magic));  in libdivide_s32_do_vector()
1360 …ibdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_do_vector()  argument
1361     int32_t magic = denom->magic;  in libdivide_s32_branchfree_do_vector()
1362     uint8_t more = denom->more;  in libdivide_s32_branchfree_do_vector()
1383 __m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom) {  in libdivide_s64_do_vector()  argument
1384     uint8_t more = denom->more;  in libdivide_s64_do_vector()
1385     int64_t magic = denom->magic;  in libdivide_s64_do_vector()
1413 …ibdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_do_vector()  argument
1414     int64_t magic = denom->magic;  in libdivide_s64_branchfree_do_vector()
1415     uint8_t more = denom->more;  in libdivide_s64_branchfree_do_vector()
1438 static inline __m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom);
1439 static inline __m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom);
1440 static inline __m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom);
1441 static inline __m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom);
1443 …libdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom);
1444 …libdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom);
1445 …libdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom);
1446 …libdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom);
1518 __m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom) {  in libdivide_u32_do_vector()  argument
1519     uint8_t more = denom->more;  in libdivide_u32_do_vector()
1520     if (!denom->magic) {  in libdivide_u32_do_vector()
1524         __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic));  in libdivide_u32_do_vector()
1538 …ibdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom) {  in libdivide_u32_branchfree_do_vector()  argument
1539     __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic));  in libdivide_u32_branchfree_do_vector()
1541     return _mm256_srli_epi32(t, denom->more);  in libdivide_u32_branchfree_do_vector()
1546 __m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom) {  in libdivide_u64_do_vector()  argument
1547     uint8_t more = denom->more;  in libdivide_u64_do_vector()
1548     if (!denom->magic) {  in libdivide_u64_do_vector()
1552         __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic));  in libdivide_u64_do_vector()
1566 …ibdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom) {  in libdivide_u64_branchfree_do_vector()  argument
1567     __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic));  in libdivide_u64_branchfree_do_vector()
1569     return _mm256_srli_epi64(t, denom->more);  in libdivide_u64_branchfree_do_vector()
1574 __m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom) {  in libdivide_s32_do_vector()  argument
1575     uint8_t more = denom->more;  in libdivide_s32_do_vector()
1576     if (!denom->magic) {  in libdivide_s32_do_vector()
1589         __m256i q = libdivide_mullhi_s32_vector(numers, _mm256_set1_epi32(denom->magic));  in libdivide_s32_do_vector()
1603 …ibdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_do_vector()  argument
1604     int32_t magic = denom->magic;  in libdivide_s32_branchfree_do_vector()
1605     uint8_t more = denom->more;  in libdivide_s32_branchfree_do_vector()
1626 __m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom) {  in libdivide_s64_do_vector()  argument
1627     uint8_t more = denom->more;  in libdivide_s64_do_vector()
1628     int64_t magic = denom->magic;  in libdivide_s64_do_vector()
1656 …ibdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_do_vector()  argument
1657     int64_t magic = denom->magic;  in libdivide_s64_branchfree_do_vector()
1658     uint8_t more = denom->more;  in libdivide_s64_branchfree_do_vector()
1681 static inline __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom);
1682 static inline __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom);
1683 static inline __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom);
1684 static inline __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom);
1686 …libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom);
1687 …libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom);
1688 …libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom);
1689 …libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom);
1765 __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom) {  in libdivide_u32_do_vector()  argument
1766     uint8_t more = denom->more;  in libdivide_u32_do_vector()
1767     if (!denom->magic) {  in libdivide_u32_do_vector()
1771         __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic));  in libdivide_u32_do_vector()
1785 …ibdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom) {  in libdivide_u32_branchfree_do_vector()  argument
1786     __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic));  in libdivide_u32_branchfree_do_vector()
1788     return _mm_srli_epi32(t, denom->more);  in libdivide_u32_branchfree_do_vector()
1793 __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom) {  in libdivide_u64_do_vector()  argument
1794     uint8_t more = denom->more;  in libdivide_u64_do_vector()
1795     if (!denom->magic) {  in libdivide_u64_do_vector()
1799         __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic));  in libdivide_u64_do_vector()
1813 …ibdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom) {  in libdivide_u64_branchfree_do_vector()  argument
1814     __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic));  in libdivide_u64_branchfree_do_vector()
1816     return _mm_srli_epi64(t, denom->more);  in libdivide_u64_branchfree_do_vector()
1821 __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom) {  in libdivide_s32_do_vector()  argument
1822     uint8_t more = denom->more;  in libdivide_s32_do_vector()
1823     if (!denom->magic) {  in libdivide_s32_do_vector()
1836         __m128i q = libdivide_mullhi_s32_vector(numers, _mm_set1_epi32(denom->magic));  in libdivide_s32_do_vector()
1850 …ibdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_do_vector()  argument
1851     int32_t magic = denom->magic;  in libdivide_s32_branchfree_do_vector()
1852     uint8_t more = denom->more;  in libdivide_s32_branchfree_do_vector()
1873 __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom) {  in libdivide_s64_do_vector()  argument
1874     uint8_t more = denom->more;  in libdivide_s64_do_vector()
1875     int64_t magic = denom->magic;  in libdivide_s64_do_vector()
1903 …ibdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_do_vector()  argument
1904     int64_t magic = denom->magic;  in libdivide_s64_branchfree_do_vector()
1905     uint8_t more = denom->more;  in libdivide_s64_branchfree_do_vector()
1954             return libdivide_##ALGO##_do_vector(n, &denom); \
1961     libdivide_##ALGO##_t denom; \
1964         : denom(libdivide_##ALGO##_gen(d)) \
1967         return libdivide_##ALGO##_do(n, &denom); \
1971         return libdivide_##ALGO##_recover(&denom); \
2013         return div.denom.magic == other.denom.magic &&
2014                div.denom.more == other.denom.more;