204 static inline int32_t  libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom);
205 static inline uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom);
206 static inline int64_t  libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom);
207 static inline uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom);
209 …nt32_t  libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom);
210 …nt32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom);
211 …nt64_t  libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom);
212 …nt64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom);
214 static inline int32_t  libdivide_s32_recover(const struct libdivide_s32_t *denom);
215 static inline uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom);
216 static inline int64_t  libdivide_s64_recover(const struct libdivide_s64_t *denom);
217 static inline uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom);
219 …c inline int32_t  libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom);
220 …c inline uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom);
221 …c inline int64_t  libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom);
222 …c inline uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom);
626 uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) {  in libdivide_u32_do()  argument
627     uint8_t more = denom->more;  in libdivide_u32_do()
628     if (!denom->magic) {  in libdivide_u32_do()
632         uint32_t q = libdivide_mullhi_u32(denom->magic, numer);  in libdivide_u32_do()
645 uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom…  in libdivide_u32_branchfree_do()  argument
646     uint32_t q = libdivide_mullhi_u32(denom->magic, numer);  in libdivide_u32_branchfree_do()
648     return t >> denom->more;  in libdivide_u32_branchfree_do()
651 uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) {  in libdivide_u32_recover()  argument
652     uint8_t more = denom->more;  in libdivide_u32_recover()
655     if (!denom->magic) {  in libdivide_u32_recover()
665         return 1 + libdivide_64_div_32_to_32(hi_dividend, 0, denom->magic, &rem_ignored);  in libdivide_u32_recover()
673         uint64_t d = (1ULL << 32) | denom->magic;  in libdivide_u32_recover()
689 uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom) {  in libdivide_u32_branchfree_recover()  argument
690     uint8_t more = denom->more;  in libdivide_u32_branchfree_recover()
693     if (!denom->magic) {  in libdivide_u32_branchfree_recover()
702         uint64_t d = (1ULL << 32) | denom->magic;  in libdivide_u32_branchfree_recover()
785 uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) {  in libdivide_u64_do()  argument
786     uint8_t more = denom->more;  in libdivide_u64_do()
787     if (!denom->magic) {  in libdivide_u64_do()
791         uint64_t q = libdivide_mullhi_u64(denom->magic, numer);  in libdivide_u64_do()
804 uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom…  in libdivide_u64_branchfree_do()  argument
805     uint64_t q = libdivide_mullhi_u64(denom->magic, numer);  in libdivide_u64_branchfree_do()
807     return t >> denom->more;  in libdivide_u64_branchfree_do()
810 uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) {  in libdivide_u64_recover()  argument
811     uint8_t more = denom->more;  in libdivide_u64_recover()
814     if (!denom->magic) {  in libdivide_u64_recover()
824         return 1 + libdivide_128_div_64_to_64(hi_dividend, 0, denom->magic, &rem_ignored);  in libdivide_u64_recover()
836         const uint64_t d_hi = 1, d_lo = denom->magic;  in libdivide_u64_recover()
853 uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom) {  in libdivide_u64_branchfree_recover()  argument
854     uint8_t more = denom->more;  in libdivide_u64_branchfree_recover()
857     if (!denom->magic) {  in libdivide_u64_branchfree_recover()
870         const uint64_t d_hi = 1, d_lo = denom->magic;  in libdivide_u64_branchfree_recover()
964 int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) {  in libdivide_s32_do()  argument
965     uint8_t more = denom->more;  in libdivide_s32_do()
968     if (!denom->magic) {  in libdivide_s32_do()
977         uint32_t uq = (uint32_t)libdivide_mullhi_s32(denom->magic, numer);  in libdivide_s32_do()
992 int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_do()  argument
993     uint8_t more = denom->more;  in libdivide_s32_branchfree_do()
997     int32_t magic = denom->magic;  in libdivide_s32_branchfree_do()
1016 int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) {  in libdivide_s32_recover()  argument
1017     uint8_t more = denom->more;  in libdivide_s32_recover()
1019     if (!denom->magic) {  in libdivide_s32_recover()
1035             ? denom->magic > 0 : denom->magic < 0;  in libdivide_s32_recover()
1038         if (denom->magic == 0) {  in libdivide_s32_recover()
1043         uint32_t d = (uint32_t)(magic_was_negated ? -denom->magic : denom->magic);  in libdivide_s32_recover()
1052 int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_recover()  argument
1053     return libdivide_s32_recover((const struct libdivide_s32_t *)denom);  in libdivide_s32_branchfree_recover()
1133 int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) {  in libdivide_s64_do()  argument
1134     uint8_t more = denom->more;  in libdivide_s64_do()
1137     if (!denom->magic) { // shift path  in libdivide_s64_do()
1147         uint64_t uq = (uint64_t)libdivide_mullhi_s64(denom->magic, numer);  in libdivide_s64_do()
1162 int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_do()  argument
1163     uint8_t more = denom->more;  in libdivide_s64_branchfree_do()
1167     int64_t magic = denom->magic;  in libdivide_s64_branchfree_do()
1186 int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) {  in libdivide_s64_recover()  argument
1187     uint8_t more = denom->more;  in libdivide_s64_recover()
1189     if (denom->magic == 0) { // shift path  in libdivide_s64_recover()
1199             ? denom->magic > 0 : denom->magic < 0;  in libdivide_s64_recover()
1201         uint64_t d = (uint64_t)(magic_was_negated ? -denom->magic : denom->magic);  in libdivide_s64_recover()
1213 int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_recover()  argument
1214     return libdivide_s64_recover((const struct libdivide_s64_t *)denom);  in libdivide_s64_branchfree_recover()
1219 static inline __m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom);
1220 static inline __m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom);
1221 static inline __m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom);
1222 static inline __m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom);
1224 …libdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom);
1225 …libdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom);
1226 …libdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom);
1227 …libdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom);
1291 __m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom) {  in libdivide_u32_do_vector()  argument
1292     uint8_t more = denom->more;  in libdivide_u32_do_vector()
1293     if (!denom->magic) {  in libdivide_u32_do_vector()
1297         __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic));  in libdivide_u32_do_vector()
1311 …ibdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom) {  in libdivide_u32_branchfree_do_vector()  argument
1312     __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic));  in libdivide_u32_branchfree_do_vector()
1314     return _mm512_srli_epi32(t, denom->more);  in libdivide_u32_branchfree_do_vector()
1319 __m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom) {  in libdivide_u64_do_vector()  argument
1320     uint8_t more = denom->more;  in libdivide_u64_do_vector()
1321     if (!denom->magic) {  in libdivide_u64_do_vector()
1325         __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic));  in libdivide_u64_do_vector()
1339 …ibdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom) {  in libdivide_u64_branchfree_do_vector()  argument
1340     __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic));  in libdivide_u64_branchfree_do_vector()
1342     return _mm512_srli_epi64(t, denom->more);  in libdivide_u64_branchfree_do_vector()
1347 __m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom) {  in libdivide_s32_do_vector()  argument
1348     uint8_t more = denom->more;  in libdivide_s32_do_vector()
1349     if (!denom->magic) {  in libdivide_s32_do_vector()
1362         __m512i q = libdivide_mullhi_s32_vector(numers, _mm512_set1_epi32(denom->magic));  in libdivide_s32_do_vector()
1376 …ibdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_do_vector()  argument
1377     int32_t magic = denom->magic;  in libdivide_s32_branchfree_do_vector()
1378     uint8_t more = denom->more;  in libdivide_s32_branchfree_do_vector()
1399 __m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom) {  in libdivide_s64_do_vector()  argument
1400     uint8_t more = denom->more;  in libdivide_s64_do_vector()
1401     int64_t magic = denom->magic;  in libdivide_s64_do_vector()
1429 …ibdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_do_vector()  argument
1430     int64_t magic = denom->magic;  in libdivide_s64_branchfree_do_vector()
1431     uint8_t more = denom->more;  in libdivide_s64_branchfree_do_vector()
1454 static inline __m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom);
1455 static inline __m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom);
1456 static inline __m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom);
1457 static inline __m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom);
1459 …libdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom);
1460 …libdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom);
1461 …libdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom);
1462 …libdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom);
1534 __m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom) {  in libdivide_u32_do_vector()  argument
1535     uint8_t more = denom->more;  in libdivide_u32_do_vector()
1536     if (!denom->magic) {  in libdivide_u32_do_vector()
1540         __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic));  in libdivide_u32_do_vector()
1554 …ibdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom) {  in libdivide_u32_branchfree_do_vector()  argument
1555     __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic));  in libdivide_u32_branchfree_do_vector()
1557     return _mm256_srli_epi32(t, denom->more);  in libdivide_u32_branchfree_do_vector()
1562 __m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom) {  in libdivide_u64_do_vector()  argument
1563     uint8_t more = denom->more;  in libdivide_u64_do_vector()
1564     if (!denom->magic) {  in libdivide_u64_do_vector()
1568         __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic));  in libdivide_u64_do_vector()
1582 …ibdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom) {  in libdivide_u64_branchfree_do_vector()  argument
1583     __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic));  in libdivide_u64_branchfree_do_vector()
1585     return _mm256_srli_epi64(t, denom->more);  in libdivide_u64_branchfree_do_vector()
1590 __m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom) {  in libdivide_s32_do_vector()  argument
1591     uint8_t more = denom->more;  in libdivide_s32_do_vector()
1592     if (!denom->magic) {  in libdivide_s32_do_vector()
1605         __m256i q = libdivide_mullhi_s32_vector(numers, _mm256_set1_epi32(denom->magic));  in libdivide_s32_do_vector()
1619 …ibdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_do_vector()  argument
1620     int32_t magic = denom->magic;  in libdivide_s32_branchfree_do_vector()
1621     uint8_t more = denom->more;  in libdivide_s32_branchfree_do_vector()
1642 __m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom) {  in libdivide_s64_do_vector()  argument
1643     uint8_t more = denom->more;  in libdivide_s64_do_vector()
1644     int64_t magic = denom->magic;  in libdivide_s64_do_vector()
1672 …ibdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_do_vector()  argument
1673     int64_t magic = denom->magic;  in libdivide_s64_branchfree_do_vector()
1674     uint8_t more = denom->more;  in libdivide_s64_branchfree_do_vector()
1697 static inline __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom);
1698 static inline __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom);
1699 static inline __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom);
1700 static inline __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom);
1702 …libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom);
1703 …libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom);
1704 …libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom);
1705 …libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom);
1781 __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom) {  in libdivide_u32_do_vector()  argument
1782     uint8_t more = denom->more;  in libdivide_u32_do_vector()
1783     if (!denom->magic) {  in libdivide_u32_do_vector()
1787         __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic));  in libdivide_u32_do_vector()
1801 …ibdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom) {  in libdivide_u32_branchfree_do_vector()  argument
1802     __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic));  in libdivide_u32_branchfree_do_vector()
1804     return _mm_srli_epi32(t, denom->more);  in libdivide_u32_branchfree_do_vector()
1809 __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom) {  in libdivide_u64_do_vector()  argument
1810     uint8_t more = denom->more;  in libdivide_u64_do_vector()
1811     if (!denom->magic) {  in libdivide_u64_do_vector()
1815         __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic));  in libdivide_u64_do_vector()
1829 …ibdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom) {  in libdivide_u64_branchfree_do_vector()  argument
1830     __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic));  in libdivide_u64_branchfree_do_vector()
1832     return _mm_srli_epi64(t, denom->more);  in libdivide_u64_branchfree_do_vector()
1837 __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom) {  in libdivide_s32_do_vector()  argument
1838     uint8_t more = denom->more;  in libdivide_s32_do_vector()
1839     if (!denom->magic) {  in libdivide_s32_do_vector()
1852         __m128i q = libdivide_mullhi_s32_vector(numers, _mm_set1_epi32(denom->magic));  in libdivide_s32_do_vector()
1866 …ibdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom) {  in libdivide_s32_branchfree_do_vector()  argument
1867     int32_t magic = denom->magic;  in libdivide_s32_branchfree_do_vector()
1868     uint8_t more = denom->more;  in libdivide_s32_branchfree_do_vector()
1889 __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom) {  in libdivide_s64_do_vector()  argument
1890     uint8_t more = denom->more;  in libdivide_s64_do_vector()
1891     int64_t magic = denom->magic;  in libdivide_s64_do_vector()
1919 …ibdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom) {  in libdivide_s64_branchfree_do_vector()  argument
1920     int64_t magic = denom->magic;  in libdivide_s64_branchfree_do_vector()
1921     uint8_t more = denom->more;  in libdivide_s64_branchfree_do_vector()
1970             return libdivide_##ALGO##_do_vector(n, &denom); \
1977     libdivide_##ALGO##_t denom; \
1980         : denom(libdivide_##ALGO##_gen(d)) \
1983         return libdivide_##ALGO##_do(n, &denom); \
1987         return libdivide_##ALGO##_recover(&denom); \
2029         return div.denom.magic == other.denom.magic &&
2030                div.denom.more == other.denom.more;