1 /* Copyright (C) 1995-2011, 2016 Mark Adler 2 * Copyright (C) 2017 ARM Holdings Inc. 3 * Author: Adenilson Cavalcanti <adenilson.cavalcanti@arm.com> 4 * 5 * For conditions of distribution and use, see copyright notice in zlib.h 6 */ 7 #ifdef ARM_NEON_ADLER32 8 #ifdef _M_ARM64 9 # include <arm64_neon.h> 10 #else 11 # include <arm_neon.h> 12 #endif 13 #include "../../zutil.h" 14 #include "../../adler32_p.h" 15 16 static void NEON_accum32(uint32_t *s, const unsigned char *buf, size_t len) { 17 static const uint8_t taps[32] = { 18 32, 31, 30, 29, 28, 27, 26, 25, 19 24, 23, 22, 21, 20, 19, 18, 17, 20 16, 15, 14, 13, 12, 11, 10, 9, 21 8, 7, 6, 5, 4, 3, 2, 1 }; 22 23 uint32x2_t adacc2, s2acc2, as; 24 uint8x16_t t0 = vld1q_u8(taps), t1 = vld1q_u8(taps + 16); 25 26 uint32x4_t adacc = vdupq_n_u32(0), s2acc = vdupq_n_u32(0); 27 adacc = vsetq_lane_u32(s[0], adacc, 0); 28 s2acc = vsetq_lane_u32(s[1], s2acc, 0); 29 30 while (len >= 2) { 31 uint8x16_t d0 = vld1q_u8(buf), d1 = vld1q_u8(buf + 16); 32 uint16x8_t adler, sum2; 33 s2acc = vaddq_u32(s2acc, vshlq_n_u32(adacc, 5)); 34 adler = vpaddlq_u8( d0); 35 adler = vpadalq_u8(adler, d1); 36 sum2 = vmull_u8( vget_low_u8(t0), vget_low_u8(d0)); 37 sum2 = vmlal_u8(sum2, vget_high_u8(t0), vget_high_u8(d0)); 38 sum2 = vmlal_u8(sum2, vget_low_u8(t1), vget_low_u8(d1)); 39 sum2 = vmlal_u8(sum2, vget_high_u8(t1), vget_high_u8(d1)); 40 adacc = vpadalq_u16(adacc, adler); 41 s2acc = vpadalq_u16(s2acc, sum2); 42 len -= 2; 43 buf += 32; 44 } 45 46 while (len > 0) { 47 uint8x16_t d0 = vld1q_u8(buf); 48 uint16x8_t adler, sum2; 49 s2acc = vaddq_u32(s2acc, vshlq_n_u32(adacc, 4)); 50 adler = vpaddlq_u8(d0); 51 sum2 = vmull_u8( vget_low_u8(t1), vget_low_u8(d0)); 52 sum2 = vmlal_u8(sum2, vget_high_u8(t1), vget_high_u8(d0)); 53 adacc = vpadalq_u16(adacc, adler); 54 s2acc = vpadalq_u16(s2acc, sum2); 55 buf += 16; 56 len--; 57 } 58 59 adacc2 = vpadd_u32(vget_low_u32(adacc), vget_high_u32(adacc)); 60 s2acc2 = vpadd_u32(vget_low_u32(s2acc), vget_high_u32(s2acc)); 61 as = vpadd_u32(adacc2, s2acc2); 62 s[0] = vget_lane_u32(as, 0); 63 s[1] = vget_lane_u32(as, 1); 64 } 65 66 static void NEON_handle_tail(uint32_t *pair, const unsigned char *buf, size_t len) { 67 unsigned int i; 68 for (i = 0; i < len; ++i) { 69 pair[0] += buf[i]; 70 pair[1] += pair[0]; 71 } 72 } 73 74 uint32_t adler32_neon(uint32_t adler, const unsigned char *buf, size_t len) { 75 /* split Adler-32 into component sums */ 76 uint32_t sum2 = (adler >> 16) & 0xffff; 77 adler &= 0xffff; 78 79 /* in case user likes doing a byte at a time, keep it fast */ 80 if (len == 1) 81 return adler32_len_1(adler, buf, sum2); 82 83 /* initial Adler-32 value (deferred check for len == 1 speed) */ 84 if (buf == NULL) 85 return 1L; 86 87 /* in case short lengths are provided, keep it somewhat fast */ 88 if (len < 16) 89 return adler32_len_16(adler, buf, len, sum2); 90 91 uint32_t pair[2]; 92 int n = NMAX; 93 unsigned int done = 0; 94 unsigned int i; 95 96 /* Split Adler-32 into component sums, it can be supplied by 97 * the caller sites (e.g. in a PNG file). 98 */ 99 pair[0] = adler; 100 pair[1] = sum2; 101 102 for (i = 0; i < len; i += n) { 103 if ((i + n) > len) 104 n = (int)(len - i); 105 106 if (n < 16) 107 break; 108 109 NEON_accum32(pair, buf + i, n / 16); 110 pair[0] %= BASE; 111 pair[1] %= BASE; 112 113 done += (n / 16) * 16; 114 } 115 116 /* Handle the tail elements. */ 117 if (done < len) { 118 NEON_handle_tail(pair, (buf + done), len - done); 119 pair[0] %= BASE; 120 pair[1] %= BASE; 121 } 122 123 /* D = B * 65536 + A, see: https://en.wikipedia.org/wiki/Adler-32. */ 124 return (pair[1] << 16) | pair[0]; 125 } 126 #endif 127