1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5 #ifdef FREEBL_NO_DEPEND
6 #include "stubs.h"
7 #endif
8 #include "gcm.h"
9 #include "secerr.h"
10
11 /* old gcc doesn't support some poly64x2_t intrinsic */
12 #if defined(__aarch64__) && defined(IS_LITTLE_ENDIAN) && \
13 (defined(__clang__) || defined(__GNUC__) && __GNUC__ > 6)
14
15 #include <arm_neon.h>
16
17 SECStatus
gcm_HashWrite_hw(gcmHashContext * ghash,unsigned char * outbuf)18 gcm_HashWrite_hw(gcmHashContext *ghash, unsigned char *outbuf)
19 {
20 uint8x16_t ci = vrbitq_u8(vreinterpretq_u8_u64(ghash->x));
21 vst1q_u8(outbuf, ci);
22 return SECSuccess;
23 }
24
25 SECStatus
gcm_HashMult_hw(gcmHashContext * ghash,const unsigned char * buf,unsigned int count)26 gcm_HashMult_hw(gcmHashContext *ghash, const unsigned char *buf,
27 unsigned int count)
28 {
29 const poly64x2_t p = vdupq_n_p64(0x87);
30 const uint8x16_t zero = vdupq_n_u8(0);
31 const uint64x2_t h = ghash->h;
32 uint64x2_t ci = ghash->x;
33 unsigned int i;
34 uint8x16_t z_low, z_high;
35 uint8x16_t t_low, t_high;
36 poly64x2_t t1;
37 uint8x16_t t2;
38
39 for (i = 0; i < count; i++, buf += 16) {
40 ci = vreinterpretq_u64_u8(veorq_u8(vreinterpretq_u8_u64(ci),
41 vrbitq_u8(vld1q_u8(buf))));
42
43 /* Do binary mult ghash->X = Ci * ghash->H. */
44 z_low = vreinterpretq_u8_p128(
45 vmull_p64((poly64_t)vget_low_p64(vreinterpretq_p64_u64(ci)),
46 (poly64_t)vget_low_p64(vreinterpretq_p64_u64(h))));
47 z_high = vreinterpretq_u8_p128(
48 vmull_high_p64(vreinterpretq_p64_u64(ci), vreinterpretq_p64_u64(h)));
49 t1 = vreinterpretq_p64_u8(
50 vextq_u8(vreinterpretq_u8_u64(h), vreinterpretq_u8_u64(h), 8));
51 t_low = vreinterpretq_u8_p128(
52 vmull_p64((poly64_t)vget_low_p64(vreinterpretq_p64_u64(ci)),
53 (poly64_t)vget_low_p64(t1)));
54 t_high = vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u64(ci), t1));
55 t2 = veorq_u8(t_high, t_low);
56 z_low = veorq_u8(z_low, vextq_u8(zero, t2, 8));
57 z_high = veorq_u8(z_high, vextq_u8(t2, zero, 8));
58
59 /* polynomial reduction */
60 t2 = vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u8(z_high), p));
61 z_high = veorq_u8(z_high, vextq_u8(t2, zero, 8));
62 z_low = veorq_u8(z_low, vextq_u8(zero, t2, 8));
63 ci = veorq_u64(vreinterpretq_u64_u8(z_low),
64 vreinterpretq_u64_p128(
65 vmull_p64((poly64_t)vget_low_p64(vreinterpretq_p64_u8(z_high)),
66 (poly64_t)vget_low_p64(p))));
67 }
68
69 ghash->x = ci;
70 return SECSuccess;
71 }
72
73 SECStatus
gcm_HashInit_hw(gcmHashContext * ghash)74 gcm_HashInit_hw(gcmHashContext *ghash)
75 {
76 /* Workaround of "used uninitialized in this function" error */
77 uint64x2_t h = vdupq_n_u64(0);
78
79 ghash->ghash_mul = gcm_HashMult_hw;
80 ghash->x = vdupq_n_u64(0);
81 h = vsetq_lane_u64(__builtin_bswap64(ghash->h_low), h, 1);
82 h = vsetq_lane_u64(__builtin_bswap64(ghash->h_high), h, 0);
83 h = vreinterpretq_u64_u8(vrbitq_u8(vreinterpretq_u8_u64(h)));
84 ghash->h = h;
85 ghash->hw = PR_TRUE;
86 return SECSuccess;
87 }
88
89 SECStatus
gcm_HashZeroX_hw(gcmHashContext * ghash)90 gcm_HashZeroX_hw(gcmHashContext *ghash)
91 {
92 ghash->x = vdupq_n_u64(0);
93 return SECSuccess;
94 }
95
96 #endif /* defined(__clang__) || (defined(__GNUC__) && __GNUC__ > 6) */
97