1 /**
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0.
4 */
5
6 /* No instrics defined for 32-bit MSVC */
7 #if (defined(_M_ARM64) || defined(__aarch64__) || defined(__arm__))
8 # include <aws/checksums/private/crc_priv.h>
9 # ifdef _M_ARM64
10 # include <arm64_neon.h>
11 # define PREFETCH(p) __prefetch(p)
12 # else
13 # include <arm_acle.h>
14 # define PREFETCH(p) __builtin_prefetch(p)
15 # endif
16
aws_checksums_crc32c_hw(const uint8_t * data,int length,uint32_t previousCrc32)17 uint32_t aws_checksums_crc32c_hw(const uint8_t *data, int length, uint32_t previousCrc32) {
18 uint32_t crc = ~previousCrc32;
19
20 // Align data if it's not aligned
21 while (((uintptr_t)data & 7) && length > 0) {
22 crc = __crc32cb(crc, *(uint8_t *)data);
23 data++;
24 length--;
25 }
26
27 while (length >= 64) {
28 PREFETCH(data + 384);
29 uint64_t *d = (uint64_t *)data;
30 crc = __crc32cd(crc, d[0]);
31 crc = __crc32cd(crc, d[1]);
32 crc = __crc32cd(crc, d[2]);
33 crc = __crc32cd(crc, d[3]);
34 crc = __crc32cd(crc, d[4]);
35 crc = __crc32cd(crc, d[5]);
36 crc = __crc32cd(crc, d[6]);
37 crc = __crc32cd(crc, d[7]);
38 data += 64;
39 length -= 64;
40 }
41
42 while (length >= 8) {
43 crc = __crc32cd(crc, *(uint64_t *)data);
44 data += 8;
45 length -= 8;
46 }
47
48 while (length > 0) {
49 crc = __crc32cb(crc, *(uint8_t *)data);
50 data++;
51 length--;
52 }
53
54 return ~crc;
55 }
56
aws_checksums_crc32_hw(const uint8_t * data,int length,uint32_t previousCrc32)57 uint32_t aws_checksums_crc32_hw(const uint8_t *data, int length, uint32_t previousCrc32) {
58 uint32_t crc = ~previousCrc32;
59
60 // Align data if it's not aligned
61 while (((uintptr_t)data & 7) && length > 0) {
62 crc = __crc32b(crc, *(uint8_t *)data);
63 data++;
64 length--;
65 }
66
67 while (length >= 64) {
68 PREFETCH(data + 384);
69 uint64_t *d = (uint64_t *)data;
70 crc = __crc32d(crc, d[0]);
71 crc = __crc32d(crc, d[1]);
72 crc = __crc32d(crc, d[2]);
73 crc = __crc32d(crc, d[3]);
74 crc = __crc32d(crc, d[4]);
75 crc = __crc32d(crc, d[5]);
76 crc = __crc32d(crc, d[6]);
77 crc = __crc32d(crc, d[7]);
78 data += 64;
79 length -= 64;
80 }
81
82 while (length >= 8) {
83 crc = __crc32d(crc, *(uint64_t *)data);
84 data += 8;
85 length -= 8;
86 }
87
88 while (length > 0) {
89 crc = __crc32b(crc, *(uint8_t *)data);
90 data++;
91 length--;
92 }
93
94 return ~crc;
95 }
96
97 #endif
98