1 #ifndef BLAKE3_IMPL_H
2 #define BLAKE3_IMPL_H
3
4 #include <assert.h>
5 #include <stdbool.h>
6 #include <stddef.h>
7 #include <stdint.h>
8 #include <string.h>
9 #include <stdio.h>
10
11 #include "blake3.h"
12
13 // internal flags
14 enum blake3_flags {
15 CHUNK_START = 1 << 0,
16 CHUNK_END = 1 << 1,
17 PARENT = 1 << 2,
18 ROOT = 1 << 3,
19 KEYED_HASH = 1 << 4,
20 DERIVE_KEY_CONTEXT = 1 << 5,
21 DERIVE_KEY_MATERIAL = 1 << 6,
22 };
23
24 // This C implementation tries to support recent versions of GCC, Clang, and
25 // MSVC.
26 #if defined(_MSC_VER)
27 #define INLINE static __forceinline
28 #else
29 #define INLINE static inline __attribute__((always_inline))
30 #endif
31
32 #if defined(__x86_64__) || defined(_M_X64)
33 #define IS_X86
34 #define IS_X86_64
35 #endif
36
37 #if defined(__i386__) || defined(_M_IX86)
38 #define IS_X86
39 #define IS_X86_32
40 #endif
41
42 #if defined(IS_X86)
43 #if defined(_MSC_VER)
44 #include <intrin.h>
45 #endif
46 #include <immintrin.h>
47 #endif
48
49 #if defined(IS_X86)
50 #define MAX_SIMD_DEGREE 16
51 #elif defined(BLAKE3_USE_NEON)
52 #define MAX_SIMD_DEGREE 4
53 #else
54 #define MAX_SIMD_DEGREE 1
55 #endif
56
57 // There are some places where we want a static size that's equal to the
58 // MAX_SIMD_DEGREE, but also at least 2.
59 #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
60
61 static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
62 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
63 0x1F83D9ABUL, 0x5BE0CD19UL};
64
65 static const uint8_t MSG_SCHEDULE[7][16] = {
66 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
67 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
68 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
69 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
70 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
71 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
72 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
73 };
74
75 /* Find index of the highest set bit */
76 /* x is assumed to be nonzero. */
highest_one(uint64_t x)77 static unsigned int highest_one(uint64_t x) {
78 #if defined(__GNUC__) || defined(__clang__)
79 return 63 ^ __builtin_clzll(x);
80 #elif defined(_MSC_VER) && defined(IS_X86_64)
81 unsigned long index;
82 _BitScanReverse64(&index, x);
83 return index;
84 #elif defined(_MSC_VER) && defined(IS_X86_32)
85 if(x >> 32) {
86 unsigned long index;
87 _BitScanReverse(&index, x >> 32);
88 return 32 + index;
89 } else {
90 unsigned long index;
91 _BitScanReverse(&index, x);
92 return index;
93 }
94 #else
95 unsigned int c = 0;
96 if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
97 if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
98 if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
99 if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
100 if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }
101 if(x & 0x0000000000000002ULL) { c += 1; }
102 return c;
103 #endif
104 }
105
106 // Count the number of 1 bits.
popcnt(uint64_t x)107 INLINE unsigned int popcnt(uint64_t x) {
108 #if defined(__GNUC__) || defined(__clang__)
109 return __builtin_popcountll(x);
110 #else
111 unsigned int count = 0;
112 while (x != 0) {
113 count += 1;
114 x &= x - 1;
115 }
116 return count;
117 #endif
118 }
119
120 // Largest power of two less than or equal to x. As a special case, returns 1
121 // when x is 0.
round_down_to_power_of_2(uint64_t x)122 INLINE uint64_t round_down_to_power_of_2(uint64_t x) {
123 return 1ULL << highest_one(x | 1);
124 }
125
counter_low(uint64_t counter)126 INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }
127
counter_high(uint64_t counter)128 INLINE uint32_t counter_high(uint64_t counter) {
129 return (uint32_t)(counter >> 32);
130 }
131
load32(const void * src)132 INLINE uint32_t load32(const void *src) {
133 const uint8_t *p = (const uint8_t *)src;
134 return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
135 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
136 }
137
load_key_words(const uint8_t key[BLAKE3_KEY_LEN],uint32_t key_words[8])138 INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
139 uint32_t key_words[8]) {
140 key_words[0] = load32(&key[0 * 4]);
141 key_words[1] = load32(&key[1 * 4]);
142 key_words[2] = load32(&key[2 * 4]);
143 key_words[3] = load32(&key[3 * 4]);
144 key_words[4] = load32(&key[4 * 4]);
145 key_words[5] = load32(&key[5 * 4]);
146 key_words[6] = load32(&key[6 * 4]);
147 key_words[7] = load32(&key[7 * 4]);
148 }
149
150 void blake3_compress_in_place(uint32_t cv[8],
151 const uint8_t block[BLAKE3_BLOCK_LEN],
152 uint8_t block_len, uint64_t counter,
153 uint8_t flags);
154
155 void blake3_compress_xof(const uint32_t cv[8],
156 const uint8_t block[BLAKE3_BLOCK_LEN],
157 uint8_t block_len, uint64_t counter, uint8_t flags,
158 uint8_t out[64]);
159
160 void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
161 size_t blocks, const uint32_t key[8], uint64_t counter,
162 bool increment_counter, uint8_t flags,
163 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
164
165 size_t blake3_simd_degree(void);
166
167
168 // Declarations for implementation-specific functions.
169 void blake3_compress_in_place_portable(uint32_t cv[8],
170 const uint8_t block[BLAKE3_BLOCK_LEN],
171 uint8_t block_len, uint64_t counter,
172 uint8_t flags);
173
174 void blake3_compress_xof_portable(const uint32_t cv[8],
175 const uint8_t block[BLAKE3_BLOCK_LEN],
176 uint8_t block_len, uint64_t counter,
177 uint8_t flags, uint8_t out[64]);
178
179 void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
180 size_t blocks, const uint32_t key[8],
181 uint64_t counter, bool increment_counter,
182 uint8_t flags, uint8_t flags_start,
183 uint8_t flags_end, uint8_t *out);
184
185 #if defined(IS_X86)
186 #if !defined(BLAKE3_NO_SSE41)
187 void blake3_compress_in_place_sse41(uint32_t cv[8],
188 const uint8_t block[BLAKE3_BLOCK_LEN],
189 uint8_t block_len, uint64_t counter,
190 uint8_t flags);
191 void blake3_compress_xof_sse41(const uint32_t cv[8],
192 const uint8_t block[BLAKE3_BLOCK_LEN],
193 uint8_t block_len, uint64_t counter,
194 uint8_t flags, uint8_t out[64]);
195 void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
196 size_t blocks, const uint32_t key[8],
197 uint64_t counter, bool increment_counter,
198 uint8_t flags, uint8_t flags_start,
199 uint8_t flags_end, uint8_t *out);
200 #endif
201 #if !defined(BLAKE3_NO_AVX2)
202 void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
203 size_t blocks, const uint32_t key[8],
204 uint64_t counter, bool increment_counter,
205 uint8_t flags, uint8_t flags_start,
206 uint8_t flags_end, uint8_t *out);
207 #endif
208 #if !defined(BLAKE3_NO_AVX512)
209 void blake3_compress_in_place_avx512(uint32_t cv[8],
210 const uint8_t block[BLAKE3_BLOCK_LEN],
211 uint8_t block_len, uint64_t counter,
212 uint8_t flags);
213
214 void blake3_compress_xof_avx512(const uint32_t cv[8],
215 const uint8_t block[BLAKE3_BLOCK_LEN],
216 uint8_t block_len, uint64_t counter,
217 uint8_t flags, uint8_t out[64]);
218
219 void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
220 size_t blocks, const uint32_t key[8],
221 uint64_t counter, bool increment_counter,
222 uint8_t flags, uint8_t flags_start,
223 uint8_t flags_end, uint8_t *out);
224 #endif
225 #endif
226
227 #if defined(BLAKE3_USE_NEON)
228 void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
229 size_t blocks, const uint32_t key[8],
230 uint64_t counter, bool increment_counter,
231 uint8_t flags, uint8_t flags_start,
232 uint8_t flags_end, uint8_t *out);
233 #endif
234
235
236 #endif /* BLAKE3_IMPL_H */
237