1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3 24 * Copyright (c) 2019-2020 Samuel Neves and Jack O'Connor 25 * Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de> 26 */ 27 28 #ifndef BLAKE3_IMPL_H 29 #define BLAKE3_IMPL_H 30 31 #ifdef __cplusplus 32 extern "C" { 33 #endif 34 35 #include <sys/types.h> 36 #include <sys/blake3.h> 37 #include <sys/simd.h> 38 #include <sys/asm_linkage.h> 39 40 /* 41 * Methods used to define BLAKE3 assembler implementations 42 */ 43 typedef void (*blake3_compress_in_place_f)(uint32_t cv[8], 44 const uint8_t block[BLAKE3_BLOCK_LEN], 45 uint8_t block_len, uint64_t counter, 46 uint8_t flags); 47 48 typedef void (*blake3_compress_xof_f)(const uint32_t cv[8], 49 const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, 50 uint64_t counter, uint8_t flags, uint8_t out[64]); 51 52 typedef void (*blake3_hash_many_f)(const uint8_t * const *inputs, 53 size_t num_inputs, size_t blocks, const uint32_t key[8], 54 uint64_t counter, boolean_t increment_counter, uint8_t flags, 55 uint8_t flags_start, uint8_t flags_end, uint8_t *out); 56 57 typedef boolean_t (*blake3_is_supported_f)(void); 58 59 typedef struct blake3_impl_ops { 60 blake3_compress_in_place_f compress_in_place; 61 blake3_compress_xof_f compress_xof; 62 blake3_hash_many_f hash_many; 63 blake3_is_supported_f is_supported; 64 int degree; 65 const char *name; 66 } blake3_ops_t; 67 68 /* Return selected BLAKE3 implementation ops */ 69 extern const blake3_ops_t *blake3_impl_get_ops(void); 70 71 extern const blake3_ops_t blake3_generic_impl; 72 73 #if defined(__aarch64__) || \ 74 (defined(__x86_64) && defined(HAVE_SSE2)) || \ 75 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) 76 extern const blake3_ops_t blake3_sse2_impl; 77 #endif 78 79 #if defined(__aarch64__) || \ 80 (defined(__x86_64) && defined(HAVE_SSE4_1)) || \ 81 (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) 82 extern const blake3_ops_t blake3_sse41_impl; 83 #endif 84 85 #if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2) 86 extern const blake3_ops_t blake3_avx2_impl; 87 #endif 88 89 #if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL) 90 extern const blake3_ops_t blake3_avx512_impl; 91 #endif 92 93 #if defined(__x86_64) 94 #define MAX_SIMD_DEGREE 16 95 #else 96 #define MAX_SIMD_DEGREE 4 97 #endif 98 99 #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) 100 101 static const uint32_t BLAKE3_IV[8] = { 102 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, 103 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL}; 104 105 static const uint8_t BLAKE3_MSG_SCHEDULE[7][16] = { 106 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, 107 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, 108 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, 109 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, 110 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, 111 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, 112 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, 113 }; 114 115 /* Find index of the highest set bit */ 116 static inline unsigned int highest_one(uint64_t x) { 117 #if defined(__GNUC__) || defined(__clang__) 118 return (63 ^ __builtin_clzll(x)); 119 #elif defined(_MSC_VER) && defined(IS_X86_64) 120 unsigned long index; 121 _BitScanReverse64(&index, x); 122 return (index); 123 #elif defined(_MSC_VER) && defined(IS_X86_32) 124 if (x >> 32) { 125 unsigned long index; 126 _BitScanReverse(&index, x >> 32); 127 return (32 + index); 128 } else { 129 unsigned long index; 130 _BitScanReverse(&index, x); 131 return (index); 132 } 133 #else 134 unsigned int c = 0; 135 if (x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } 136 if (x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } 137 if (x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } 138 if (x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } 139 if (x & 0x000000000000000cULL) { x >>= 2; c += 2; } 140 if (x & 0x0000000000000002ULL) { c += 1; } 141 return (c); 142 #endif 143 } 144 145 /* Count the number of 1 bits. */ 146 static inline unsigned int popcnt(uint64_t x) { 147 unsigned int count = 0; 148 149 while (x != 0) { 150 count += 1; 151 x &= x - 1; 152 } 153 154 return (count); 155 } 156 157 /* 158 * Largest power of two less than or equal to x. 159 * As a special case, returns 1 when x is 0. 160 */ 161 static inline uint64_t round_down_to_power_of_2(uint64_t x) { 162 return (1ULL << highest_one(x | 1)); 163 } 164 165 static inline uint32_t counter_low(uint64_t counter) { 166 return ((uint32_t)counter); 167 } 168 169 static inline uint32_t counter_high(uint64_t counter) { 170 return ((uint32_t)(counter >> 32)); 171 } 172 173 static inline uint32_t load32(const void *src) { 174 const uint8_t *p = (const uint8_t *)src; 175 return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | 176 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); 177 } 178 179 static inline void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], 180 uint32_t key_words[8]) { 181 key_words[0] = load32(&key[0 * 4]); 182 key_words[1] = load32(&key[1 * 4]); 183 key_words[2] = load32(&key[2 * 4]); 184 key_words[3] = load32(&key[3 * 4]); 185 key_words[4] = load32(&key[4 * 4]); 186 key_words[5] = load32(&key[5 * 4]); 187 key_words[6] = load32(&key[6 * 4]); 188 key_words[7] = load32(&key[7 * 4]); 189 } 190 191 static inline void store32(void *dst, uint32_t w) { 192 uint8_t *p = (uint8_t *)dst; 193 p[0] = (uint8_t)(w >> 0); 194 p[1] = (uint8_t)(w >> 8); 195 p[2] = (uint8_t)(w >> 16); 196 p[3] = (uint8_t)(w >> 24); 197 } 198 199 static inline void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { 200 store32(&bytes_out[0 * 4], cv_words[0]); 201 store32(&bytes_out[1 * 4], cv_words[1]); 202 store32(&bytes_out[2 * 4], cv_words[2]); 203 store32(&bytes_out[3 * 4], cv_words[3]); 204 store32(&bytes_out[4 * 4], cv_words[4]); 205 store32(&bytes_out[5 * 4], cv_words[5]); 206 store32(&bytes_out[6 * 4], cv_words[6]); 207 store32(&bytes_out[7 * 4], cv_words[7]); 208 } 209 210 #ifdef __cplusplus 211 } 212 #endif 213 214 #endif /* BLAKE3_IMPL_H */ 215