1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de> 24 */ 25 26 #include <sys/simd.h> 27 #include <sys/zfs_context.h> 28 #include <sys/zfs_impl.h> 29 #include <sys/sha2.h> 30 31 #include <sha2/sha2_impl.h> 32 #include <sys/asm_linkage.h> 33 34 #define TF(E, N) \ 35 extern void ASMABI E(uint64_t s[8], const void *, size_t); \ 36 static inline void N(uint64_t s[8], const void *d, size_t b) { \ 37 kfpu_begin(); E(s, d, b); kfpu_end(); \ 38 } 39 40 /* some implementation is always okay */ 41 static inline boolean_t sha2_is_supported(void) 42 { 43 return (B_TRUE); 44 } 45 46 #if defined(__x86_64) 47 48 /* Users of ASMABI requires all calls to be from wrappers */ 49 extern void ASMABI 50 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t); 51 52 static inline void 53 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b) 54 { 55 zfs_sha512_transform_x64(s, d, b); 56 } 57 const sha512_ops_t sha512_x64_impl = { 58 .is_supported = sha2_is_supported, 59 .transform = tf_sha512_transform_x64, 60 .name = "x64" 61 }; 62 63 #if defined(HAVE_AVX) 64 static boolean_t sha2_have_avx(void) 65 { 66 return (kfpu_allowed() && zfs_avx_available()); 67 } 68 69 TF(zfs_sha512_transform_avx, tf_sha512_avx); 70 const sha512_ops_t sha512_avx_impl = { 71 .is_supported = sha2_have_avx, 72 .transform = tf_sha512_avx, 73 .name = "avx" 74 }; 75 #endif 76 77 #if defined(HAVE_AVX2) 78 static boolean_t sha2_have_avx2(void) 79 { 80 return (kfpu_allowed() && zfs_avx2_available()); 81 } 82 83 TF(zfs_sha512_transform_avx2, tf_sha512_avx2); 84 const sha512_ops_t sha512_avx2_impl = { 85 .is_supported = sha2_have_avx2, 86 .transform = tf_sha512_avx2, 87 .name = "avx2" 88 }; 89 #endif 90 91 #elif defined(__aarch64__) || defined(__arm__) 92 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t); 93 const sha512_ops_t sha512_armv7_impl = { 94 .is_supported = sha2_is_supported, 95 .transform = zfs_sha512_block_armv7, 96 .name = "armv7" 97 }; 98 99 #if defined(__aarch64__) 100 static boolean_t sha512_have_armv8ce(void) 101 { 102 return (kfpu_allowed() && zfs_sha512_available()); 103 } 104 105 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce); 106 const sha512_ops_t sha512_armv8_impl = { 107 .is_supported = sha512_have_armv8ce, 108 .transform = tf_sha512_armv8ce, 109 .name = "armv8-ce" 110 }; 111 #endif 112 113 #if defined(__arm__) && __ARM_ARCH > 6 114 static boolean_t sha512_have_neon(void) 115 { 116 return (kfpu_allowed() && zfs_neon_available()); 117 } 118 119 TF(zfs_sha512_block_neon, tf_sha512_neon); 120 const sha512_ops_t sha512_neon_impl = { 121 .is_supported = sha512_have_neon, 122 .transform = tf_sha512_neon, 123 .name = "neon" 124 }; 125 #endif 126 127 #elif defined(__PPC64__) 128 TF(zfs_sha512_ppc, tf_sha512_ppc); 129 const sha512_ops_t sha512_ppc_impl = { 130 .is_supported = sha2_is_supported, 131 .transform = tf_sha512_ppc, 132 .name = "ppc" 133 }; 134 135 static boolean_t sha512_have_isa207(void) 136 { 137 return (kfpu_allowed() && zfs_isa207_available()); 138 } 139 140 TF(zfs_sha512_power8, tf_sha512_power8); 141 const sha512_ops_t sha512_power8_impl = { 142 .is_supported = sha512_have_isa207, 143 .transform = tf_sha512_power8, 144 .name = "power8" 145 }; 146 #endif /* __PPC64__ */ 147 148 /* the two generic ones */ 149 extern const sha512_ops_t sha512_generic_impl; 150 151 /* array with all sha512 implementations */ 152 static const sha512_ops_t *const sha512_impls[] = { 153 &sha512_generic_impl, 154 #if defined(__x86_64) 155 &sha512_x64_impl, 156 #endif 157 #if defined(__x86_64) && defined(HAVE_AVX) 158 &sha512_avx_impl, 159 #endif 160 #if defined(__x86_64) && defined(HAVE_AVX2) 161 &sha512_avx2_impl, 162 #endif 163 #if defined(__aarch64__) || defined(__arm__) 164 &sha512_armv7_impl, 165 #if defined(__aarch64__) 166 &sha512_armv8_impl, 167 #endif 168 #if defined(__arm__) && __ARM_ARCH > 6 169 &sha512_neon_impl, 170 #endif 171 #endif 172 #if defined(__PPC64__) 173 &sha512_ppc_impl, 174 &sha512_power8_impl, 175 #endif /* __PPC64__ */ 176 }; 177 178 /* use the generic implementation functions */ 179 #define IMPL_NAME "sha512" 180 #define IMPL_OPS_T sha512_ops_t 181 #define IMPL_ARRAY sha512_impls 182 #define IMPL_GET_OPS sha512_get_ops 183 #define ZFS_IMPL_OPS zfs_sha512_ops 184 #include <generic_impl.c> 185 186 #ifdef _KERNEL 187 188 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 189 190 #if defined(__linux__) 191 192 static int 193 sha512_param_get(char *buffer, zfs_kernel_param_t *unused) 194 { 195 const uint32_t impl = IMPL_READ(generic_impl_chosen); 196 char *fmt; 197 int cnt = 0; 198 199 /* cycling */ 200 fmt = IMPL_FMT(impl, IMPL_CYCLE); 201 cnt += sprintf(buffer + cnt, fmt, "cycle"); 202 203 /* list fastest */ 204 fmt = IMPL_FMT(impl, IMPL_FASTEST); 205 cnt += sprintf(buffer + cnt, fmt, "fastest"); 206 207 /* list all supported implementations */ 208 generic_impl_init(); 209 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 210 fmt = IMPL_FMT(impl, i); 211 cnt += sprintf(buffer + cnt, fmt, 212 generic_supp_impls[i]->name); 213 } 214 215 return (cnt); 216 } 217 218 static int 219 sha512_param_set(const char *val, zfs_kernel_param_t *unused) 220 { 221 (void) unused; 222 return (generic_impl_setname(val)); 223 } 224 225 #elif defined(__FreeBSD__) 226 227 #include <sys/sbuf.h> 228 229 static int 230 sha512_param(ZFS_MODULE_PARAM_ARGS) 231 { 232 int err; 233 234 generic_impl_init(); 235 if (req->newptr == NULL) { 236 const uint32_t impl = IMPL_READ(generic_impl_chosen); 237 const int init_buflen = 64; 238 const char *fmt; 239 struct sbuf *s; 240 241 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 242 243 /* cycling */ 244 fmt = IMPL_FMT(impl, IMPL_CYCLE); 245 (void) sbuf_printf(s, fmt, "cycle"); 246 247 /* list fastest */ 248 fmt = IMPL_FMT(impl, IMPL_FASTEST); 249 (void) sbuf_printf(s, fmt, "fastest"); 250 251 /* list all supported implementations */ 252 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) { 253 fmt = IMPL_FMT(impl, i); 254 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name); 255 } 256 257 err = sbuf_finish(s); 258 sbuf_delete(s); 259 260 return (err); 261 } 262 263 /* we got module parameter */ 264 char buf[16]; 265 266 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 267 if (err) { 268 return (err); 269 } 270 271 return (-generic_impl_setname(buf)); 272 } 273 #endif 274 275 #undef IMPL_FMT 276 277 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl, 278 sha512_param_set, sha512_param_get, ZMOD_RW, \ 279 "Select SHA512 implementation."); 280 #endif 281 282 #undef TF 283