1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
24 */
25
26 #include <sys/simd.h>
27 #include <sys/zfs_context.h>
28 #include <sys/zfs_impl.h>
29 #include <sys/sha2.h>
30
31 #include <sha2/sha2_impl.h>
32 #include <sys/asm_linkage.h>
33
34 #define TF(E, N) \
35 extern void ASMABI E(uint32_t s[8], const void *, size_t); \
36 static inline void N(uint32_t s[8], const void *d, size_t b) { \
37 kfpu_begin(); E(s, d, b); kfpu_end(); \
38 }
39
40 /* some implementation is always okay */
sha2_is_supported(void)41 static inline boolean_t sha2_is_supported(void)
42 {
43 return (B_TRUE);
44 }
45
46 #if defined(__x86_64)
47
48 /* Users of ASMABI requires all calls to be from wrappers */
49 extern void ASMABI
50 zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
51
52 static inline void
tf_sha256_transform_x64(uint32_t s[8],const void * d,size_t b)53 tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b)
54 {
55 zfs_sha256_transform_x64(s, d, b);
56 }
57
58 const sha256_ops_t sha256_x64_impl = {
59 .is_supported = sha2_is_supported,
60 .transform = tf_sha256_transform_x64,
61 .name = "x64"
62 };
63
64 #if defined(HAVE_SSSE3)
sha2_have_ssse3(void)65 static boolean_t sha2_have_ssse3(void)
66 {
67 return (kfpu_allowed() && zfs_ssse3_available());
68 }
69
70 TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3);
71 const sha256_ops_t sha256_ssse3_impl = {
72 .is_supported = sha2_have_ssse3,
73 .transform = tf_sha256_ssse3,
74 .name = "ssse3"
75 };
76 #endif
77
78 #if defined(HAVE_AVX)
sha2_have_avx(void)79 static boolean_t sha2_have_avx(void)
80 {
81 return (kfpu_allowed() && zfs_avx_available());
82 }
83
84 TF(zfs_sha256_transform_avx, tf_sha256_avx);
85 const sha256_ops_t sha256_avx_impl = {
86 .is_supported = sha2_have_avx,
87 .transform = tf_sha256_avx,
88 .name = "avx"
89 };
90 #endif
91
92 #if defined(HAVE_AVX2)
sha2_have_avx2(void)93 static boolean_t sha2_have_avx2(void)
94 {
95 return (kfpu_allowed() && zfs_avx2_available());
96 }
97
98 TF(zfs_sha256_transform_avx2, tf_sha256_avx2);
99 const sha256_ops_t sha256_avx2_impl = {
100 .is_supported = sha2_have_avx2,
101 .transform = tf_sha256_avx2,
102 .name = "avx2"
103 };
104 #endif
105
106 #if defined(HAVE_SSE4_1)
sha2_have_shani(void)107 static boolean_t sha2_have_shani(void)
108 {
109 return (kfpu_allowed() && zfs_sse4_1_available() && \
110 zfs_shani_available());
111 }
112
113 TF(zfs_sha256_transform_shani, tf_sha256_shani);
114 const sha256_ops_t sha256_shani_impl = {
115 .is_supported = sha2_have_shani,
116 .transform = tf_sha256_shani,
117 .name = "shani"
118 };
119 #endif
120
121 #elif defined(__aarch64__) || defined(__arm__)
122 extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t);
123 const sha256_ops_t sha256_armv7_impl = {
124 .is_supported = sha2_is_supported,
125 .transform = zfs_sha256_block_armv7,
126 .name = "armv7"
127 };
128
129 #if __ARM_ARCH > 6
sha256_have_neon(void)130 static boolean_t sha256_have_neon(void)
131 {
132 return (kfpu_allowed() && zfs_neon_available());
133 }
134
sha256_have_armv8ce(void)135 static boolean_t sha256_have_armv8ce(void)
136 {
137 return (kfpu_allowed() && zfs_sha256_available());
138 }
139
140 TF(zfs_sha256_block_neon, tf_sha256_neon);
141 const sha256_ops_t sha256_neon_impl = {
142 .is_supported = sha256_have_neon,
143 .transform = tf_sha256_neon,
144 .name = "neon"
145 };
146
147 TF(zfs_sha256_block_armv8, tf_sha256_armv8ce);
148 const sha256_ops_t sha256_armv8_impl = {
149 .is_supported = sha256_have_armv8ce,
150 .transform = tf_sha256_armv8ce,
151 .name = "armv8-ce"
152 };
153 #endif
154
155 #elif defined(__PPC64__)
sha256_have_isa207(void)156 static boolean_t sha256_have_isa207(void)
157 {
158 return (kfpu_allowed() && zfs_isa207_available());
159 }
160
161 TF(zfs_sha256_ppc, tf_sha256_ppc);
162 const sha256_ops_t sha256_ppc_impl = {
163 .is_supported = sha2_is_supported,
164 .transform = tf_sha256_ppc,
165 .name = "ppc"
166 };
167
168 TF(zfs_sha256_power8, tf_sha256_power8);
169 const sha256_ops_t sha256_power8_impl = {
170 .is_supported = sha256_have_isa207,
171 .transform = tf_sha256_power8,
172 .name = "power8"
173 };
174 #endif /* __PPC64__ */
175
176 /* the two generic ones */
177 extern const sha256_ops_t sha256_generic_impl;
178
179 /* array with all sha256 implementations */
180 static const sha256_ops_t *const sha256_impls[] = {
181 &sha256_generic_impl,
182 #if defined(__x86_64)
183 &sha256_x64_impl,
184 #endif
185 #if defined(__x86_64) && defined(HAVE_SSSE3)
186 &sha256_ssse3_impl,
187 #endif
188 #if defined(__x86_64) && defined(HAVE_AVX)
189 &sha256_avx_impl,
190 #endif
191 #if defined(__x86_64) && defined(HAVE_AVX2)
192 &sha256_avx2_impl,
193 #endif
194 #if defined(__x86_64) && defined(HAVE_SSE4_1)
195 &sha256_shani_impl,
196 #endif
197 #if defined(__aarch64__) || defined(__arm__)
198 &sha256_armv7_impl,
199 #if __ARM_ARCH > 6
200 &sha256_neon_impl,
201 &sha256_armv8_impl,
202 #endif
203 #endif
204 #if defined(__PPC64__)
205 &sha256_ppc_impl,
206 &sha256_power8_impl,
207 #endif /* __PPC64__ */
208 };
209
210 /* use the generic implementation functions */
211 #define IMPL_NAME "sha256"
212 #define IMPL_OPS_T sha256_ops_t
213 #define IMPL_ARRAY sha256_impls
214 #define IMPL_GET_OPS sha256_get_ops
215 #define ZFS_IMPL_OPS zfs_sha256_ops
216 #include <generic_impl.c>
217
218 #ifdef _KERNEL
219
220 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
221
222 #if defined(__linux__)
223
224 static int
sha256_param_get(char * buffer,zfs_kernel_param_t * unused)225 sha256_param_get(char *buffer, zfs_kernel_param_t *unused)
226 {
227 const uint32_t impl = IMPL_READ(generic_impl_chosen);
228 char *fmt;
229 int cnt = 0;
230
231 /* cycling */
232 fmt = IMPL_FMT(impl, IMPL_CYCLE);
233 cnt += sprintf(buffer + cnt, fmt, "cycle");
234
235 /* list fastest */
236 fmt = IMPL_FMT(impl, IMPL_FASTEST);
237 cnt += sprintf(buffer + cnt, fmt, "fastest");
238
239 /* list all supported implementations */
240 generic_impl_init();
241 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
242 fmt = IMPL_FMT(impl, i);
243 cnt += sprintf(buffer + cnt, fmt,
244 generic_supp_impls[i]->name);
245 }
246
247 return (cnt);
248 }
249
250 static int
sha256_param_set(const char * val,zfs_kernel_param_t * unused)251 sha256_param_set(const char *val, zfs_kernel_param_t *unused)
252 {
253 (void) unused;
254 return (generic_impl_setname(val));
255 }
256
257 #elif defined(__FreeBSD__)
258
259 #include <sys/sbuf.h>
260
261 static int
sha256_param(ZFS_MODULE_PARAM_ARGS)262 sha256_param(ZFS_MODULE_PARAM_ARGS)
263 {
264 int err;
265
266 generic_impl_init();
267 if (req->newptr == NULL) {
268 const uint32_t impl = IMPL_READ(generic_impl_chosen);
269 const int init_buflen = 64;
270 const char *fmt;
271 struct sbuf *s;
272
273 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
274
275 /* cycling */
276 fmt = IMPL_FMT(impl, IMPL_CYCLE);
277 (void) sbuf_printf(s, fmt, "cycle");
278
279 /* list fastest */
280 fmt = IMPL_FMT(impl, IMPL_FASTEST);
281 (void) sbuf_printf(s, fmt, "fastest");
282
283 /* list all supported implementations */
284 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
285 fmt = IMPL_FMT(impl, i);
286 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
287 }
288
289 err = sbuf_finish(s);
290 sbuf_delete(s);
291
292 return (err);
293 }
294
295 char buf[16];
296
297 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
298 if (err) {
299 return (err);
300 }
301
302 return (-generic_impl_setname(buf));
303 }
304 #endif
305
306 #undef IMPL_FMT
307
308 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl,
309 sha256_param_set, sha256_param_get, ZMOD_RW, \
310 "Select SHA256 implementation.");
311 #endif
312
313 #undef TF
314