1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <sys/zfs_context.h>
26 #include <sys/crypto/icp.h>
27 #include <sys/crypto/spi.h>
28 #include <sys/simd.h>
29 #include <modes/modes.h>
30 #include <aes/aes_impl.h>
31
32 /*
33 * Initialize AES encryption and decryption key schedules.
34 *
35 * Parameters:
36 * cipherKey User key
37 * keyBits AES key size (128, 192, or 256 bits)
38 * keysched AES key schedule to be initialized, of type aes_key_t.
39 * Allocated by aes_alloc_keysched().
40 */
41 void
aes_init_keysched(const uint8_t * cipherKey,uint_t keyBits,void * keysched)42 aes_init_keysched(const uint8_t *cipherKey, uint_t keyBits, void *keysched)
43 {
44 const aes_impl_ops_t *ops = aes_impl_get_ops();
45 aes_key_t *newbie = keysched;
46 uint_t keysize, i, j;
47 union {
48 uint64_t ka64[4];
49 uint32_t ka32[8];
50 } keyarr;
51
52 switch (keyBits) {
53 case 128:
54 newbie->nr = 10;
55 break;
56
57 case 192:
58 newbie->nr = 12;
59 break;
60
61 case 256:
62 newbie->nr = 14;
63 break;
64
65 default:
66 /* should never get here */
67 return;
68 }
69 keysize = CRYPTO_BITS2BYTES(keyBits);
70
71 /*
72 * Generic C implementation requires byteswap for little endian
73 * machines, various accelerated implementations for various
74 * architectures may not.
75 */
76 if (!ops->needs_byteswap) {
77 /* no byteswap needed */
78 if (IS_P2ALIGNED(cipherKey, sizeof (uint64_t))) {
79 for (i = 0, j = 0; j < keysize; i++, j += 8) {
80 /* LINTED: pointer alignment */
81 keyarr.ka64[i] = *((uint64_t *)&cipherKey[j]);
82 }
83 } else {
84 memcpy(keyarr.ka32, cipherKey, keysize);
85 }
86 } else {
87 /* byte swap */
88 for (i = 0, j = 0; j < keysize; i++, j += 4) {
89 keyarr.ka32[i] =
90 htonl(*(uint32_t *)(void *)&cipherKey[j]);
91 }
92 }
93
94 ops->generate(newbie, keyarr.ka32, keyBits);
95 newbie->ops = ops;
96
97 /*
98 * Note: if there are systems that need the AES_64BIT_KS type in the
99 * future, move setting key schedule type to individual implementations
100 */
101 newbie->type = AES_32BIT_KS;
102 }
103
104
105 /*
106 * Encrypt one block using AES.
107 * Align if needed and (for x86 32-bit only) byte-swap.
108 *
109 * Parameters:
110 * ks Key schedule, of type aes_key_t
111 * pt Input block (plain text)
112 * ct Output block (crypto text). Can overlap with pt
113 */
114 int
aes_encrypt_block(const void * ks,const uint8_t * pt,uint8_t * ct)115 aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct)
116 {
117 aes_key_t *ksch = (aes_key_t *)ks;
118 const aes_impl_ops_t *ops = ksch->ops;
119
120 if (IS_P2ALIGNED2(pt, ct, sizeof (uint32_t)) && !ops->needs_byteswap) {
121 /* LINTED: pointer alignment */
122 ops->encrypt(&ksch->encr_ks.ks32[0], ksch->nr,
123 /* LINTED: pointer alignment */
124 (uint32_t *)pt, (uint32_t *)ct);
125 } else {
126 uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)];
127
128 /* Copy input block into buffer */
129 if (ops->needs_byteswap) {
130 buffer[0] = htonl(*(uint32_t *)(void *)&pt[0]);
131 buffer[1] = htonl(*(uint32_t *)(void *)&pt[4]);
132 buffer[2] = htonl(*(uint32_t *)(void *)&pt[8]);
133 buffer[3] = htonl(*(uint32_t *)(void *)&pt[12]);
134 } else
135 memcpy(&buffer, pt, AES_BLOCK_LEN);
136
137 ops->encrypt(&ksch->encr_ks.ks32[0], ksch->nr, buffer, buffer);
138
139 /* Copy result from buffer to output block */
140 if (ops->needs_byteswap) {
141 *(uint32_t *)(void *)&ct[0] = htonl(buffer[0]);
142 *(uint32_t *)(void *)&ct[4] = htonl(buffer[1]);
143 *(uint32_t *)(void *)&ct[8] = htonl(buffer[2]);
144 *(uint32_t *)(void *)&ct[12] = htonl(buffer[3]);
145 } else
146 memcpy(ct, &buffer, AES_BLOCK_LEN);
147 }
148 return (CRYPTO_SUCCESS);
149 }
150
151
152 /*
153 * Decrypt one block using AES.
154 * Align and byte-swap if needed.
155 *
156 * Parameters:
157 * ks Key schedule, of type aes_key_t
158 * ct Input block (crypto text)
159 * pt Output block (plain text). Can overlap with pt
160 */
161 int
aes_decrypt_block(const void * ks,const uint8_t * ct,uint8_t * pt)162 aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt)
163 {
164 aes_key_t *ksch = (aes_key_t *)ks;
165 const aes_impl_ops_t *ops = ksch->ops;
166
167 if (IS_P2ALIGNED2(ct, pt, sizeof (uint32_t)) && !ops->needs_byteswap) {
168 /* LINTED: pointer alignment */
169 ops->decrypt(&ksch->decr_ks.ks32[0], ksch->nr,
170 /* LINTED: pointer alignment */
171 (uint32_t *)ct, (uint32_t *)pt);
172 } else {
173 uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)];
174
175 /* Copy input block into buffer */
176 if (ops->needs_byteswap) {
177 buffer[0] = htonl(*(uint32_t *)(void *)&ct[0]);
178 buffer[1] = htonl(*(uint32_t *)(void *)&ct[4]);
179 buffer[2] = htonl(*(uint32_t *)(void *)&ct[8]);
180 buffer[3] = htonl(*(uint32_t *)(void *)&ct[12]);
181 } else
182 memcpy(&buffer, ct, AES_BLOCK_LEN);
183
184 ops->decrypt(&ksch->decr_ks.ks32[0], ksch->nr, buffer, buffer);
185
186 /* Copy result from buffer to output block */
187 if (ops->needs_byteswap) {
188 *(uint32_t *)(void *)&pt[0] = htonl(buffer[0]);
189 *(uint32_t *)(void *)&pt[4] = htonl(buffer[1]);
190 *(uint32_t *)(void *)&pt[8] = htonl(buffer[2]);
191 *(uint32_t *)(void *)&pt[12] = htonl(buffer[3]);
192 } else
193 memcpy(pt, &buffer, AES_BLOCK_LEN);
194 }
195 return (CRYPTO_SUCCESS);
196 }
197
198
199 /*
200 * Allocate key schedule for AES.
201 *
202 * Return the pointer and set size to the number of bytes allocated.
203 * Memory allocated must be freed by the caller when done.
204 *
205 * Parameters:
206 * size Size of key schedule allocated, in bytes
207 * kmflag Flag passed to kmem_alloc(9F); ignored in userland.
208 */
209 void *
aes_alloc_keysched(size_t * size,int kmflag)210 aes_alloc_keysched(size_t *size, int kmflag)
211 {
212 aes_key_t *keysched;
213
214 keysched = kmem_alloc(sizeof (aes_key_t), kmflag);
215 if (keysched != NULL) {
216 *size = sizeof (aes_key_t);
217 return (keysched);
218 }
219 return (NULL);
220 }
221
222 /* AES implementation that contains the fastest methods */
223 static aes_impl_ops_t aes_fastest_impl = {
224 .name = "fastest"
225 };
226
227 /* All compiled in implementations */
228 static const aes_impl_ops_t *aes_all_impl[] = {
229 &aes_generic_impl,
230 #if defined(__x86_64)
231 &aes_x86_64_impl,
232 #endif
233 #if defined(__x86_64) && defined(HAVE_AES)
234 &aes_aesni_impl,
235 #endif
236 };
237
238 /* Indicate that benchmark has been completed */
239 static boolean_t aes_impl_initialized = B_FALSE;
240
241 /* Select aes implementation */
242 #define IMPL_FASTEST (UINT32_MAX)
243 #define IMPL_CYCLE (UINT32_MAX-1)
244
245 #define AES_IMPL_READ(i) (*(volatile uint32_t *) &(i))
246
247 static uint32_t icp_aes_impl = IMPL_FASTEST;
248 static uint32_t user_sel_impl = IMPL_FASTEST;
249
250 /* Hold all supported implementations */
251 static size_t aes_supp_impl_cnt = 0;
252 static aes_impl_ops_t *aes_supp_impl[ARRAY_SIZE(aes_all_impl)];
253
254 /*
255 * Returns the AES operations for encrypt/decrypt/key setup. When a
256 * SIMD implementation is not allowed in the current context, then
257 * fallback to the fastest generic implementation.
258 */
259 const aes_impl_ops_t *
aes_impl_get_ops(void)260 aes_impl_get_ops(void)
261 {
262 if (!kfpu_allowed())
263 return (&aes_generic_impl);
264
265 const aes_impl_ops_t *ops = NULL;
266 const uint32_t impl = AES_IMPL_READ(icp_aes_impl);
267
268 switch (impl) {
269 case IMPL_FASTEST:
270 ASSERT(aes_impl_initialized);
271 ops = &aes_fastest_impl;
272 break;
273 case IMPL_CYCLE:
274 /* Cycle through supported implementations */
275 ASSERT(aes_impl_initialized);
276 ASSERT3U(aes_supp_impl_cnt, >, 0);
277 static size_t cycle_impl_idx = 0;
278 size_t idx = (++cycle_impl_idx) % aes_supp_impl_cnt;
279 ops = aes_supp_impl[idx];
280 break;
281 default:
282 ASSERT3U(impl, <, aes_supp_impl_cnt);
283 ASSERT3U(aes_supp_impl_cnt, >, 0);
284 if (impl < ARRAY_SIZE(aes_all_impl))
285 ops = aes_supp_impl[impl];
286 break;
287 }
288
289 ASSERT3P(ops, !=, NULL);
290
291 return (ops);
292 }
293
294 /*
295 * Initialize all supported implementations.
296 */
297 void
aes_impl_init(void)298 aes_impl_init(void)
299 {
300 aes_impl_ops_t *curr_impl;
301 int i, c;
302
303 /* Move supported implementations into aes_supp_impls */
304 for (i = 0, c = 0; i < ARRAY_SIZE(aes_all_impl); i++) {
305 curr_impl = (aes_impl_ops_t *)aes_all_impl[i];
306
307 if (curr_impl->is_supported())
308 aes_supp_impl[c++] = (aes_impl_ops_t *)curr_impl;
309 }
310 aes_supp_impl_cnt = c;
311
312 /*
313 * Set the fastest implementation given the assumption that the
314 * hardware accelerated version is the fastest.
315 */
316 #if defined(__x86_64)
317 #if defined(HAVE_AES)
318 if (aes_aesni_impl.is_supported()) {
319 memcpy(&aes_fastest_impl, &aes_aesni_impl,
320 sizeof (aes_fastest_impl));
321 } else
322 #endif
323 {
324 memcpy(&aes_fastest_impl, &aes_x86_64_impl,
325 sizeof (aes_fastest_impl));
326 }
327 #else
328 memcpy(&aes_fastest_impl, &aes_generic_impl,
329 sizeof (aes_fastest_impl));
330 #endif
331
332 strlcpy(aes_fastest_impl.name, "fastest", AES_IMPL_NAME_MAX);
333
334 /* Finish initialization */
335 atomic_swap_32(&icp_aes_impl, user_sel_impl);
336 aes_impl_initialized = B_TRUE;
337 }
338
339 static const struct {
340 const char *name;
341 uint32_t sel;
342 } aes_impl_opts[] = {
343 { "cycle", IMPL_CYCLE },
344 { "fastest", IMPL_FASTEST },
345 };
346
347 /*
348 * Function sets desired aes implementation.
349 *
350 * If we are called before init(), user preference will be saved in
351 * user_sel_impl, and applied in later init() call. This occurs when module
352 * parameter is specified on module load. Otherwise, directly update
353 * icp_aes_impl.
354 *
355 * @val Name of aes implementation to use
356 * @param Unused.
357 */
358 int
aes_impl_set(const char * val)359 aes_impl_set(const char *val)
360 {
361 int err = -EINVAL;
362 char req_name[AES_IMPL_NAME_MAX];
363 uint32_t impl = AES_IMPL_READ(user_sel_impl);
364 size_t i;
365
366 /* sanitize input */
367 i = strnlen(val, AES_IMPL_NAME_MAX);
368 if (i == 0 || i >= AES_IMPL_NAME_MAX)
369 return (err);
370
371 strlcpy(req_name, val, AES_IMPL_NAME_MAX);
372 while (i > 0 && isspace(req_name[i-1]))
373 i--;
374 req_name[i] = '\0';
375
376 /* Check mandatory options */
377 for (i = 0; i < ARRAY_SIZE(aes_impl_opts); i++) {
378 if (strcmp(req_name, aes_impl_opts[i].name) == 0) {
379 impl = aes_impl_opts[i].sel;
380 err = 0;
381 break;
382 }
383 }
384
385 /* check all supported impl if init() was already called */
386 if (err != 0 && aes_impl_initialized) {
387 /* check all supported implementations */
388 for (i = 0; i < aes_supp_impl_cnt; i++) {
389 if (strcmp(req_name, aes_supp_impl[i]->name) == 0) {
390 impl = i;
391 err = 0;
392 break;
393 }
394 }
395 }
396
397 if (err == 0) {
398 if (aes_impl_initialized)
399 atomic_swap_32(&icp_aes_impl, impl);
400 else
401 atomic_swap_32(&user_sel_impl, impl);
402 }
403
404 return (err);
405 }
406
407 #if defined(_KERNEL) && defined(__linux__)
408
409 static int
icp_aes_impl_set(const char * val,zfs_kernel_param_t * kp)410 icp_aes_impl_set(const char *val, zfs_kernel_param_t *kp)
411 {
412 return (aes_impl_set(val));
413 }
414
415 static int
icp_aes_impl_get(char * buffer,zfs_kernel_param_t * kp)416 icp_aes_impl_get(char *buffer, zfs_kernel_param_t *kp)
417 {
418 int i, cnt = 0;
419 char *fmt;
420 const uint32_t impl = AES_IMPL_READ(icp_aes_impl);
421
422 ASSERT(aes_impl_initialized);
423
424 /* list mandatory options */
425 for (i = 0; i < ARRAY_SIZE(aes_impl_opts); i++) {
426 fmt = (impl == aes_impl_opts[i].sel) ? "[%s] " : "%s ";
427 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,
428 aes_impl_opts[i].name);
429 }
430
431 /* list all supported implementations */
432 for (i = 0; i < aes_supp_impl_cnt; i++) {
433 fmt = (i == impl) ? "[%s] " : "%s ";
434 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,
435 aes_supp_impl[i]->name);
436 }
437
438 return (cnt);
439 }
440
441 module_param_call(icp_aes_impl, icp_aes_impl_set, icp_aes_impl_get,
442 NULL, 0644);
443 MODULE_PARM_DESC(icp_aes_impl, "Select aes implementation.");
444 #endif
445