1 /*- 2 * Copyright (c) 2016 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Andrew Turner under 6 * sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * This code is built with floating-point enabled. Make sure to have entered 32 * into floating-point context before calling any of these functions. 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 43 #include <opencrypto/cryptodev.h> 44 #include <crypto/armv8/armv8_crypto.h> 45 46 #include <arm_neon.h> 47 48 static uint8x16_t 49 armv8_aes_enc(int rounds, const uint8x16_t *keysched, const uint8x16_t from) 50 { 51 uint8x16_t tmp; 52 int i; 53 54 tmp = from; 55 for (i = 0; i < rounds - 1; i += 2) { 56 tmp = vaeseq_u8(tmp, keysched[i]); 57 tmp = vaesmcq_u8(tmp); 58 tmp = vaeseq_u8(tmp, keysched[i + 1]); 59 tmp = vaesmcq_u8(tmp); 60 } 61 62 tmp = vaeseq_u8(tmp, keysched[rounds - 1]); 63 tmp = vaesmcq_u8(tmp); 64 tmp = vaeseq_u8(tmp, keysched[rounds]); 65 tmp = veorq_u8(tmp, keysched[rounds + 1]); 66 67 return (tmp); 68 } 69 70 static uint8x16_t 71 armv8_aes_dec(int rounds, const uint8x16_t *keysched, const uint8x16_t from) 72 { 73 uint8x16_t tmp; 74 int i; 75 76 tmp = from; 77 for (i = 0; i < rounds - 1; i += 2) { 78 tmp = vaesdq_u8(tmp, keysched[i]); 79 tmp = vaesimcq_u8(tmp); 80 tmp = vaesdq_u8(tmp, keysched[i+1]); 81 tmp = vaesimcq_u8(tmp); 82 } 83 84 tmp = vaesdq_u8(tmp, keysched[rounds - 1]); 85 tmp = vaesimcq_u8(tmp); 86 tmp = vaesdq_u8(tmp, keysched[rounds]); 87 tmp = veorq_u8(tmp, keysched[rounds + 1]); 88 89 return (tmp); 90 } 91 92 void 93 armv8_aes_encrypt_cbc(int rounds, const void *key_schedule, size_t len, 94 const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN]) 95 { 96 uint8x16_t tot, ivreg, tmp; 97 size_t i; 98 99 len /= AES_BLOCK_LEN; 100 ivreg = vld1q_u8(iv); 101 for (i = 0; i < len; i++) { 102 tmp = vld1q_u8(from); 103 tot = armv8_aes_enc(rounds - 1, key_schedule, 104 veorq_u8(tmp, ivreg)); 105 ivreg = tot; 106 vst1q_u8(to, tot); 107 from += AES_BLOCK_LEN; 108 to += AES_BLOCK_LEN; 109 } 110 } 111 112 void 113 armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len, 114 uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN]) 115 { 116 uint8x16_t ivreg, nextiv, tmp; 117 size_t i; 118 119 len /= AES_BLOCK_LEN; 120 ivreg = vld1q_u8(iv); 121 for (i = 0; i < len; i++) { 122 nextiv = vld1q_u8(buf); 123 tmp = armv8_aes_dec(rounds - 1, key_schedule, nextiv); 124 vst1q_u8(buf, veorq_u8(tmp, ivreg)); 125 ivreg = nextiv; 126 buf += AES_BLOCK_LEN; 127 } 128 } 129