1 /* $NetBSD: aes_neon_impl.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(1, "$NetBSD: aes_neon_impl.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $"); 31 32 #include <sys/types.h> 33 #include <sys/proc.h> 34 35 #include <crypto/aes/aes.h> 36 #include <crypto/aes/aes_impl.h> 37 #include <crypto/aes/arch/arm/aes_neon.h> 38 39 #ifdef __aarch64__ 40 #include <aarch64/armreg.h> 41 #endif 42 43 #ifdef _KERNEL 44 #ifndef __aarch64__ 45 #include <arm/locore.h> 46 #endif 47 #include <arm/fpu.h> 48 #else 49 #include <sys/sysctl.h> 50 #include <stddef.h> 51 #define fpu_kern_enter() ((void)0) 52 #define fpu_kern_leave() ((void)0) 53 #endif 54 55 static void 56 aes_neon_setenckey_impl(struct aesenc *enc, const uint8_t *key, 57 uint32_t nrounds) 58 { 59 60 fpu_kern_enter(); 61 aes_neon_setenckey(enc, key, nrounds); 62 fpu_kern_leave(); 63 } 64 65 static void 66 aes_neon_setdeckey_impl(struct aesdec *dec, const uint8_t *key, 67 uint32_t nrounds) 68 { 69 70 fpu_kern_enter(); 71 aes_neon_setdeckey(dec, key, nrounds); 72 fpu_kern_leave(); 73 } 74 75 static void 76 aes_neon_enc_impl(const struct aesenc *enc, const uint8_t in[static 16], 77 uint8_t out[static 16], uint32_t nrounds) 78 { 79 80 fpu_kern_enter(); 81 aes_neon_enc(enc, in, out, nrounds); 82 fpu_kern_leave(); 83 } 84 85 static void 86 aes_neon_dec_impl(const struct aesdec *dec, const uint8_t in[static 16], 87 uint8_t out[static 16], uint32_t nrounds) 88 { 89 90 fpu_kern_enter(); 91 aes_neon_dec(dec, in, out, nrounds); 92 fpu_kern_leave(); 93 } 94 95 static void 96 aes_neon_cbc_enc_impl(const struct aesenc *enc, const uint8_t in[static 16], 97 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 98 uint32_t nrounds) 99 { 100 101 if (nbytes == 0) 102 return; 103 fpu_kern_enter(); 104 aes_neon_cbc_enc(enc, in, out, nbytes, iv, nrounds); 105 fpu_kern_leave(); 106 } 107 108 static void 109 aes_neon_cbc_dec_impl(const struct aesdec *dec, const uint8_t in[static 16], 110 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 111 uint32_t nrounds) 112 { 113 114 if (nbytes == 0) 115 return; 116 fpu_kern_enter(); 117 aes_neon_cbc_dec(dec, in, out, nbytes, iv, nrounds); 118 fpu_kern_leave(); 119 } 120 121 static void 122 aes_neon_xts_enc_impl(const struct aesenc *enc, const uint8_t in[static 16], 123 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 124 uint32_t nrounds) 125 { 126 127 if (nbytes == 0) 128 return; 129 fpu_kern_enter(); 130 aes_neon_xts_enc(enc, in, out, nbytes, iv, nrounds); 131 fpu_kern_leave(); 132 } 133 134 static void 135 aes_neon_xts_dec_impl(const struct aesdec *dec, const uint8_t in[static 16], 136 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 137 uint32_t nrounds) 138 { 139 140 if (nbytes == 0) 141 return; 142 fpu_kern_enter(); 143 aes_neon_xts_dec(dec, in, out, nbytes, iv, nrounds); 144 fpu_kern_leave(); 145 } 146 147 static int 148 aes_neon_probe(void) 149 { 150 #ifdef __aarch64__ 151 struct aarch64_sysctl_cpu_id *id; 152 #endif 153 int result = 0; 154 155 /* Verify that the CPU supports NEON. */ 156 #ifdef __aarch64__ 157 #ifdef _KERNEL 158 id = &curcpu()->ci_id; 159 #else 160 struct aarch64_sysctl_cpu_id ids; 161 size_t idlen; 162 id = &ids; 163 idlen = sizeof ids; 164 if (sysctlbyname("machdep.cpu0.cpu_id", id, &idlen, NULL, 0)) 165 return -1; 166 if (idlen != sizeof ids) 167 return -1; 168 #endif 169 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) { 170 case ID_AA64PFR0_EL1_ADV_SIMD_IMPL: 171 break; 172 default: 173 return -1; 174 } 175 #else 176 #ifdef _KERNEL 177 if (!cpu_neon_present) 178 return -1; 179 #else 180 int neon; 181 size_t neonlen = sizeof neon; 182 if (0 && sysctlbyname("machdep.neon_present", &neon, &neonlen, NULL, 0)) 183 return -1; 184 if (0 && !neon) 185 return -1; 186 #endif 187 #endif 188 189 fpu_kern_enter(); 190 result = aes_neon_selftest(); 191 fpu_kern_leave(); 192 193 return result; 194 } 195 196 struct aes_impl aes_neon_impl = { 197 .ai_name = "ARM NEON vpaes", 198 .ai_probe = aes_neon_probe, 199 .ai_setenckey = aes_neon_setenckey_impl, 200 .ai_setdeckey = aes_neon_setdeckey_impl, 201 .ai_enc = aes_neon_enc_impl, 202 .ai_dec = aes_neon_dec_impl, 203 .ai_cbc_enc = aes_neon_cbc_enc_impl, 204 .ai_cbc_dec = aes_neon_cbc_dec_impl, 205 .ai_xts_enc = aes_neon_xts_enc_impl, 206 .ai_xts_dec = aes_neon_xts_dec_impl, 207 }; 208