1b36d8c09SArd Biesheuvel // SPDX-License-Identifier: GPL-2.0
2b36d8c09SArd Biesheuvel /*
3b36d8c09SArd Biesheuvel * ARM NEON accelerated ChaCha and XChaCha stream ciphers,
4b36d8c09SArd Biesheuvel * including ChaCha20 (RFC7539)
5b36d8c09SArd Biesheuvel *
6b36d8c09SArd Biesheuvel * Copyright (C) 2016-2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
7b36d8c09SArd Biesheuvel * Copyright (C) 2015 Martin Willi
8b36d8c09SArd Biesheuvel */
9b36d8c09SArd Biesheuvel
10b36d8c09SArd Biesheuvel #include <crypto/algapi.h>
11b36d8c09SArd Biesheuvel #include <crypto/internal/chacha.h>
12b36d8c09SArd Biesheuvel #include <crypto/internal/simd.h>
13b36d8c09SArd Biesheuvel #include <crypto/internal/skcipher.h>
14a44a3430SArd Biesheuvel #include <linux/jump_label.h>
15b36d8c09SArd Biesheuvel #include <linux/kernel.h>
16b36d8c09SArd Biesheuvel #include <linux/module.h>
17b36d8c09SArd Biesheuvel
18b36d8c09SArd Biesheuvel #include <asm/cputype.h>
19b36d8c09SArd Biesheuvel #include <asm/hwcap.h>
20b36d8c09SArd Biesheuvel #include <asm/neon.h>
21b36d8c09SArd Biesheuvel #include <asm/simd.h>
22b36d8c09SArd Biesheuvel
23b36d8c09SArd Biesheuvel asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
24b36d8c09SArd Biesheuvel int nrounds);
25b36d8c09SArd Biesheuvel asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
2686cd97ecSArd Biesheuvel int nrounds, unsigned int nbytes);
27b36d8c09SArd Biesheuvel asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds);
28b36d8c09SArd Biesheuvel asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
29b36d8c09SArd Biesheuvel
30b36d8c09SArd Biesheuvel asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
31b36d8c09SArd Biesheuvel const u32 *state, int nrounds);
32b36d8c09SArd Biesheuvel
33a44a3430SArd Biesheuvel static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);
34a44a3430SArd Biesheuvel
neon_usable(void)35b36d8c09SArd Biesheuvel static inline bool neon_usable(void)
36b36d8c09SArd Biesheuvel {
37a44a3430SArd Biesheuvel return static_branch_likely(&use_neon) && crypto_simd_usable();
38b36d8c09SArd Biesheuvel }
39b36d8c09SArd Biesheuvel
chacha_doneon(u32 * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)40b36d8c09SArd Biesheuvel static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
41b36d8c09SArd Biesheuvel unsigned int bytes, int nrounds)
42b36d8c09SArd Biesheuvel {
43b36d8c09SArd Biesheuvel u8 buf[CHACHA_BLOCK_SIZE];
44b36d8c09SArd Biesheuvel
4586cd97ecSArd Biesheuvel while (bytes > CHACHA_BLOCK_SIZE) {
4686cd97ecSArd Biesheuvel unsigned int l = min(bytes, CHACHA_BLOCK_SIZE * 4U);
4786cd97ecSArd Biesheuvel
4886cd97ecSArd Biesheuvel chacha_4block_xor_neon(state, dst, src, nrounds, l);
4986cd97ecSArd Biesheuvel bytes -= l;
5086cd97ecSArd Biesheuvel src += l;
5186cd97ecSArd Biesheuvel dst += l;
5286cd97ecSArd Biesheuvel state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);
53b36d8c09SArd Biesheuvel }
54b36d8c09SArd Biesheuvel if (bytes) {
5586cd97ecSArd Biesheuvel const u8 *s = src;
5686cd97ecSArd Biesheuvel u8 *d = dst;
5786cd97ecSArd Biesheuvel
5886cd97ecSArd Biesheuvel if (bytes != CHACHA_BLOCK_SIZE)
5986cd97ecSArd Biesheuvel s = d = memcpy(buf, src, bytes);
6086cd97ecSArd Biesheuvel chacha_block_xor_neon(state, d, s, nrounds);
6186cd97ecSArd Biesheuvel if (d != dst)
62b36d8c09SArd Biesheuvel memcpy(dst, buf, bytes);
63*fd16931aSArd Biesheuvel state[12]++;
64b36d8c09SArd Biesheuvel }
65b36d8c09SArd Biesheuvel }
66b36d8c09SArd Biesheuvel
hchacha_block_arch(const u32 * state,u32 * stream,int nrounds)67a44a3430SArd Biesheuvel void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
68a44a3430SArd Biesheuvel {
69a44a3430SArd Biesheuvel if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
70a44a3430SArd Biesheuvel hchacha_block_arm(state, stream, nrounds);
71a44a3430SArd Biesheuvel } else {
72a44a3430SArd Biesheuvel kernel_neon_begin();
73a44a3430SArd Biesheuvel hchacha_block_neon(state, stream, nrounds);
74a44a3430SArd Biesheuvel kernel_neon_end();
75a44a3430SArd Biesheuvel }
76a44a3430SArd Biesheuvel }
77a44a3430SArd Biesheuvel EXPORT_SYMBOL(hchacha_block_arch);
78a44a3430SArd Biesheuvel
chacha_init_arch(u32 * state,const u32 * key,const u8 * iv)79a44a3430SArd Biesheuvel void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
80a44a3430SArd Biesheuvel {
81a44a3430SArd Biesheuvel chacha_init_generic(state, key, iv);
82a44a3430SArd Biesheuvel }
83a44a3430SArd Biesheuvel EXPORT_SYMBOL(chacha_init_arch);
84a44a3430SArd Biesheuvel
chacha_crypt_arch(u32 * state,u8 * dst,const u8 * src,unsigned int bytes,int nrounds)85a44a3430SArd Biesheuvel void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
86a44a3430SArd Biesheuvel int nrounds)
87a44a3430SArd Biesheuvel {
88a44a3430SArd Biesheuvel if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||
89a44a3430SArd Biesheuvel bytes <= CHACHA_BLOCK_SIZE) {
90a44a3430SArd Biesheuvel chacha_doarm(dst, src, bytes, state, nrounds);
91a44a3430SArd Biesheuvel state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
92a44a3430SArd Biesheuvel return;
93a44a3430SArd Biesheuvel }
94a44a3430SArd Biesheuvel
95706024a5SJason A. Donenfeld do {
96706024a5SJason A. Donenfeld unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
97706024a5SJason A. Donenfeld
98a44a3430SArd Biesheuvel kernel_neon_begin();
99706024a5SJason A. Donenfeld chacha_doneon(state, dst, src, todo, nrounds);
100a44a3430SArd Biesheuvel kernel_neon_end();
101706024a5SJason A. Donenfeld
102706024a5SJason A. Donenfeld bytes -= todo;
103706024a5SJason A. Donenfeld src += todo;
104706024a5SJason A. Donenfeld dst += todo;
105706024a5SJason A. Donenfeld } while (bytes);
106a44a3430SArd Biesheuvel }
107a44a3430SArd Biesheuvel EXPORT_SYMBOL(chacha_crypt_arch);
108a44a3430SArd Biesheuvel
chacha_stream_xor(struct skcipher_request * req,const struct chacha_ctx * ctx,const u8 * iv,bool neon)109b36d8c09SArd Biesheuvel static int chacha_stream_xor(struct skcipher_request *req,
110b36d8c09SArd Biesheuvel const struct chacha_ctx *ctx, const u8 *iv,
111b36d8c09SArd Biesheuvel bool neon)
112b36d8c09SArd Biesheuvel {
113b36d8c09SArd Biesheuvel struct skcipher_walk walk;
114b36d8c09SArd Biesheuvel u32 state[16];
115b36d8c09SArd Biesheuvel int err;
116b36d8c09SArd Biesheuvel
117b36d8c09SArd Biesheuvel err = skcipher_walk_virt(&walk, req, false);
118b36d8c09SArd Biesheuvel
119b36d8c09SArd Biesheuvel chacha_init_generic(state, ctx->key, iv);
120b36d8c09SArd Biesheuvel
121b36d8c09SArd Biesheuvel while (walk.nbytes > 0) {
122b36d8c09SArd Biesheuvel unsigned int nbytes = walk.nbytes;
123b36d8c09SArd Biesheuvel
124b36d8c09SArd Biesheuvel if (nbytes < walk.total)
125b36d8c09SArd Biesheuvel nbytes = round_down(nbytes, walk.stride);
126b36d8c09SArd Biesheuvel
1270bc81767SArd Biesheuvel if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
128b36d8c09SArd Biesheuvel chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr,
129b36d8c09SArd Biesheuvel nbytes, state, ctx->nrounds);
130b36d8c09SArd Biesheuvel state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE);
131b36d8c09SArd Biesheuvel } else {
132b36d8c09SArd Biesheuvel kernel_neon_begin();
133b36d8c09SArd Biesheuvel chacha_doneon(state, walk.dst.virt.addr,
134b36d8c09SArd Biesheuvel walk.src.virt.addr, nbytes, ctx->nrounds);
135b36d8c09SArd Biesheuvel kernel_neon_end();
136b36d8c09SArd Biesheuvel }
137b36d8c09SArd Biesheuvel err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
138b36d8c09SArd Biesheuvel }
139b36d8c09SArd Biesheuvel
140b36d8c09SArd Biesheuvel return err;
141b36d8c09SArd Biesheuvel }
142b36d8c09SArd Biesheuvel
do_chacha(struct skcipher_request * req,bool neon)143b36d8c09SArd Biesheuvel static int do_chacha(struct skcipher_request *req, bool neon)
144b36d8c09SArd Biesheuvel {
145b36d8c09SArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
146b36d8c09SArd Biesheuvel struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
147b36d8c09SArd Biesheuvel
148b36d8c09SArd Biesheuvel return chacha_stream_xor(req, ctx, req->iv, neon);
149b36d8c09SArd Biesheuvel }
150b36d8c09SArd Biesheuvel
chacha_arm(struct skcipher_request * req)151b36d8c09SArd Biesheuvel static int chacha_arm(struct skcipher_request *req)
152b36d8c09SArd Biesheuvel {
153b36d8c09SArd Biesheuvel return do_chacha(req, false);
154b36d8c09SArd Biesheuvel }
155b36d8c09SArd Biesheuvel
chacha_neon(struct skcipher_request * req)156b36d8c09SArd Biesheuvel static int chacha_neon(struct skcipher_request *req)
157b36d8c09SArd Biesheuvel {
158b36d8c09SArd Biesheuvel return do_chacha(req, neon_usable());
159b36d8c09SArd Biesheuvel }
160b36d8c09SArd Biesheuvel
do_xchacha(struct skcipher_request * req,bool neon)161b36d8c09SArd Biesheuvel static int do_xchacha(struct skcipher_request *req, bool neon)
162b36d8c09SArd Biesheuvel {
163b36d8c09SArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
164b36d8c09SArd Biesheuvel struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
165b36d8c09SArd Biesheuvel struct chacha_ctx subctx;
166b36d8c09SArd Biesheuvel u32 state[16];
167b36d8c09SArd Biesheuvel u8 real_iv[16];
168b36d8c09SArd Biesheuvel
169b36d8c09SArd Biesheuvel chacha_init_generic(state, ctx->key, req->iv);
170b36d8c09SArd Biesheuvel
1710bc81767SArd Biesheuvel if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
172b36d8c09SArd Biesheuvel hchacha_block_arm(state, subctx.key, ctx->nrounds);
173b36d8c09SArd Biesheuvel } else {
174b36d8c09SArd Biesheuvel kernel_neon_begin();
175b36d8c09SArd Biesheuvel hchacha_block_neon(state, subctx.key, ctx->nrounds);
176b36d8c09SArd Biesheuvel kernel_neon_end();
177b36d8c09SArd Biesheuvel }
178b36d8c09SArd Biesheuvel subctx.nrounds = ctx->nrounds;
179b36d8c09SArd Biesheuvel
180b36d8c09SArd Biesheuvel memcpy(&real_iv[0], req->iv + 24, 8);
181b36d8c09SArd Biesheuvel memcpy(&real_iv[8], req->iv + 16, 8);
182b36d8c09SArd Biesheuvel return chacha_stream_xor(req, &subctx, real_iv, neon);
183b36d8c09SArd Biesheuvel }
184b36d8c09SArd Biesheuvel
xchacha_arm(struct skcipher_request * req)185b36d8c09SArd Biesheuvel static int xchacha_arm(struct skcipher_request *req)
186b36d8c09SArd Biesheuvel {
187b36d8c09SArd Biesheuvel return do_xchacha(req, false);
188b36d8c09SArd Biesheuvel }
189b36d8c09SArd Biesheuvel
xchacha_neon(struct skcipher_request * req)190b36d8c09SArd Biesheuvel static int xchacha_neon(struct skcipher_request *req)
191b36d8c09SArd Biesheuvel {
192b36d8c09SArd Biesheuvel return do_xchacha(req, neon_usable());
193b36d8c09SArd Biesheuvel }
194b36d8c09SArd Biesheuvel
195b36d8c09SArd Biesheuvel static struct skcipher_alg arm_algs[] = {
196b36d8c09SArd Biesheuvel {
197b36d8c09SArd Biesheuvel .base.cra_name = "chacha20",
198b36d8c09SArd Biesheuvel .base.cra_driver_name = "chacha20-arm",
199b36d8c09SArd Biesheuvel .base.cra_priority = 200,
200b36d8c09SArd Biesheuvel .base.cra_blocksize = 1,
201b36d8c09SArd Biesheuvel .base.cra_ctxsize = sizeof(struct chacha_ctx),
202b36d8c09SArd Biesheuvel .base.cra_module = THIS_MODULE,
203b36d8c09SArd Biesheuvel
204b36d8c09SArd Biesheuvel .min_keysize = CHACHA_KEY_SIZE,
205b36d8c09SArd Biesheuvel .max_keysize = CHACHA_KEY_SIZE,
206b36d8c09SArd Biesheuvel .ivsize = CHACHA_IV_SIZE,
207b36d8c09SArd Biesheuvel .chunksize = CHACHA_BLOCK_SIZE,
208b36d8c09SArd Biesheuvel .setkey = chacha20_setkey,
209b36d8c09SArd Biesheuvel .encrypt = chacha_arm,
210b36d8c09SArd Biesheuvel .decrypt = chacha_arm,
211b36d8c09SArd Biesheuvel }, {
212b36d8c09SArd Biesheuvel .base.cra_name = "xchacha20",
213b36d8c09SArd Biesheuvel .base.cra_driver_name = "xchacha20-arm",
214b36d8c09SArd Biesheuvel .base.cra_priority = 200,
215b36d8c09SArd Biesheuvel .base.cra_blocksize = 1,
216b36d8c09SArd Biesheuvel .base.cra_ctxsize = sizeof(struct chacha_ctx),
217b36d8c09SArd Biesheuvel .base.cra_module = THIS_MODULE,
218b36d8c09SArd Biesheuvel
219b36d8c09SArd Biesheuvel .min_keysize = CHACHA_KEY_SIZE,
220b36d8c09SArd Biesheuvel .max_keysize = CHACHA_KEY_SIZE,
221b36d8c09SArd Biesheuvel .ivsize = XCHACHA_IV_SIZE,
222b36d8c09SArd Biesheuvel .chunksize = CHACHA_BLOCK_SIZE,
223b36d8c09SArd Biesheuvel .setkey = chacha20_setkey,
224b36d8c09SArd Biesheuvel .encrypt = xchacha_arm,
225b36d8c09SArd Biesheuvel .decrypt = xchacha_arm,
226b36d8c09SArd Biesheuvel }, {
227b36d8c09SArd Biesheuvel .base.cra_name = "xchacha12",
228b36d8c09SArd Biesheuvel .base.cra_driver_name = "xchacha12-arm",
229b36d8c09SArd Biesheuvel .base.cra_priority = 200,
230b36d8c09SArd Biesheuvel .base.cra_blocksize = 1,
231b36d8c09SArd Biesheuvel .base.cra_ctxsize = sizeof(struct chacha_ctx),
232b36d8c09SArd Biesheuvel .base.cra_module = THIS_MODULE,
233b36d8c09SArd Biesheuvel
234b36d8c09SArd Biesheuvel .min_keysize = CHACHA_KEY_SIZE,
235b36d8c09SArd Biesheuvel .max_keysize = CHACHA_KEY_SIZE,
236b36d8c09SArd Biesheuvel .ivsize = XCHACHA_IV_SIZE,
237b36d8c09SArd Biesheuvel .chunksize = CHACHA_BLOCK_SIZE,
238b36d8c09SArd Biesheuvel .setkey = chacha12_setkey,
239b36d8c09SArd Biesheuvel .encrypt = xchacha_arm,
240b36d8c09SArd Biesheuvel .decrypt = xchacha_arm,
241b36d8c09SArd Biesheuvel },
242b36d8c09SArd Biesheuvel };
243b36d8c09SArd Biesheuvel
244b36d8c09SArd Biesheuvel static struct skcipher_alg neon_algs[] = {
245b36d8c09SArd Biesheuvel {
246b36d8c09SArd Biesheuvel .base.cra_name = "chacha20",
247b36d8c09SArd Biesheuvel .base.cra_driver_name = "chacha20-neon",
248b36d8c09SArd Biesheuvel .base.cra_priority = 300,
249b36d8c09SArd Biesheuvel .base.cra_blocksize = 1,
250b36d8c09SArd Biesheuvel .base.cra_ctxsize = sizeof(struct chacha_ctx),
251b36d8c09SArd Biesheuvel .base.cra_module = THIS_MODULE,
252b36d8c09SArd Biesheuvel
253b36d8c09SArd Biesheuvel .min_keysize = CHACHA_KEY_SIZE,
254b36d8c09SArd Biesheuvel .max_keysize = CHACHA_KEY_SIZE,
255b36d8c09SArd Biesheuvel .ivsize = CHACHA_IV_SIZE,
256b36d8c09SArd Biesheuvel .chunksize = CHACHA_BLOCK_SIZE,
257b36d8c09SArd Biesheuvel .walksize = 4 * CHACHA_BLOCK_SIZE,
258b36d8c09SArd Biesheuvel .setkey = chacha20_setkey,
259b36d8c09SArd Biesheuvel .encrypt = chacha_neon,
260b36d8c09SArd Biesheuvel .decrypt = chacha_neon,
261b36d8c09SArd Biesheuvel }, {
262b36d8c09SArd Biesheuvel .base.cra_name = "xchacha20",
263b36d8c09SArd Biesheuvel .base.cra_driver_name = "xchacha20-neon",
264b36d8c09SArd Biesheuvel .base.cra_priority = 300,
265b36d8c09SArd Biesheuvel .base.cra_blocksize = 1,
266b36d8c09SArd Biesheuvel .base.cra_ctxsize = sizeof(struct chacha_ctx),
267b36d8c09SArd Biesheuvel .base.cra_module = THIS_MODULE,
268b36d8c09SArd Biesheuvel
269b36d8c09SArd Biesheuvel .min_keysize = CHACHA_KEY_SIZE,
270b36d8c09SArd Biesheuvel .max_keysize = CHACHA_KEY_SIZE,
271b36d8c09SArd Biesheuvel .ivsize = XCHACHA_IV_SIZE,
272b36d8c09SArd Biesheuvel .chunksize = CHACHA_BLOCK_SIZE,
273b36d8c09SArd Biesheuvel .walksize = 4 * CHACHA_BLOCK_SIZE,
274b36d8c09SArd Biesheuvel .setkey = chacha20_setkey,
275b36d8c09SArd Biesheuvel .encrypt = xchacha_neon,
276b36d8c09SArd Biesheuvel .decrypt = xchacha_neon,
277b36d8c09SArd Biesheuvel }, {
278b36d8c09SArd Biesheuvel .base.cra_name = "xchacha12",
279b36d8c09SArd Biesheuvel .base.cra_driver_name = "xchacha12-neon",
280b36d8c09SArd Biesheuvel .base.cra_priority = 300,
281b36d8c09SArd Biesheuvel .base.cra_blocksize = 1,
282b36d8c09SArd Biesheuvel .base.cra_ctxsize = sizeof(struct chacha_ctx),
283b36d8c09SArd Biesheuvel .base.cra_module = THIS_MODULE,
284b36d8c09SArd Biesheuvel
285b36d8c09SArd Biesheuvel .min_keysize = CHACHA_KEY_SIZE,
286b36d8c09SArd Biesheuvel .max_keysize = CHACHA_KEY_SIZE,
287b36d8c09SArd Biesheuvel .ivsize = XCHACHA_IV_SIZE,
288b36d8c09SArd Biesheuvel .chunksize = CHACHA_BLOCK_SIZE,
289b36d8c09SArd Biesheuvel .walksize = 4 * CHACHA_BLOCK_SIZE,
290b36d8c09SArd Biesheuvel .setkey = chacha12_setkey,
291b36d8c09SArd Biesheuvel .encrypt = xchacha_neon,
292b36d8c09SArd Biesheuvel .decrypt = xchacha_neon,
293b36d8c09SArd Biesheuvel }
294b36d8c09SArd Biesheuvel };
295b36d8c09SArd Biesheuvel
chacha_simd_mod_init(void)296b36d8c09SArd Biesheuvel static int __init chacha_simd_mod_init(void)
297b36d8c09SArd Biesheuvel {
2988394bfecSJason A. Donenfeld int err = 0;
299b36d8c09SArd Biesheuvel
3008394bfecSJason A. Donenfeld if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
301b36d8c09SArd Biesheuvel err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
302b36d8c09SArd Biesheuvel if (err)
303b36d8c09SArd Biesheuvel return err;
3048394bfecSJason A. Donenfeld }
305b36d8c09SArd Biesheuvel
306b36d8c09SArd Biesheuvel if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
307b36d8c09SArd Biesheuvel int i;
308b36d8c09SArd Biesheuvel
309b36d8c09SArd Biesheuvel switch (read_cpuid_part()) {
310b36d8c09SArd Biesheuvel case ARM_CPU_PART_CORTEX_A7:
311b36d8c09SArd Biesheuvel case ARM_CPU_PART_CORTEX_A5:
312b36d8c09SArd Biesheuvel /*
313b36d8c09SArd Biesheuvel * The Cortex-A7 and Cortex-A5 do not perform well with
314b36d8c09SArd Biesheuvel * the NEON implementation but do incredibly with the
315b36d8c09SArd Biesheuvel * scalar one and use less power.
316b36d8c09SArd Biesheuvel */
317b36d8c09SArd Biesheuvel for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
318b36d8c09SArd Biesheuvel neon_algs[i].base.cra_priority = 0;
319b36d8c09SArd Biesheuvel break;
320a44a3430SArd Biesheuvel default:
321a44a3430SArd Biesheuvel static_branch_enable(&use_neon);
322b36d8c09SArd Biesheuvel }
323b36d8c09SArd Biesheuvel
3248394bfecSJason A. Donenfeld if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
325b36d8c09SArd Biesheuvel err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
326b36d8c09SArd Biesheuvel if (err)
327b36d8c09SArd Biesheuvel crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
328b36d8c09SArd Biesheuvel }
3298394bfecSJason A. Donenfeld }
330b36d8c09SArd Biesheuvel return err;
331b36d8c09SArd Biesheuvel }
332b36d8c09SArd Biesheuvel
chacha_simd_mod_fini(void)333b36d8c09SArd Biesheuvel static void __exit chacha_simd_mod_fini(void)
334b36d8c09SArd Biesheuvel {
3358394bfecSJason A. Donenfeld if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
336b36d8c09SArd Biesheuvel crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
337b36d8c09SArd Biesheuvel if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON))
338b36d8c09SArd Biesheuvel crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
339b36d8c09SArd Biesheuvel }
3408394bfecSJason A. Donenfeld }
341b36d8c09SArd Biesheuvel
342b36d8c09SArd Biesheuvel module_init(chacha_simd_mod_init);
343b36d8c09SArd Biesheuvel module_exit(chacha_simd_mod_fini);
344b36d8c09SArd Biesheuvel
345b36d8c09SArd Biesheuvel MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)");
346b36d8c09SArd Biesheuvel MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
347b36d8c09SArd Biesheuvel MODULE_LICENSE("GPL v2");
348b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("chacha20");
349b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("chacha20-arm");
350b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("xchacha20");
351b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("xchacha20-arm");
352b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("xchacha12");
353b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("xchacha12-arm");
354b36d8c09SArd Biesheuvel #ifdef CONFIG_KERNEL_MODE_NEON
355b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("chacha20-neon");
356b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("xchacha20-neon");
357b36d8c09SArd Biesheuvel MODULE_ALIAS_CRYPTO("xchacha12-neon");
358b36d8c09SArd Biesheuvel #endif
359