1*9d2c0b48SNathan Huckleberry // SPDX-License-Identifier: GPL-2.0-only
2*9d2c0b48SNathan Huckleberry /*
3*9d2c0b48SNathan Huckleberry * Glue code for POLYVAL using ARMv8 Crypto Extensions
4*9d2c0b48SNathan Huckleberry *
5*9d2c0b48SNathan Huckleberry * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
6*9d2c0b48SNathan Huckleberry * Copyright (c) 2009 Intel Corp.
7*9d2c0b48SNathan Huckleberry * Author: Huang Ying <ying.huang@intel.com>
8*9d2c0b48SNathan Huckleberry * Copyright 2021 Google LLC
9*9d2c0b48SNathan Huckleberry */
10*9d2c0b48SNathan Huckleberry
11*9d2c0b48SNathan Huckleberry /*
12*9d2c0b48SNathan Huckleberry * Glue code based on ghash-clmulni-intel_glue.c.
13*9d2c0b48SNathan Huckleberry *
14*9d2c0b48SNathan Huckleberry * This implementation of POLYVAL uses montgomery multiplication accelerated by
15*9d2c0b48SNathan Huckleberry * ARMv8 Crypto Extensions instructions to implement the finite field operations.
16*9d2c0b48SNathan Huckleberry */
17*9d2c0b48SNathan Huckleberry
18*9d2c0b48SNathan Huckleberry #include <crypto/algapi.h>
19*9d2c0b48SNathan Huckleberry #include <crypto/internal/hash.h>
20*9d2c0b48SNathan Huckleberry #include <crypto/internal/simd.h>
21*9d2c0b48SNathan Huckleberry #include <crypto/polyval.h>
22*9d2c0b48SNathan Huckleberry #include <linux/crypto.h>
23*9d2c0b48SNathan Huckleberry #include <linux/init.h>
24*9d2c0b48SNathan Huckleberry #include <linux/kernel.h>
25*9d2c0b48SNathan Huckleberry #include <linux/module.h>
26*9d2c0b48SNathan Huckleberry #include <linux/cpufeature.h>
27*9d2c0b48SNathan Huckleberry #include <asm/neon.h>
28*9d2c0b48SNathan Huckleberry #include <asm/simd.h>
29*9d2c0b48SNathan Huckleberry
30*9d2c0b48SNathan Huckleberry #define NUM_KEY_POWERS 8
31*9d2c0b48SNathan Huckleberry
32*9d2c0b48SNathan Huckleberry struct polyval_tfm_ctx {
33*9d2c0b48SNathan Huckleberry /*
34*9d2c0b48SNathan Huckleberry * These powers must be in the order h^8, ..., h^1.
35*9d2c0b48SNathan Huckleberry */
36*9d2c0b48SNathan Huckleberry u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE];
37*9d2c0b48SNathan Huckleberry };
38*9d2c0b48SNathan Huckleberry
39*9d2c0b48SNathan Huckleberry struct polyval_desc_ctx {
40*9d2c0b48SNathan Huckleberry u8 buffer[POLYVAL_BLOCK_SIZE];
41*9d2c0b48SNathan Huckleberry u32 bytes;
42*9d2c0b48SNathan Huckleberry };
43*9d2c0b48SNathan Huckleberry
44*9d2c0b48SNathan Huckleberry asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys,
45*9d2c0b48SNathan Huckleberry const u8 *in, size_t nblocks, u8 *accumulator);
46*9d2c0b48SNathan Huckleberry asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2);
47*9d2c0b48SNathan Huckleberry
internal_polyval_update(const struct polyval_tfm_ctx * keys,const u8 * in,size_t nblocks,u8 * accumulator)48*9d2c0b48SNathan Huckleberry static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
49*9d2c0b48SNathan Huckleberry const u8 *in, size_t nblocks, u8 *accumulator)
50*9d2c0b48SNathan Huckleberry {
51*9d2c0b48SNathan Huckleberry if (likely(crypto_simd_usable())) {
52*9d2c0b48SNathan Huckleberry kernel_neon_begin();
53*9d2c0b48SNathan Huckleberry pmull_polyval_update(keys, in, nblocks, accumulator);
54*9d2c0b48SNathan Huckleberry kernel_neon_end();
55*9d2c0b48SNathan Huckleberry } else {
56*9d2c0b48SNathan Huckleberry polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
57*9d2c0b48SNathan Huckleberry nblocks, accumulator);
58*9d2c0b48SNathan Huckleberry }
59*9d2c0b48SNathan Huckleberry }
60*9d2c0b48SNathan Huckleberry
internal_polyval_mul(u8 * op1,const u8 * op2)61*9d2c0b48SNathan Huckleberry static void internal_polyval_mul(u8 *op1, const u8 *op2)
62*9d2c0b48SNathan Huckleberry {
63*9d2c0b48SNathan Huckleberry if (likely(crypto_simd_usable())) {
64*9d2c0b48SNathan Huckleberry kernel_neon_begin();
65*9d2c0b48SNathan Huckleberry pmull_polyval_mul(op1, op2);
66*9d2c0b48SNathan Huckleberry kernel_neon_end();
67*9d2c0b48SNathan Huckleberry } else {
68*9d2c0b48SNathan Huckleberry polyval_mul_non4k(op1, op2);
69*9d2c0b48SNathan Huckleberry }
70*9d2c0b48SNathan Huckleberry }
71*9d2c0b48SNathan Huckleberry
polyval_arm64_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int keylen)72*9d2c0b48SNathan Huckleberry static int polyval_arm64_setkey(struct crypto_shash *tfm,
73*9d2c0b48SNathan Huckleberry const u8 *key, unsigned int keylen)
74*9d2c0b48SNathan Huckleberry {
75*9d2c0b48SNathan Huckleberry struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm);
76*9d2c0b48SNathan Huckleberry int i;
77*9d2c0b48SNathan Huckleberry
78*9d2c0b48SNathan Huckleberry if (keylen != POLYVAL_BLOCK_SIZE)
79*9d2c0b48SNathan Huckleberry return -EINVAL;
80*9d2c0b48SNathan Huckleberry
81*9d2c0b48SNathan Huckleberry memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
82*9d2c0b48SNathan Huckleberry
83*9d2c0b48SNathan Huckleberry for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
84*9d2c0b48SNathan Huckleberry memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
85*9d2c0b48SNathan Huckleberry internal_polyval_mul(tctx->key_powers[i],
86*9d2c0b48SNathan Huckleberry tctx->key_powers[i+1]);
87*9d2c0b48SNathan Huckleberry }
88*9d2c0b48SNathan Huckleberry
89*9d2c0b48SNathan Huckleberry return 0;
90*9d2c0b48SNathan Huckleberry }
91*9d2c0b48SNathan Huckleberry
polyval_arm64_init(struct shash_desc * desc)92*9d2c0b48SNathan Huckleberry static int polyval_arm64_init(struct shash_desc *desc)
93*9d2c0b48SNathan Huckleberry {
94*9d2c0b48SNathan Huckleberry struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
95*9d2c0b48SNathan Huckleberry
96*9d2c0b48SNathan Huckleberry memset(dctx, 0, sizeof(*dctx));
97*9d2c0b48SNathan Huckleberry
98*9d2c0b48SNathan Huckleberry return 0;
99*9d2c0b48SNathan Huckleberry }
100*9d2c0b48SNathan Huckleberry
polyval_arm64_update(struct shash_desc * desc,const u8 * src,unsigned int srclen)101*9d2c0b48SNathan Huckleberry static int polyval_arm64_update(struct shash_desc *desc,
102*9d2c0b48SNathan Huckleberry const u8 *src, unsigned int srclen)
103*9d2c0b48SNathan Huckleberry {
104*9d2c0b48SNathan Huckleberry struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
105*9d2c0b48SNathan Huckleberry const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
106*9d2c0b48SNathan Huckleberry u8 *pos;
107*9d2c0b48SNathan Huckleberry unsigned int nblocks;
108*9d2c0b48SNathan Huckleberry unsigned int n;
109*9d2c0b48SNathan Huckleberry
110*9d2c0b48SNathan Huckleberry if (dctx->bytes) {
111*9d2c0b48SNathan Huckleberry n = min(srclen, dctx->bytes);
112*9d2c0b48SNathan Huckleberry pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
113*9d2c0b48SNathan Huckleberry
114*9d2c0b48SNathan Huckleberry dctx->bytes -= n;
115*9d2c0b48SNathan Huckleberry srclen -= n;
116*9d2c0b48SNathan Huckleberry
117*9d2c0b48SNathan Huckleberry while (n--)
118*9d2c0b48SNathan Huckleberry *pos++ ^= *src++;
119*9d2c0b48SNathan Huckleberry
120*9d2c0b48SNathan Huckleberry if (!dctx->bytes)
121*9d2c0b48SNathan Huckleberry internal_polyval_mul(dctx->buffer,
122*9d2c0b48SNathan Huckleberry tctx->key_powers[NUM_KEY_POWERS-1]);
123*9d2c0b48SNathan Huckleberry }
124*9d2c0b48SNathan Huckleberry
125*9d2c0b48SNathan Huckleberry while (srclen >= POLYVAL_BLOCK_SIZE) {
126*9d2c0b48SNathan Huckleberry /* allow rescheduling every 4K bytes */
127*9d2c0b48SNathan Huckleberry nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
128*9d2c0b48SNathan Huckleberry internal_polyval_update(tctx, src, nblocks, dctx->buffer);
129*9d2c0b48SNathan Huckleberry srclen -= nblocks * POLYVAL_BLOCK_SIZE;
130*9d2c0b48SNathan Huckleberry src += nblocks * POLYVAL_BLOCK_SIZE;
131*9d2c0b48SNathan Huckleberry }
132*9d2c0b48SNathan Huckleberry
133*9d2c0b48SNathan Huckleberry if (srclen) {
134*9d2c0b48SNathan Huckleberry dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
135*9d2c0b48SNathan Huckleberry pos = dctx->buffer;
136*9d2c0b48SNathan Huckleberry while (srclen--)
137*9d2c0b48SNathan Huckleberry *pos++ ^= *src++;
138*9d2c0b48SNathan Huckleberry }
139*9d2c0b48SNathan Huckleberry
140*9d2c0b48SNathan Huckleberry return 0;
141*9d2c0b48SNathan Huckleberry }
142*9d2c0b48SNathan Huckleberry
polyval_arm64_final(struct shash_desc * desc,u8 * dst)143*9d2c0b48SNathan Huckleberry static int polyval_arm64_final(struct shash_desc *desc, u8 *dst)
144*9d2c0b48SNathan Huckleberry {
145*9d2c0b48SNathan Huckleberry struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
146*9d2c0b48SNathan Huckleberry const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
147*9d2c0b48SNathan Huckleberry
148*9d2c0b48SNathan Huckleberry if (dctx->bytes) {
149*9d2c0b48SNathan Huckleberry internal_polyval_mul(dctx->buffer,
150*9d2c0b48SNathan Huckleberry tctx->key_powers[NUM_KEY_POWERS-1]);
151*9d2c0b48SNathan Huckleberry }
152*9d2c0b48SNathan Huckleberry
153*9d2c0b48SNathan Huckleberry memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
154*9d2c0b48SNathan Huckleberry
155*9d2c0b48SNathan Huckleberry return 0;
156*9d2c0b48SNathan Huckleberry }
157*9d2c0b48SNathan Huckleberry
158*9d2c0b48SNathan Huckleberry static struct shash_alg polyval_alg = {
159*9d2c0b48SNathan Huckleberry .digestsize = POLYVAL_DIGEST_SIZE,
160*9d2c0b48SNathan Huckleberry .init = polyval_arm64_init,
161*9d2c0b48SNathan Huckleberry .update = polyval_arm64_update,
162*9d2c0b48SNathan Huckleberry .final = polyval_arm64_final,
163*9d2c0b48SNathan Huckleberry .setkey = polyval_arm64_setkey,
164*9d2c0b48SNathan Huckleberry .descsize = sizeof(struct polyval_desc_ctx),
165*9d2c0b48SNathan Huckleberry .base = {
166*9d2c0b48SNathan Huckleberry .cra_name = "polyval",
167*9d2c0b48SNathan Huckleberry .cra_driver_name = "polyval-ce",
168*9d2c0b48SNathan Huckleberry .cra_priority = 200,
169*9d2c0b48SNathan Huckleberry .cra_blocksize = POLYVAL_BLOCK_SIZE,
170*9d2c0b48SNathan Huckleberry .cra_ctxsize = sizeof(struct polyval_tfm_ctx),
171*9d2c0b48SNathan Huckleberry .cra_module = THIS_MODULE,
172*9d2c0b48SNathan Huckleberry },
173*9d2c0b48SNathan Huckleberry };
174*9d2c0b48SNathan Huckleberry
polyval_ce_mod_init(void)175*9d2c0b48SNathan Huckleberry static int __init polyval_ce_mod_init(void)
176*9d2c0b48SNathan Huckleberry {
177*9d2c0b48SNathan Huckleberry return crypto_register_shash(&polyval_alg);
178*9d2c0b48SNathan Huckleberry }
179*9d2c0b48SNathan Huckleberry
polyval_ce_mod_exit(void)180*9d2c0b48SNathan Huckleberry static void __exit polyval_ce_mod_exit(void)
181*9d2c0b48SNathan Huckleberry {
182*9d2c0b48SNathan Huckleberry crypto_unregister_shash(&polyval_alg);
183*9d2c0b48SNathan Huckleberry }
184*9d2c0b48SNathan Huckleberry
185*9d2c0b48SNathan Huckleberry module_cpu_feature_match(PMULL, polyval_ce_mod_init)
186*9d2c0b48SNathan Huckleberry module_exit(polyval_ce_mod_exit);
187*9d2c0b48SNathan Huckleberry
188*9d2c0b48SNathan Huckleberry MODULE_LICENSE("GPL");
189*9d2c0b48SNathan Huckleberry MODULE_DESCRIPTION("POLYVAL hash function accelerated by ARMv8 Crypto Extensions");
190*9d2c0b48SNathan Huckleberry MODULE_ALIAS_CRYPTO("polyval");
191*9d2c0b48SNathan Huckleberry MODULE_ALIAS_CRYPTO("polyval-ce");
192