1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * GHASH using the RISC-V vector crypto extensions
4 *
5 * Copyright (C) 2023 VRULL GmbH
6 * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
7 *
8 * Copyright (C) 2023 SiFive, Inc.
9 * Author: Jerry Shih <jerry.shih@sifive.com>
10 */
11
12 #include <asm/simd.h>
13 #include <asm/vector.h>
14 #include <crypto/ghash.h>
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/simd.h>
17 #include <linux/linkage.h>
18 #include <linux/module.h>
19
20 asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
21 size_t len);
22
23 struct riscv64_ghash_tfm_ctx {
24 be128 key;
25 };
26
27 struct riscv64_ghash_desc_ctx {
28 be128 accumulator;
29 u8 buffer[GHASH_BLOCK_SIZE];
30 u32 bytes;
31 };
32
riscv64_ghash_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int keylen)33 static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
34 unsigned int keylen)
35 {
36 struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
37
38 if (keylen != GHASH_BLOCK_SIZE)
39 return -EINVAL;
40
41 memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
42
43 return 0;
44 }
45
riscv64_ghash_init(struct shash_desc * desc)46 static int riscv64_ghash_init(struct shash_desc *desc)
47 {
48 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
49
50 *dctx = (struct riscv64_ghash_desc_ctx){};
51
52 return 0;
53 }
54
55 static inline void
riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx * tctx,struct riscv64_ghash_desc_ctx * dctx,const u8 * src,size_t srclen)56 riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
57 struct riscv64_ghash_desc_ctx *dctx,
58 const u8 *src, size_t srclen)
59 {
60 /* The srclen is nonzero and a multiple of 16. */
61 if (crypto_simd_usable()) {
62 kernel_vector_begin();
63 ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
64 kernel_vector_end();
65 } else {
66 do {
67 crypto_xor((u8 *)&dctx->accumulator, src,
68 GHASH_BLOCK_SIZE);
69 gf128mul_lle(&dctx->accumulator, &tctx->key);
70 src += GHASH_BLOCK_SIZE;
71 srclen -= GHASH_BLOCK_SIZE;
72 } while (srclen);
73 }
74 }
75
riscv64_ghash_update(struct shash_desc * desc,const u8 * src,unsigned int srclen)76 static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
77 unsigned int srclen)
78 {
79 const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
80 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
81 unsigned int len;
82
83 if (dctx->bytes) {
84 if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) {
85 memcpy(dctx->buffer + dctx->bytes, src, srclen);
86 dctx->bytes += srclen;
87 return 0;
88 }
89 memcpy(dctx->buffer + dctx->bytes, src,
90 GHASH_BLOCK_SIZE - dctx->bytes);
91 riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
92 GHASH_BLOCK_SIZE);
93 src += GHASH_BLOCK_SIZE - dctx->bytes;
94 srclen -= GHASH_BLOCK_SIZE - dctx->bytes;
95 dctx->bytes = 0;
96 }
97
98 len = round_down(srclen, GHASH_BLOCK_SIZE);
99 if (len) {
100 riscv64_ghash_blocks(tctx, dctx, src, len);
101 src += len;
102 srclen -= len;
103 }
104
105 if (srclen) {
106 memcpy(dctx->buffer, src, srclen);
107 dctx->bytes = srclen;
108 }
109
110 return 0;
111 }
112
riscv64_ghash_final(struct shash_desc * desc,u8 * out)113 static int riscv64_ghash_final(struct shash_desc *desc, u8 *out)
114 {
115 const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
116 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
117 int i;
118
119 if (dctx->bytes) {
120 for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++)
121 dctx->buffer[i] = 0;
122
123 riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
124 GHASH_BLOCK_SIZE);
125 }
126
127 memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
128 return 0;
129 }
130
131 static struct shash_alg riscv64_ghash_alg = {
132 .init = riscv64_ghash_init,
133 .update = riscv64_ghash_update,
134 .final = riscv64_ghash_final,
135 .setkey = riscv64_ghash_setkey,
136 .descsize = sizeof(struct riscv64_ghash_desc_ctx),
137 .digestsize = GHASH_DIGEST_SIZE,
138 .base = {
139 .cra_blocksize = GHASH_BLOCK_SIZE,
140 .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
141 .cra_priority = 300,
142 .cra_name = "ghash",
143 .cra_driver_name = "ghash-riscv64-zvkg",
144 .cra_module = THIS_MODULE,
145 },
146 };
147
riscv64_ghash_mod_init(void)148 static int __init riscv64_ghash_mod_init(void)
149 {
150 if (riscv_isa_extension_available(NULL, ZVKG) &&
151 riscv_vector_vlen() >= 128)
152 return crypto_register_shash(&riscv64_ghash_alg);
153
154 return -ENODEV;
155 }
156
riscv64_ghash_mod_exit(void)157 static void __exit riscv64_ghash_mod_exit(void)
158 {
159 crypto_unregister_shash(&riscv64_ghash_alg);
160 }
161
162 module_init(riscv64_ghash_mod_init);
163 module_exit(riscv64_ghash_mod_exit);
164
165 MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
166 MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
167 MODULE_LICENSE("GPL");
168 MODULE_ALIAS_CRYPTO("ghash");
169