1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011-2019 Pawel Jakub Dawidek <pawel@dawidek.net> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #ifdef _KERNEL 34 #include <sys/kernel.h> 35 #include <sys/malloc.h> 36 #include <sys/sysctl.h> 37 #include <sys/systm.h> 38 #endif /* _KERNEL */ 39 #include <sys/queue.h> 40 #include <sys/tree.h> 41 42 #include <geom/geom.h> 43 44 #include <geom/eli/g_eli.h> 45 46 #ifdef _KERNEL 47 MALLOC_DECLARE(M_ELI); 48 49 SYSCTL_DECL(_kern_geom_eli); 50 /* 51 * The default limit (8192 keys) will allow to cache all keys for 4TB 52 * provider with 512 bytes sectors and will take around 1MB of memory. 53 */ 54 static u_int g_eli_key_cache_limit = 8192; 55 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, key_cache_limit, CTLFLAG_RDTUN, 56 &g_eli_key_cache_limit, 0, "Maximum number of encryption keys to cache"); 57 static uint64_t g_eli_key_cache_hits; 58 SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_hits, CTLFLAG_RW, 59 &g_eli_key_cache_hits, 0, "Key cache hits"); 60 static uint64_t g_eli_key_cache_misses; 61 SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_misses, CTLFLAG_RW, 62 &g_eli_key_cache_misses, 0, "Key cache misses"); 63 64 static int 65 g_eli_key_cmp(const struct g_eli_key *a, const struct g_eli_key *b) 66 { 67 68 if (a->gek_keyno > b->gek_keyno) 69 return (1); 70 else if (a->gek_keyno < b->gek_keyno) 71 return (-1); 72 return (0); 73 } 74 #endif /* _KERNEL */ 75 76 void 77 g_eli_key_fill(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno) 78 { 79 const uint8_t *ekey; 80 struct { 81 char magic[4]; 82 uint8_t keyno[8]; 83 } __packed hmacdata; 84 85 if ((sc->sc_flags & G_ELI_FLAG_ENC_IVKEY) != 0) 86 ekey = sc->sc_mkey; 87 else 88 ekey = sc->sc_ekey; 89 90 bcopy("ekey", hmacdata.magic, 4); 91 le64enc(hmacdata.keyno, keyno); 92 g_eli_crypto_hmac(ekey, G_ELI_MAXKEYLEN, (uint8_t *)&hmacdata, 93 sizeof(hmacdata), key->gek_key, 0); 94 key->gek_keyno = keyno; 95 key->gek_count = 0; 96 key->gek_magic = G_ELI_KEY_MAGIC; 97 } 98 99 #ifdef _KERNEL 100 RB_PROTOTYPE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp); 101 RB_GENERATE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp); 102 103 static struct g_eli_key * 104 g_eli_key_allocate(struct g_eli_softc *sc, uint64_t keyno) 105 { 106 struct g_eli_key *key, *ekey, keysearch; 107 108 mtx_assert(&sc->sc_ekeys_lock, MA_OWNED); 109 mtx_unlock(&sc->sc_ekeys_lock); 110 111 key = malloc(sizeof(*key), M_ELI, M_WAITOK); 112 g_eli_key_fill(sc, key, keyno); 113 114 mtx_lock(&sc->sc_ekeys_lock); 115 /* 116 * Recheck if the key wasn't added while we weren't holding the lock. 117 */ 118 keysearch.gek_keyno = keyno; 119 ekey = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch); 120 if (ekey != NULL) { 121 explicit_bzero(key, sizeof(*key)); 122 free(key, M_ELI); 123 key = ekey; 124 TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next); 125 } else { 126 RB_INSERT(g_eli_key_tree, &sc->sc_ekeys_tree, key); 127 sc->sc_ekeys_allocated++; 128 } 129 TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next); 130 131 return (key); 132 } 133 134 static struct g_eli_key * 135 g_eli_key_find_last(struct g_eli_softc *sc) 136 { 137 struct g_eli_key *key; 138 139 mtx_assert(&sc->sc_ekeys_lock, MA_OWNED); 140 141 TAILQ_FOREACH(key, &sc->sc_ekeys_queue, gek_next) { 142 if (key->gek_count == 0) 143 break; 144 } 145 146 return (key); 147 } 148 149 static void 150 g_eli_key_replace(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno) 151 { 152 153 mtx_assert(&sc->sc_ekeys_lock, MA_OWNED); 154 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid magic.")); 155 156 RB_REMOVE(g_eli_key_tree, &sc->sc_ekeys_tree, key); 157 TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next); 158 159 KASSERT(key->gek_count == 0, ("gek_count=%d", key->gek_count)); 160 161 g_eli_key_fill(sc, key, keyno); 162 163 RB_INSERT(g_eli_key_tree, &sc->sc_ekeys_tree, key); 164 TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next); 165 } 166 167 static void 168 g_eli_key_remove(struct g_eli_softc *sc, struct g_eli_key *key) 169 { 170 171 mtx_assert(&sc->sc_ekeys_lock, MA_OWNED); 172 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid magic.")); 173 KASSERT(key->gek_count == 0, ("gek_count=%d", key->gek_count)); 174 175 RB_REMOVE(g_eli_key_tree, &sc->sc_ekeys_tree, key); 176 TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next); 177 sc->sc_ekeys_allocated--; 178 explicit_bzero(key, sizeof(*key)); 179 free(key, M_ELI); 180 } 181 182 void 183 g_eli_key_init(struct g_eli_softc *sc) 184 { 185 uint8_t *mkey; 186 187 mtx_lock(&sc->sc_ekeys_lock); 188 189 mkey = sc->sc_mkey + sizeof(sc->sc_ivkey); 190 if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0) 191 bcopy(mkey, sc->sc_ekey, G_ELI_DATAKEYLEN); 192 else { 193 /* 194 * The encryption key is: ekey = HMAC_SHA512(Data-Key, 0x10) 195 */ 196 g_eli_crypto_hmac(mkey, G_ELI_MAXKEYLEN, "\x10", 1, 197 sc->sc_ekey, 0); 198 } 199 200 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) { 201 sc->sc_ekeys_total = 1; 202 sc->sc_ekeys_allocated = 0; 203 } else { 204 off_t mediasize; 205 size_t blocksize; 206 207 if ((sc->sc_flags & G_ELI_FLAG_AUTH) != 0) { 208 struct g_provider *pp; 209 210 pp = LIST_FIRST(&sc->sc_geom->consumer)->provider; 211 mediasize = pp->mediasize; 212 blocksize = pp->sectorsize; 213 } else { 214 mediasize = sc->sc_mediasize; 215 blocksize = sc->sc_sectorsize; 216 } 217 sc->sc_ekeys_total = 218 ((mediasize - 1) >> G_ELI_KEY_SHIFT) / blocksize + 1; 219 sc->sc_ekeys_allocated = 0; 220 TAILQ_INIT(&sc->sc_ekeys_queue); 221 RB_INIT(&sc->sc_ekeys_tree); 222 if (sc->sc_ekeys_total <= g_eli_key_cache_limit) { 223 uint64_t keyno; 224 225 for (keyno = 0; keyno < sc->sc_ekeys_total; keyno++) 226 (void)g_eli_key_allocate(sc, keyno); 227 KASSERT(sc->sc_ekeys_total == sc->sc_ekeys_allocated, 228 ("sc_ekeys_total=%ju != sc_ekeys_allocated=%ju", 229 (uintmax_t)sc->sc_ekeys_total, 230 (uintmax_t)sc->sc_ekeys_allocated)); 231 } 232 } 233 234 mtx_unlock(&sc->sc_ekeys_lock); 235 } 236 237 void 238 g_eli_key_destroy(struct g_eli_softc *sc) 239 { 240 241 mtx_lock(&sc->sc_ekeys_lock); 242 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) { 243 explicit_bzero(sc->sc_ekey, sizeof(sc->sc_ekey)); 244 } else { 245 struct g_eli_key *key; 246 247 while ((key = TAILQ_FIRST(&sc->sc_ekeys_queue)) != NULL) 248 g_eli_key_remove(sc, key); 249 TAILQ_INIT(&sc->sc_ekeys_queue); 250 RB_INIT(&sc->sc_ekeys_tree); 251 } 252 mtx_unlock(&sc->sc_ekeys_lock); 253 } 254 255 void 256 g_eli_key_resize(struct g_eli_softc *sc) 257 { 258 uint64_t new_ekeys_total; 259 off_t mediasize; 260 size_t blocksize; 261 262 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) { 263 return; 264 } 265 266 mtx_lock(&sc->sc_ekeys_lock); 267 268 if ((sc->sc_flags & G_ELI_FLAG_AUTH) != 0) { 269 struct g_provider *pp; 270 271 pp = LIST_FIRST(&sc->sc_geom->consumer)->provider; 272 mediasize = pp->mediasize; 273 blocksize = pp->sectorsize; 274 } else { 275 mediasize = sc->sc_mediasize; 276 blocksize = sc->sc_sectorsize; 277 } 278 new_ekeys_total = ((mediasize - 1) >> G_ELI_KEY_SHIFT) / blocksize + 1; 279 /* We only allow to grow. */ 280 KASSERT(new_ekeys_total >= sc->sc_ekeys_total, 281 ("new_ekeys_total=%ju < sc_ekeys_total=%ju", 282 (uintmax_t)new_ekeys_total, (uintmax_t)sc->sc_ekeys_total)); 283 if (new_ekeys_total <= g_eli_key_cache_limit) { 284 uint64_t keyno; 285 286 for (keyno = sc->sc_ekeys_total; keyno < new_ekeys_total; 287 keyno++) { 288 (void)g_eli_key_allocate(sc, keyno); 289 } 290 KASSERT(new_ekeys_total == sc->sc_ekeys_allocated, 291 ("new_ekeys_total=%ju != sc_ekeys_allocated=%ju", 292 (uintmax_t)new_ekeys_total, 293 (uintmax_t)sc->sc_ekeys_allocated)); 294 } 295 296 sc->sc_ekeys_total = new_ekeys_total; 297 298 mtx_unlock(&sc->sc_ekeys_lock); 299 } 300 301 /* 302 * Select encryption key. If G_ELI_FLAG_SINGLE_KEY is present we only have one 303 * key available for all the data. If the flag is not present select the key 304 * based on data offset. 305 */ 306 uint8_t * 307 g_eli_key_hold(struct g_eli_softc *sc, off_t offset, size_t blocksize) 308 { 309 struct g_eli_key *key, keysearch; 310 uint64_t keyno; 311 312 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) 313 return (sc->sc_ekey); 314 315 /* We switch key every 2^G_ELI_KEY_SHIFT blocks. */ 316 keyno = (offset >> G_ELI_KEY_SHIFT) / blocksize; 317 318 KASSERT(keyno < sc->sc_ekeys_total, 319 ("%s: keyno=%ju >= sc_ekeys_total=%ju", 320 __func__, (uintmax_t)keyno, (uintmax_t)sc->sc_ekeys_total)); 321 322 keysearch.gek_keyno = keyno; 323 324 if (sc->sc_ekeys_total == sc->sc_ekeys_allocated) { 325 /* We have all the keys, so avoid some overhead. */ 326 key = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch); 327 KASSERT(key != NULL, ("No key %ju found.", (uintmax_t)keyno)); 328 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, 329 ("Invalid key magic.")); 330 return (key->gek_key); 331 } 332 333 mtx_lock(&sc->sc_ekeys_lock); 334 key = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch); 335 if (key != NULL) { 336 g_eli_key_cache_hits++; 337 TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next); 338 TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next); 339 } else { 340 /* 341 * No key in cache, find the least recently unreferenced key 342 * or allocate one if we haven't reached our limit yet. 343 */ 344 if (sc->sc_ekeys_allocated < g_eli_key_cache_limit) { 345 key = g_eli_key_allocate(sc, keyno); 346 } else { 347 g_eli_key_cache_misses++; 348 key = g_eli_key_find_last(sc); 349 if (key != NULL) { 350 g_eli_key_replace(sc, key, keyno); 351 } else { 352 /* All keys are referenced? Allocate one. */ 353 key = g_eli_key_allocate(sc, keyno); 354 } 355 } 356 } 357 key->gek_count++; 358 mtx_unlock(&sc->sc_ekeys_lock); 359 360 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid key magic.")); 361 362 return (key->gek_key); 363 } 364 365 void 366 g_eli_key_drop(struct g_eli_softc *sc, uint8_t *rawkey) 367 { 368 struct g_eli_key *key = (struct g_eli_key *)rawkey; 369 370 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) 371 return; 372 373 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid key magic.")); 374 375 if (sc->sc_ekeys_total == sc->sc_ekeys_allocated) 376 return; 377 378 mtx_lock(&sc->sc_ekeys_lock); 379 KASSERT(key->gek_count > 0, ("key->gek_count=%d", key->gek_count)); 380 key->gek_count--; 381 while (sc->sc_ekeys_allocated > g_eli_key_cache_limit) { 382 key = g_eli_key_find_last(sc); 383 if (key == NULL) 384 break; 385 g_eli_key_remove(sc, key); 386 } 387 mtx_unlock(&sc->sc_ekeys_lock); 388 } 389 #endif /* _KERNEL */ 390