1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/endian.h> 36 #include <sys/kernel.h> 37 #include <sys/ktls.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <opencrypto/cryptodev.h> 45 46 struct ocf_session { 47 crypto_session_t sid; 48 struct mtx lock; 49 }; 50 51 struct ocf_operation { 52 struct ocf_session *os; 53 bool done; 54 struct iovec iov[0]; 55 }; 56 57 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 58 59 SYSCTL_DECL(_kern_ipc_tls); 60 SYSCTL_DECL(_kern_ipc_tls_stats); 61 62 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, 63 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 64 "Kernel TLS offload via OCF stats"); 65 66 static counter_u64_t ocf_tls12_gcm_crypts; 67 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, 68 CTLFLAG_RD, &ocf_tls12_gcm_crypts, 69 "Total number of OCF TLS 1.2 GCM encryption operations"); 70 71 static counter_u64_t ocf_tls13_gcm_crypts; 72 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, 73 CTLFLAG_RD, &ocf_tls13_gcm_crypts, 74 "Total number of OCF TLS 1.3 GCM encryption operations"); 75 76 static counter_u64_t ocf_inplace; 77 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace, 78 CTLFLAG_RD, &ocf_inplace, 79 "Total number of OCF in-place operations"); 80 81 static counter_u64_t ocf_separate_output; 82 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output, 83 CTLFLAG_RD, &ocf_separate_output, 84 "Total number of OCF operations with a separate output buffer"); 85 86 static counter_u64_t ocf_retries; 87 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 88 &ocf_retries, 89 "Number of OCF encryption operation retries"); 90 91 static int 92 ktls_ocf_callback(struct cryptop *crp) 93 { 94 struct ocf_operation *oo; 95 96 oo = crp->crp_opaque; 97 mtx_lock(&oo->os->lock); 98 oo->done = true; 99 mtx_unlock(&oo->os->lock); 100 wakeup(oo); 101 return (0); 102 } 103 104 static int 105 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, 106 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 107 struct iovec *outiov, int iovcnt, uint64_t seqno, 108 uint8_t record_type __unused) 109 { 110 struct uio uio, out_uio, *tag_uio; 111 struct tls_aead_data ad; 112 struct cryptop *crp; 113 struct ocf_session *os; 114 struct ocf_operation *oo; 115 int i, error; 116 uint16_t tls_comp_len; 117 bool inplace; 118 119 os = tls->cipher; 120 121 oo = malloc(sizeof(*oo) + (iovcnt + 1) * sizeof(struct iovec), 122 M_KTLS_OCF, M_WAITOK | M_ZERO); 123 oo->os = os; 124 125 uio.uio_iov = iniov; 126 uio.uio_iovcnt = iovcnt; 127 uio.uio_offset = 0; 128 uio.uio_segflg = UIO_SYSSPACE; 129 uio.uio_td = curthread; 130 131 out_uio.uio_iov = outiov; 132 out_uio.uio_iovcnt = iovcnt; 133 out_uio.uio_offset = 0; 134 out_uio.uio_segflg = UIO_SYSSPACE; 135 out_uio.uio_td = curthread; 136 137 crp = crypto_getreq(os->sid, M_WAITOK); 138 139 /* Setup the IV. */ 140 memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 141 memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); 142 143 /* Setup the AAD. */ 144 tls_comp_len = ntohs(hdr->tls_length) - 145 (AES_GMAC_HASH_LEN + sizeof(uint64_t)); 146 ad.seq = htobe64(seqno); 147 ad.type = hdr->tls_type; 148 ad.tls_vmajor = hdr->tls_vmajor; 149 ad.tls_vminor = hdr->tls_vminor; 150 ad.tls_length = htons(tls_comp_len); 151 crp->crp_aad = &ad; 152 crp->crp_aad_length = sizeof(ad); 153 154 /* Compute payload length and determine if encryption is in place. */ 155 inplace = true; 156 crp->crp_payload_start = 0; 157 for (i = 0; i < iovcnt; i++) { 158 if (iniov[i].iov_base != outiov[i].iov_base) 159 inplace = false; 160 crp->crp_payload_length += iniov[i].iov_len; 161 } 162 uio.uio_resid = crp->crp_payload_length; 163 out_uio.uio_resid = crp->crp_payload_length; 164 165 if (inplace) 166 tag_uio = &uio; 167 else 168 tag_uio = &out_uio; 169 170 /* Duplicate iovec and append vector for tag. */ 171 memcpy(oo->iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec)); 172 tag_uio->uio_iov = oo->iov; 173 tag_uio->uio_iov[iovcnt].iov_base = trailer; 174 tag_uio->uio_iov[iovcnt].iov_len = AES_GMAC_HASH_LEN; 175 tag_uio->uio_iovcnt++; 176 crp->crp_digest_start = tag_uio->uio_resid; 177 tag_uio->uio_resid += AES_GMAC_HASH_LEN; 178 179 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 180 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 181 crypto_use_uio(crp, &uio); 182 if (!inplace) 183 crypto_use_output_uio(crp, &out_uio); 184 crp->crp_opaque = oo; 185 crp->crp_callback = ktls_ocf_callback; 186 187 counter_u64_add(ocf_tls12_gcm_crypts, 1); 188 if (inplace) 189 counter_u64_add(ocf_inplace, 1); 190 else 191 counter_u64_add(ocf_separate_output, 1); 192 for (;;) { 193 error = crypto_dispatch(crp); 194 if (error) 195 break; 196 197 mtx_lock(&os->lock); 198 while (!oo->done) 199 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 200 mtx_unlock(&os->lock); 201 202 if (crp->crp_etype != EAGAIN) { 203 error = crp->crp_etype; 204 break; 205 } 206 207 crp->crp_etype = 0; 208 crp->crp_flags &= ~CRYPTO_F_DONE; 209 oo->done = false; 210 counter_u64_add(ocf_retries, 1); 211 } 212 213 crypto_freereq(crp); 214 free(oo, M_KTLS_OCF); 215 return (error); 216 } 217 218 static int 219 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, 220 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 221 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) 222 { 223 struct uio uio, out_uio; 224 struct tls_aead_data_13 ad; 225 char nonce[12]; 226 struct cryptop *crp; 227 struct ocf_session *os; 228 struct ocf_operation *oo; 229 struct iovec *iov, *out_iov; 230 int i, error; 231 bool inplace; 232 233 os = tls->cipher; 234 235 oo = malloc(sizeof(*oo) + (iovcnt + 1) * sizeof(*iov) * 2, M_KTLS_OCF, 236 M_WAITOK | M_ZERO); 237 oo->os = os; 238 iov = oo->iov; 239 out_iov = iov + iovcnt + 2; 240 241 crp = crypto_getreq(os->sid, M_WAITOK); 242 243 /* Setup the nonce. */ 244 memcpy(nonce, tls->params.iv, tls->params.iv_len); 245 *(uint64_t *)(nonce + 4) ^= htobe64(seqno); 246 247 /* Setup the AAD. */ 248 ad.type = hdr->tls_type; 249 ad.tls_vmajor = hdr->tls_vmajor; 250 ad.tls_vminor = hdr->tls_vminor; 251 ad.tls_length = hdr->tls_length; 252 crp->crp_aad = &ad; 253 crp->crp_aad_length = sizeof(ad); 254 255 /* Compute payload length and determine if encryption is in place. */ 256 inplace = true; 257 crp->crp_payload_start = 0; 258 for (i = 0; i < iovcnt; i++) { 259 if (iniov[i].iov_base != outiov[i].iov_base) 260 inplace = false; 261 crp->crp_payload_length += iniov[i].iov_len; 262 } 263 264 /* Store the record type as the first byte of the trailer. */ 265 trailer[0] = record_type; 266 crp->crp_payload_length++; 267 crp->crp_digest_start = crp->crp_payload_length; 268 269 /* 270 * Duplicate the input iov to append the trailer. Always 271 * include the full trailer as input to get the record_type 272 * even if only the first byte is used. 273 */ 274 memcpy(iov, iniov, iovcnt * sizeof(*iov)); 275 iov[iovcnt].iov_base = trailer; 276 iov[iovcnt].iov_len = AES_GMAC_HASH_LEN + 1; 277 uio.uio_iov = iov; 278 uio.uio_iovcnt = iovcnt + 1; 279 uio.uio_offset = 0; 280 uio.uio_resid = crp->crp_payload_length + AES_GMAC_HASH_LEN; 281 uio.uio_segflg = UIO_SYSSPACE; 282 uio.uio_td = curthread; 283 crypto_use_uio(crp, &uio); 284 285 if (!inplace) { 286 /* Duplicate the output iov to append the trailer. */ 287 memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov)); 288 out_iov[iovcnt] = iov[iovcnt]; 289 290 out_uio.uio_iov = out_iov; 291 out_uio.uio_iovcnt = iovcnt + 1; 292 out_uio.uio_offset = 0; 293 out_uio.uio_resid = crp->crp_payload_length + 294 AES_GMAC_HASH_LEN; 295 out_uio.uio_segflg = UIO_SYSSPACE; 296 out_uio.uio_td = curthread; 297 crypto_use_output_uio(crp, &out_uio); 298 } 299 300 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 301 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 302 crp->crp_opaque = oo; 303 crp->crp_callback = ktls_ocf_callback; 304 305 memcpy(crp->crp_iv, nonce, sizeof(nonce)); 306 307 counter_u64_add(ocf_tls13_gcm_crypts, 1); 308 if (inplace) 309 counter_u64_add(ocf_inplace, 1); 310 else 311 counter_u64_add(ocf_separate_output, 1); 312 for (;;) { 313 error = crypto_dispatch(crp); 314 if (error) 315 break; 316 317 mtx_lock(&os->lock); 318 while (!oo->done) 319 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 320 mtx_unlock(&os->lock); 321 322 if (crp->crp_etype != EAGAIN) { 323 error = crp->crp_etype; 324 break; 325 } 326 327 crp->crp_etype = 0; 328 crp->crp_flags &= ~CRYPTO_F_DONE; 329 oo->done = false; 330 counter_u64_add(ocf_retries, 1); 331 } 332 333 crypto_freereq(crp); 334 free(oo, M_KTLS_OCF); 335 return (error); 336 } 337 338 static void 339 ktls_ocf_free(struct ktls_session *tls) 340 { 341 struct ocf_session *os; 342 343 os = tls->cipher; 344 crypto_freesession(os->sid); 345 mtx_destroy(&os->lock); 346 zfree(os, M_KTLS_OCF); 347 } 348 349 static int 350 ktls_ocf_try(struct socket *so, struct ktls_session *tls) 351 { 352 struct crypto_session_params csp; 353 struct ocf_session *os; 354 int error; 355 356 memset(&csp, 0, sizeof(csp)); 357 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; 358 359 switch (tls->params.cipher_algorithm) { 360 case CRYPTO_AES_NIST_GCM_16: 361 switch (tls->params.cipher_key_len) { 362 case 128 / 8: 363 case 256 / 8: 364 break; 365 default: 366 return (EINVAL); 367 } 368 csp.csp_mode = CSP_MODE_AEAD; 369 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 370 csp.csp_cipher_key = tls->params.cipher_key; 371 csp.csp_cipher_klen = tls->params.cipher_key_len; 372 csp.csp_ivlen = AES_GCM_IV_LEN; 373 break; 374 default: 375 return (EPROTONOSUPPORT); 376 } 377 378 /* Only TLS 1.2 and 1.3 are supported. */ 379 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 380 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 381 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 382 return (EPROTONOSUPPORT); 383 384 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 385 if (os == NULL) 386 return (ENOMEM); 387 388 error = crypto_newsession(&os->sid, &csp, 389 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 390 if (error) { 391 free(os, M_KTLS_OCF); 392 return (error); 393 } 394 395 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 396 tls->cipher = os; 397 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 398 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; 399 else 400 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; 401 tls->free = ktls_ocf_free; 402 return (0); 403 } 404 405 struct ktls_crypto_backend ocf_backend = { 406 .name = "OCF", 407 .prio = 5, 408 .api_version = KTLS_API_VERSION, 409 .try = ktls_ocf_try, 410 }; 411 412 static int 413 ktls_ocf_modevent(module_t mod, int what, void *arg) 414 { 415 int error; 416 417 switch (what) { 418 case MOD_LOAD: 419 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK); 420 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK); 421 ocf_inplace = counter_u64_alloc(M_WAITOK); 422 ocf_separate_output = counter_u64_alloc(M_WAITOK); 423 ocf_retries = counter_u64_alloc(M_WAITOK); 424 return (ktls_crypto_backend_register(&ocf_backend)); 425 case MOD_UNLOAD: 426 error = ktls_crypto_backend_deregister(&ocf_backend); 427 if (error) 428 return (error); 429 counter_u64_free(ocf_tls12_gcm_crypts); 430 counter_u64_free(ocf_tls13_gcm_crypts); 431 counter_u64_free(ocf_inplace); 432 counter_u64_free(ocf_separate_output); 433 counter_u64_free(ocf_retries); 434 return (0); 435 default: 436 return (EOPNOTSUPP); 437 } 438 } 439 440 static moduledata_t ktls_ocf_moduledata = { 441 "ktls_ocf", 442 ktls_ocf_modevent, 443 NULL 444 }; 445 446 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); 447