1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /* System headers */ 5 #include <sys/param.h> 6 #include <sys/systm.h> 7 #include <sys/bus.h> 8 #include <sys/cpu.h> 9 #include <sys/kernel.h> 10 #include <sys/mbuf.h> 11 #include <sys/module.h> 12 #include <sys/mutex.h> 13 14 /* Cryptodev headers */ 15 #include <opencrypto/cryptodev.h> 16 #include "cryptodev_if.h" 17 18 /* QAT specific headers */ 19 #include "cpa.h" 20 #include "cpa_cy_im.h" 21 #include "cpa_cy_sym_dp.h" 22 #include "adf_accel_devices.h" 23 #include "adf_common_drv.h" 24 #include "lac_sym_hash_defs.h" 25 #include "lac_sym_qat_hash_defs_lookup.h" 26 27 /* To get only IRQ instances */ 28 #include "icp_accel_devices.h" 29 #include "icp_adf_accel_mgr.h" 30 #include "lac_sal_types.h" 31 32 /* To disable AEAD HW MAC verification */ 33 #include "icp_sal_user.h" 34 35 /* QAT OCF specific headers */ 36 #include "qat_ocf_mem_pool.h" 37 #include "qat_ocf_utils.h" 38 39 #define QAT_OCF_MAX_INSTANCES (256) 40 #define QAT_OCF_SESSION_WAIT_TIMEOUT_MS (1000) 41 42 MALLOC_DEFINE(M_QAT_OCF, "qat_ocf", "qat_ocf(4) memory allocations"); 43 44 /* QAT OCF internal structures */ 45 struct qat_ocf_softc { 46 device_t sc_dev; 47 int32_t cryptodev_id; 48 struct qat_ocf_instance cyInstHandles[QAT_OCF_MAX_INSTANCES]; 49 int32_t numCyInstances; 50 }; 51 52 /* Function definitions */ 53 static void qat_ocf_freesession(device_t dev, crypto_session_t cses); 54 static int qat_ocf_probesession(device_t dev, 55 const struct crypto_session_params *csp); 56 static int qat_ocf_newsession(device_t dev, 57 crypto_session_t cses, 58 const struct crypto_session_params *csp); 59 static int qat_ocf_attach(device_t dev); 60 static int qat_ocf_detach(device_t dev); 61 62 static void 63 symDpCallback(CpaCySymDpOpData *pOpData, 64 CpaStatus result, 65 CpaBoolean verifyResult) 66 { 67 struct qat_ocf_cookie *qat_cookie; 68 struct cryptop *crp; 69 struct qat_ocf_dsession *qat_dsession = NULL; 70 struct qat_ocf_session *qat_session = NULL; 71 struct qat_ocf_instance *qat_instance = NULL; 72 CpaStatus status; 73 int rc = 0; 74 75 qat_cookie = (struct qat_ocf_cookie *)pOpData->pCallbackTag; 76 if (!qat_cookie) 77 return; 78 79 crp = qat_cookie->crp_op; 80 81 qat_dsession = crypto_get_driver_session(crp->crp_session); 82 qat_instance = qat_dsession->qatInstance; 83 84 status = qat_ocf_cookie_dma_post_sync(crp, pOpData); 85 if (CPA_STATUS_SUCCESS != status) { 86 rc = EIO; 87 goto exit; 88 } 89 90 status = qat_ocf_cookie_dma_unload(crp, pOpData); 91 if (CPA_STATUS_SUCCESS != status) { 92 rc = EIO; 93 goto exit; 94 } 95 96 /* Verify result */ 97 if (CPA_STATUS_SUCCESS != result) { 98 rc = EBADMSG; 99 goto exit; 100 } 101 102 /* Verify digest by FW (GCM and CCM only) */ 103 if (CPA_TRUE != verifyResult) { 104 rc = EBADMSG; 105 goto exit; 106 } 107 108 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 109 qat_session = &qat_dsession->encSession; 110 else 111 qat_session = &qat_dsession->decSession; 112 113 /* Copy back digest result if it's stored in separated buffer */ 114 if (pOpData->digestResult && qat_session->authLen > 0) { 115 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { 116 char icv[QAT_OCF_MAX_DIGEST] = { 0 }; 117 crypto_copydata(crp, 118 crp->crp_digest_start, 119 qat_session->authLen, 120 icv); 121 if (timingsafe_bcmp(icv, 122 qat_cookie->qat_ocf_digest, 123 qat_session->authLen) != 0) { 124 rc = EBADMSG; 125 goto exit; 126 } 127 } else { 128 crypto_copyback(crp, 129 crp->crp_digest_start, 130 qat_session->authLen, 131 qat_cookie->qat_ocf_digest); 132 } 133 } 134 135 exit: 136 qat_ocf_cookie_free(qat_instance, qat_cookie); 137 crp->crp_etype = rc; 138 crypto_done(crp); 139 140 return; 141 } 142 143 static inline CpaPhysicalAddr 144 qatVirtToPhys(void *virtAddr) 145 { 146 return (CpaPhysicalAddr)vtophys(virtAddr); 147 } 148 149 static int 150 qat_ocf_probesession(device_t dev, const struct crypto_session_params *csp) 151 { 152 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 153 0) { 154 return EINVAL; 155 } 156 157 switch (csp->csp_mode) { 158 case CSP_MODE_CIPHER: 159 switch (csp->csp_cipher_alg) { 160 case CRYPTO_AES_CBC: 161 case CRYPTO_AES_ICM: 162 if (csp->csp_ivlen != AES_BLOCK_LEN) 163 return EINVAL; 164 break; 165 case CRYPTO_AES_XTS: 166 if (csp->csp_ivlen != AES_XTS_IV_LEN) 167 return EINVAL; 168 break; 169 default: 170 return EINVAL; 171 } 172 break; 173 case CSP_MODE_DIGEST: 174 switch (csp->csp_auth_alg) { 175 case CRYPTO_SHA1: 176 case CRYPTO_SHA1_HMAC: 177 case CRYPTO_SHA2_256: 178 case CRYPTO_SHA2_256_HMAC: 179 case CRYPTO_SHA2_384: 180 case CRYPTO_SHA2_384_HMAC: 181 case CRYPTO_SHA2_512: 182 case CRYPTO_SHA2_512_HMAC: 183 break; 184 case CRYPTO_AES_NIST_GMAC: 185 if (csp->csp_ivlen != AES_GCM_IV_LEN) 186 return EINVAL; 187 break; 188 default: 189 return EINVAL; 190 } 191 break; 192 case CSP_MODE_AEAD: 193 switch (csp->csp_cipher_alg) { 194 case CRYPTO_AES_NIST_GCM_16: 195 if (csp->csp_ivlen != AES_GCM_IV_LEN) 196 return EINVAL; 197 break; 198 default: 199 return EINVAL; 200 } 201 break; 202 case CSP_MODE_ETA: 203 switch (csp->csp_auth_alg) { 204 case CRYPTO_SHA1_HMAC: 205 case CRYPTO_SHA2_256_HMAC: 206 case CRYPTO_SHA2_384_HMAC: 207 case CRYPTO_SHA2_512_HMAC: 208 switch (csp->csp_cipher_alg) { 209 case CRYPTO_AES_CBC: 210 case CRYPTO_AES_ICM: 211 if (csp->csp_ivlen != AES_BLOCK_LEN) 212 return EINVAL; 213 break; 214 case CRYPTO_AES_XTS: 215 if (csp->csp_ivlen != AES_XTS_IV_LEN) 216 return EINVAL; 217 break; 218 default: 219 return EINVAL; 220 } 221 break; 222 default: 223 return EINVAL; 224 } 225 break; 226 default: 227 return EINVAL; 228 } 229 230 return CRYPTODEV_PROBE_HARDWARE; 231 } 232 233 static CpaStatus 234 qat_ocf_session_init(device_t dev, 235 struct cryptop *crp, 236 struct qat_ocf_instance *qat_instance, 237 struct qat_ocf_session *qat_ssession) 238 { 239 CpaStatus status = CPA_STATUS_SUCCESS; 240 /* Crytpodev structures */ 241 crypto_session_t cses; 242 const struct crypto_session_params *csp; 243 /* DP API Session configuration */ 244 CpaCySymSessionSetupData sessionSetupData = { 0 }; 245 CpaCySymSessionCtx sessionCtx = NULL; 246 Cpa32U sessionCtxSize = 0; 247 248 cses = crp->crp_session; 249 if (NULL == cses) { 250 device_printf(dev, "no crypto session in cryptodev request\n"); 251 return CPA_STATUS_FAIL; 252 } 253 254 csp = crypto_get_params(cses); 255 if (NULL == csp) { 256 device_printf(dev, "no session in cryptodev session\n"); 257 return CPA_STATUS_FAIL; 258 } 259 260 /* Common fields */ 261 sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH; 262 /* Cipher key */ 263 if (crp->crp_cipher_key) 264 sessionSetupData.cipherSetupData.pCipherKey = 265 crp->crp_cipher_key; 266 else 267 sessionSetupData.cipherSetupData.pCipherKey = 268 csp->csp_cipher_key; 269 sessionSetupData.cipherSetupData.cipherKeyLenInBytes = 270 csp->csp_cipher_klen; 271 272 /* Auth key */ 273 if (crp->crp_auth_key) 274 sessionSetupData.hashSetupData.authModeSetupData.authKey = 275 crp->crp_auth_key; 276 else 277 sessionSetupData.hashSetupData.authModeSetupData.authKey = 278 csp->csp_auth_key; 279 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = 280 csp->csp_auth_klen; 281 282 qat_ssession->aadLen = crp->crp_aad_length; 283 if (CPA_TRUE == is_sep_aad_supported(csp)) 284 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes = 285 crp->crp_aad_length; 286 else 287 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes = 288 0; 289 290 /* Just setup algorithm - regardless of mode */ 291 if (csp->csp_cipher_alg) { 292 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; 293 294 switch (csp->csp_cipher_alg) { 295 case CRYPTO_AES_CBC: 296 sessionSetupData.cipherSetupData.cipherAlgorithm = 297 CPA_CY_SYM_CIPHER_AES_CBC; 298 break; 299 case CRYPTO_AES_ICM: 300 sessionSetupData.cipherSetupData.cipherAlgorithm = 301 CPA_CY_SYM_CIPHER_AES_CTR; 302 break; 303 case CRYPTO_AES_XTS: 304 sessionSetupData.cipherSetupData.cipherAlgorithm = 305 CPA_CY_SYM_CIPHER_AES_XTS; 306 break; 307 case CRYPTO_AES_NIST_GCM_16: 308 sessionSetupData.cipherSetupData.cipherAlgorithm = 309 CPA_CY_SYM_CIPHER_AES_GCM; 310 sessionSetupData.hashSetupData.hashAlgorithm = 311 CPA_CY_SYM_HASH_AES_GCM; 312 sessionSetupData.hashSetupData.hashMode = 313 CPA_CY_SYM_HASH_MODE_AUTH; 314 break; 315 default: 316 device_printf(dev, 317 "cipher_alg: %d not supported\n", 318 csp->csp_cipher_alg); 319 status = CPA_STATUS_UNSUPPORTED; 320 goto fail; 321 } 322 } 323 324 if (csp->csp_auth_alg) { 325 switch (csp->csp_auth_alg) { 326 case CRYPTO_SHA1_HMAC: 327 sessionSetupData.hashSetupData.hashAlgorithm = 328 CPA_CY_SYM_HASH_SHA1; 329 sessionSetupData.hashSetupData.hashMode = 330 CPA_CY_SYM_HASH_MODE_AUTH; 331 break; 332 case CRYPTO_SHA1: 333 sessionSetupData.hashSetupData.hashAlgorithm = 334 CPA_CY_SYM_HASH_SHA1; 335 sessionSetupData.hashSetupData.hashMode = 336 CPA_CY_SYM_HASH_MODE_PLAIN; 337 break; 338 339 case CRYPTO_SHA2_256_HMAC: 340 sessionSetupData.hashSetupData.hashAlgorithm = 341 CPA_CY_SYM_HASH_SHA256; 342 sessionSetupData.hashSetupData.hashMode = 343 CPA_CY_SYM_HASH_MODE_AUTH; 344 break; 345 case CRYPTO_SHA2_256: 346 sessionSetupData.hashSetupData.hashAlgorithm = 347 CPA_CY_SYM_HASH_SHA256; 348 sessionSetupData.hashSetupData.hashMode = 349 CPA_CY_SYM_HASH_MODE_PLAIN; 350 break; 351 352 case CRYPTO_SHA2_224_HMAC: 353 sessionSetupData.hashSetupData.hashAlgorithm = 354 CPA_CY_SYM_HASH_SHA224; 355 sessionSetupData.hashSetupData.hashMode = 356 CPA_CY_SYM_HASH_MODE_AUTH; 357 break; 358 case CRYPTO_SHA2_224: 359 sessionSetupData.hashSetupData.hashAlgorithm = 360 CPA_CY_SYM_HASH_SHA224; 361 sessionSetupData.hashSetupData.hashMode = 362 CPA_CY_SYM_HASH_MODE_PLAIN; 363 break; 364 365 case CRYPTO_SHA2_384_HMAC: 366 sessionSetupData.hashSetupData.hashAlgorithm = 367 CPA_CY_SYM_HASH_SHA384; 368 sessionSetupData.hashSetupData.hashMode = 369 CPA_CY_SYM_HASH_MODE_AUTH; 370 break; 371 case CRYPTO_SHA2_384: 372 sessionSetupData.hashSetupData.hashAlgorithm = 373 CPA_CY_SYM_HASH_SHA384; 374 sessionSetupData.hashSetupData.hashMode = 375 CPA_CY_SYM_HASH_MODE_PLAIN; 376 break; 377 378 case CRYPTO_SHA2_512_HMAC: 379 sessionSetupData.hashSetupData.hashAlgorithm = 380 CPA_CY_SYM_HASH_SHA512; 381 sessionSetupData.hashSetupData.hashMode = 382 CPA_CY_SYM_HASH_MODE_AUTH; 383 break; 384 case CRYPTO_SHA2_512: 385 sessionSetupData.hashSetupData.hashAlgorithm = 386 CPA_CY_SYM_HASH_SHA512; 387 sessionSetupData.hashSetupData.hashMode = 388 CPA_CY_SYM_HASH_MODE_PLAIN; 389 break; 390 case CRYPTO_AES_NIST_GMAC: 391 sessionSetupData.hashSetupData.hashAlgorithm = 392 CPA_CY_SYM_HASH_AES_GMAC; 393 break; 394 default: 395 status = CPA_STATUS_UNSUPPORTED; 396 goto fail; 397 } 398 } /* csp->csp_auth_alg */ 399 400 /* Setting digest-length if no cipher-only mode is set */ 401 if (csp->csp_mode != CSP_MODE_CIPHER) { 402 lac_sym_qat_hash_defs_t *pHashDefsInfo = NULL; 403 if (csp->csp_auth_mlen) { 404 sessionSetupData.hashSetupData.digestResultLenInBytes = 405 csp->csp_auth_mlen; 406 qat_ssession->authLen = csp->csp_auth_mlen; 407 } else { 408 LacSymQat_HashDefsLookupGet( 409 qat_instance->cyInstHandle, 410 sessionSetupData.hashSetupData.hashAlgorithm, 411 &pHashDefsInfo); 412 if (NULL == pHashDefsInfo) { 413 device_printf( 414 dev, 415 "unable to find corresponding hash data\n"); 416 status = CPA_STATUS_UNSUPPORTED; 417 goto fail; 418 } 419 sessionSetupData.hashSetupData.digestResultLenInBytes = 420 pHashDefsInfo->algInfo->digestLength; 421 qat_ssession->authLen = 422 pHashDefsInfo->algInfo->digestLength; 423 } 424 sessionSetupData.verifyDigest = CPA_FALSE; 425 } 426 427 switch (csp->csp_mode) { 428 case CSP_MODE_AEAD: 429 case CSP_MODE_ETA: 430 sessionSetupData.symOperation = 431 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 432 /* Place the digest result in a buffer unrelated to srcBuffer */ 433 sessionSetupData.digestIsAppended = CPA_FALSE; 434 /* Due to FW limitation to verify only appended MACs */ 435 sessionSetupData.verifyDigest = CPA_FALSE; 436 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 437 sessionSetupData.cipherSetupData.cipherDirection = 438 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 439 sessionSetupData.algChainOrder = 440 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 441 } else { 442 sessionSetupData.cipherSetupData.cipherDirection = 443 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 444 sessionSetupData.algChainOrder = 445 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 446 } 447 break; 448 case CSP_MODE_CIPHER: 449 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 450 sessionSetupData.cipherSetupData.cipherDirection = 451 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 452 } else { 453 sessionSetupData.cipherSetupData.cipherDirection = 454 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 455 } 456 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; 457 break; 458 case CSP_MODE_DIGEST: 459 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; 460 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { 461 sessionSetupData.symOperation = 462 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 463 /* GMAC is always encrypt */ 464 sessionSetupData.cipherSetupData.cipherDirection = 465 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 466 sessionSetupData.algChainOrder = 467 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 468 sessionSetupData.cipherSetupData.cipherAlgorithm = 469 CPA_CY_SYM_CIPHER_AES_GCM; 470 sessionSetupData.hashSetupData.hashAlgorithm = 471 CPA_CY_SYM_HASH_AES_GMAC; 472 sessionSetupData.hashSetupData.hashMode = 473 CPA_CY_SYM_HASH_MODE_AUTH; 474 /* Same key for cipher and auth */ 475 sessionSetupData.cipherSetupData.pCipherKey = 476 csp->csp_auth_key; 477 sessionSetupData.cipherSetupData.cipherKeyLenInBytes = 478 csp->csp_auth_klen; 479 /* Generated GMAC stored in separated buffer */ 480 sessionSetupData.digestIsAppended = CPA_FALSE; 481 /* Digest verification not allowed in GMAC case */ 482 sessionSetupData.verifyDigest = CPA_FALSE; 483 /* No AAD allowed */ 484 sessionSetupData.hashSetupData.authModeSetupData 485 .aadLenInBytes = 0; 486 } else { 487 sessionSetupData.cipherSetupData.cipherDirection = 488 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 489 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; 490 sessionSetupData.digestIsAppended = CPA_FALSE; 491 } 492 break; 493 default: 494 device_printf(dev, 495 "%s: unhandled crypto algorithm %d, %d\n", 496 __func__, 497 csp->csp_cipher_alg, 498 csp->csp_auth_alg); 499 status = CPA_STATUS_FAIL; 500 goto fail; 501 } 502 503 /* Extracting session size */ 504 status = cpaCySymSessionCtxGetSize(qat_instance->cyInstHandle, 505 &sessionSetupData, 506 &sessionCtxSize); 507 if (CPA_STATUS_SUCCESS != status) { 508 device_printf(dev, "unable to get session size\n"); 509 goto fail; 510 } 511 512 /* Allocating contiguous memory for session */ 513 sessionCtx = contigmalloc(sessionCtxSize, 514 M_QAT_OCF, 515 M_NOWAIT, 516 0, 517 ~1UL, 518 1 << (bsrl(sessionCtxSize - 1) + 1), 519 0); 520 if (NULL == sessionCtx) { 521 device_printf(dev, "unable to allocate memory for session\n"); 522 status = CPA_STATUS_RESOURCE; 523 goto fail; 524 } 525 526 status = cpaCySymDpInitSession(qat_instance->cyInstHandle, 527 &sessionSetupData, 528 sessionCtx); 529 if (CPA_STATUS_SUCCESS != status) { 530 device_printf(dev, "session initialization failed\n"); 531 goto fail; 532 } 533 534 /* NOTE: lets keep double session (both directions) approach to overcome 535 * lack of direction update in FBSD QAT. 536 */ 537 qat_ssession->sessionCtx = sessionCtx; 538 qat_ssession->sessionCtxSize = sessionCtxSize; 539 540 return CPA_STATUS_SUCCESS; 541 542 fail: 543 /* Release resources if any */ 544 if (sessionCtx) 545 contigfree(sessionCtx, sessionCtxSize, M_QAT_OCF); 546 547 return status; 548 } 549 550 static int 551 qat_ocf_newsession(device_t dev, 552 crypto_session_t cses, 553 const struct crypto_session_params *csp) 554 { 555 /* Cryptodev QAT structures */ 556 struct qat_ocf_softc *qat_softc; 557 struct qat_ocf_dsession *qat_dsession; 558 struct qat_ocf_instance *qat_instance; 559 u_int cpu_id = PCPU_GET(cpuid); 560 561 /* Create cryptodev session */ 562 qat_softc = device_get_softc(dev); 563 qat_instance = 564 &qat_softc->cyInstHandles[cpu_id % qat_softc->numCyInstances]; 565 qat_dsession = crypto_get_driver_session(cses); 566 if (NULL == qat_dsession) { 567 device_printf(dev, "Unable to create new session\n"); 568 return (EINVAL); 569 } 570 571 /* Add only instance at this point remaining operations moved to 572 * lazy session init */ 573 qat_dsession->qatInstance = qat_instance; 574 575 return 0; 576 } 577 578 static CpaStatus 579 qat_ocf_remove_session(device_t dev, 580 CpaInstanceHandle cyInstHandle, 581 struct qat_ocf_session *qat_session) 582 { 583 CpaStatus status = CPA_STATUS_SUCCESS; 584 585 if (NULL == qat_session->sessionCtx) 586 return CPA_STATUS_SUCCESS; 587 588 /* User callback is executed right before decrementing pending 589 * callback atomic counter. To avoid removing session rejection 590 * we have to wait a very short while for counter update 591 * after call back execution. */ 592 status = qat_ocf_wait_for_session(qat_session->sessionCtx, 593 QAT_OCF_SESSION_WAIT_TIMEOUT_MS); 594 if (CPA_STATUS_SUCCESS != status) { 595 device_printf(dev, "waiting for session un-busy failed\n"); 596 return CPA_STATUS_FAIL; 597 } 598 599 status = cpaCySymDpRemoveSession(cyInstHandle, qat_session->sessionCtx); 600 if (CPA_STATUS_SUCCESS != status) { 601 device_printf(dev, "error while removing session\n"); 602 return CPA_STATUS_FAIL; 603 } 604 605 explicit_bzero(qat_session->sessionCtx, qat_session->sessionCtxSize); 606 contigfree(qat_session->sessionCtx, 607 qat_session->sessionCtxSize, 608 M_QAT_OCF); 609 qat_session->sessionCtx = NULL; 610 qat_session->sessionCtxSize = 0; 611 612 return CPA_STATUS_SUCCESS; 613 } 614 615 static void 616 qat_ocf_freesession(device_t dev, crypto_session_t cses) 617 { 618 CpaStatus status = CPA_STATUS_SUCCESS; 619 struct qat_ocf_dsession *qat_dsession = NULL; 620 struct qat_ocf_instance *qat_instance = NULL; 621 622 qat_dsession = crypto_get_driver_session(cses); 623 qat_instance = qat_dsession->qatInstance; 624 mtx_lock(&qat_instance->cyInstMtx); 625 status = qat_ocf_remove_session(dev, 626 qat_dsession->qatInstance->cyInstHandle, 627 &qat_dsession->encSession); 628 if (CPA_STATUS_SUCCESS != status) 629 device_printf(dev, "unable to remove encrypt session\n"); 630 status = qat_ocf_remove_session(dev, 631 qat_dsession->qatInstance->cyInstHandle, 632 &qat_dsession->decSession); 633 if (CPA_STATUS_SUCCESS != status) 634 device_printf(dev, "unable to remove decrypt session\n"); 635 mtx_unlock(&qat_instance->cyInstMtx); 636 } 637 638 /* QAT GCM/CCM FW API are only algorithms which support separated AAD. */ 639 static CpaStatus 640 qat_ocf_load_aad_gcm(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 641 { 642 CpaCySymDpOpData *pOpData; 643 644 pOpData = &qat_cookie->pOpdata; 645 646 if (NULL != crp->crp_aad) 647 memcpy(qat_cookie->qat_ocf_gcm_aad, 648 crp->crp_aad, 649 crp->crp_aad_length); 650 else 651 crypto_copydata(crp, 652 crp->crp_aad_start, 653 crp->crp_aad_length, 654 qat_cookie->qat_ocf_gcm_aad); 655 656 pOpData->pAdditionalAuthData = qat_cookie->qat_ocf_gcm_aad; 657 pOpData->additionalAuthData = qat_cookie->qat_ocf_gcm_aad_paddr; 658 659 return CPA_STATUS_SUCCESS; 660 } 661 662 static CpaStatus 663 qat_ocf_load_aad(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 664 { 665 CpaStatus status = CPA_STATUS_SUCCESS; 666 const struct crypto_session_params *csp; 667 CpaCySymDpOpData *pOpData; 668 struct qat_ocf_load_cb_arg args; 669 670 pOpData = &qat_cookie->pOpdata; 671 pOpData->pAdditionalAuthData = NULL; 672 pOpData->additionalAuthData = 0UL; 673 674 if (crp->crp_aad_length == 0) 675 return CPA_STATUS_SUCCESS; 676 677 if (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX) 678 return CPA_STATUS_FAIL; 679 680 csp = crypto_get_params(crp->crp_session); 681 682 /* Handle GCM/CCM case */ 683 if (CPA_TRUE == is_sep_aad_supported(csp)) 684 return qat_ocf_load_aad_gcm(crp, qat_cookie); 685 686 if (NULL == crp->crp_aad) { 687 /* AAD already embedded in source buffer */ 688 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 689 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start; 690 691 pOpData->messageLenToHashInBytes = 692 crp->crp_aad_length + crp->crp_payload_length; 693 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start; 694 695 return CPA_STATUS_SUCCESS; 696 } 697 698 /* Separated AAD not supported by QAT - lets place the content 699 * of ADD buffer at the very beginning of source SGL */ 700 args.crp_op = crp; 701 args.qat_cookie = qat_cookie; 702 args.pOpData = pOpData; 703 args.error = 0; 704 status = bus_dmamap_load(qat_cookie->gcm_aad_dma_mem.dma_tag, 705 qat_cookie->gcm_aad_dma_mem.dma_map, 706 crp->crp_aad, 707 crp->crp_aad_length, 708 qat_ocf_crypto_load_aadbuf_cb, 709 &args, 710 BUS_DMA_NOWAIT); 711 qat_cookie->is_sep_aad_used = CPA_TRUE; 712 713 /* Right after this step we have AAD placed in the first flat buffer 714 * in source SGL */ 715 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 716 pOpData->cryptoStartSrcOffsetInBytes = 717 crp->crp_aad_length + crp->crp_aad_start + crp->crp_payload_start; 718 719 pOpData->messageLenToHashInBytes = 720 crp->crp_aad_length + crp->crp_payload_length; 721 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start; 722 723 return status; 724 } 725 726 static CpaStatus 727 qat_ocf_load(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 728 { 729 CpaStatus status = CPA_STATUS_SUCCESS; 730 CpaCySymDpOpData *pOpData; 731 struct qat_ocf_load_cb_arg args; 732 /* cryptodev internals */ 733 const struct crypto_session_params *csp; 734 735 pOpData = &qat_cookie->pOpdata; 736 737 csp = crypto_get_params(crp->crp_session); 738 739 /* Load IV buffer if present */ 740 if (csp->csp_ivlen > 0) { 741 memset(qat_cookie->qat_ocf_iv_buf, 742 0, 743 sizeof(qat_cookie->qat_ocf_iv_buf)); 744 crypto_read_iv(crp, qat_cookie->qat_ocf_iv_buf); 745 pOpData->iv = qat_cookie->qat_ocf_iv_buf_paddr; 746 pOpData->pIv = qat_cookie->qat_ocf_iv_buf; 747 pOpData->ivLenInBytes = csp->csp_ivlen; 748 } 749 750 /* GCM/CCM - load AAD to separated buffer 751 * AES+SHA - load AAD to first flat in SGL */ 752 status = qat_ocf_load_aad(crp, qat_cookie); 753 if (CPA_STATUS_SUCCESS != status) 754 goto fail; 755 756 /* Load source buffer */ 757 args.crp_op = crp; 758 args.qat_cookie = qat_cookie; 759 args.pOpData = pOpData; 760 args.error = 0; 761 status = bus_dmamap_load_crp_buffer(qat_cookie->src_dma_mem.dma_tag, 762 qat_cookie->src_dma_mem.dma_map, 763 &crp->crp_buf, 764 qat_ocf_crypto_load_buf_cb, 765 &args, 766 BUS_DMA_NOWAIT); 767 if (CPA_STATUS_SUCCESS != status) 768 goto fail; 769 pOpData->srcBuffer = qat_cookie->src_buffer_list_paddr; 770 pOpData->srcBufferLen = CPA_DP_BUFLIST; 771 772 /* Load destination buffer */ 773 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 774 status = 775 bus_dmamap_load_crp_buffer(qat_cookie->dst_dma_mem.dma_tag, 776 qat_cookie->dst_dma_mem.dma_map, 777 &crp->crp_obuf, 778 qat_ocf_crypto_load_obuf_cb, 779 &args, 780 BUS_DMA_NOWAIT); 781 if (CPA_STATUS_SUCCESS != status) 782 goto fail; 783 pOpData->dstBuffer = qat_cookie->dst_buffer_list_paddr; 784 pOpData->dstBufferLen = CPA_DP_BUFLIST; 785 } else { 786 pOpData->dstBuffer = pOpData->srcBuffer; 787 pOpData->dstBufferLen = pOpData->srcBufferLen; 788 } 789 790 if (CPA_TRUE == is_use_sep_digest(csp)) 791 pOpData->digestResult = qat_cookie->qat_ocf_digest_paddr; 792 else 793 pOpData->digestResult = 0UL; 794 795 /* GMAC - aka zero length buffer */ 796 if (CPA_TRUE == is_gmac_exception(csp)) 797 pOpData->messageLenToCipherInBytes = 0; 798 799 fail: 800 return status; 801 } 802 803 static int 804 qat_ocf_check_input(device_t dev, struct cryptop *crp) 805 { 806 const struct crypto_session_params *csp; 807 csp = crypto_get_params(crp->crp_session); 808 809 if (crypto_buffer_len(&crp->crp_buf) > QAT_OCF_MAX_LEN) 810 return E2BIG; 811 812 if (CPA_TRUE == is_sep_aad_supported(csp) && 813 (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX)) 814 return EBADMSG; 815 816 return 0; 817 } 818 819 static int 820 qat_ocf_process(device_t dev, struct cryptop *crp, int hint) 821 { 822 CpaStatus status = CPA_STATUS_SUCCESS; 823 int rc = 0; 824 struct qat_ocf_dsession *qat_dsession = NULL; 825 struct qat_ocf_session *qat_session = NULL; 826 struct qat_ocf_instance *qat_instance = NULL; 827 CpaCySymDpOpData *pOpData = NULL; 828 struct qat_ocf_cookie *qat_cookie = NULL; 829 CpaBoolean memLoaded = CPA_FALSE; 830 831 rc = qat_ocf_check_input(dev, crp); 832 if (rc) 833 goto fail; 834 835 qat_dsession = crypto_get_driver_session(crp->crp_session); 836 837 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 838 qat_session = &qat_dsession->encSession; 839 else 840 qat_session = &qat_dsession->decSession; 841 qat_instance = qat_dsession->qatInstance; 842 843 status = qat_ocf_cookie_alloc(qat_instance, &qat_cookie); 844 if (CPA_STATUS_SUCCESS != status) { 845 rc = EAGAIN; 846 goto fail; 847 } 848 849 qat_cookie->crp_op = crp; 850 851 /* Common request fields */ 852 pOpData = &qat_cookie->pOpdata; 853 pOpData->instanceHandle = qat_instance->cyInstHandle; 854 pOpData->sessionCtx = NULL; 855 856 /* Cipher fields */ 857 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start; 858 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 859 /* Digest fields - any exceptions from this basic rules are covered 860 * in qat_ocf_load */ 861 pOpData->hashStartSrcOffsetInBytes = crp->crp_payload_start; 862 pOpData->messageLenToHashInBytes = crp->crp_payload_length; 863 864 status = qat_ocf_load(crp, qat_cookie); 865 if (CPA_STATUS_SUCCESS != status) { 866 device_printf(dev, 867 "unable to load OCF buffers to QAT DMA " 868 "transaction\n"); 869 rc = EIO; 870 goto fail; 871 } 872 memLoaded = CPA_TRUE; 873 874 status = qat_ocf_cookie_dma_pre_sync(crp, pOpData); 875 if (CPA_STATUS_SUCCESS != status) { 876 device_printf(dev, "unable to sync DMA buffers\n"); 877 rc = EIO; 878 goto fail; 879 } 880 881 mtx_lock(&qat_instance->cyInstMtx); 882 /* Session initialization at the first request. It's done 883 * in such way to overcome missing QAT specific session data 884 * such like AAD length and limited possibility to update 885 * QAT session while handling traffic. 886 */ 887 if (NULL == qat_session->sessionCtx) { 888 status = 889 qat_ocf_session_init(dev, crp, qat_instance, qat_session); 890 if (CPA_STATUS_SUCCESS != status) { 891 mtx_unlock(&qat_instance->cyInstMtx); 892 device_printf(dev, "unable to init session\n"); 893 rc = EIO; 894 goto fail; 895 } 896 } else { 897 status = qat_ocf_handle_session_update(qat_dsession, crp); 898 if (CPA_STATUS_RESOURCE == status) { 899 mtx_unlock(&qat_instance->cyInstMtx); 900 rc = EAGAIN; 901 goto fail; 902 } else if (CPA_STATUS_SUCCESS != status) { 903 mtx_unlock(&qat_instance->cyInstMtx); 904 rc = EIO; 905 goto fail; 906 } 907 } 908 pOpData->sessionCtx = qat_session->sessionCtx; 909 status = cpaCySymDpEnqueueOp(pOpData, CPA_TRUE); 910 mtx_unlock(&qat_instance->cyInstMtx); 911 if (CPA_STATUS_SUCCESS != status) { 912 if (CPA_STATUS_RETRY == status) { 913 rc = EAGAIN; 914 goto fail; 915 } 916 device_printf(dev, 917 "unable to send request. Status: %d\n", 918 status); 919 rc = EIO; 920 goto fail; 921 } 922 923 return 0; 924 fail: 925 if (qat_cookie) { 926 if (memLoaded) 927 qat_ocf_cookie_dma_unload(crp, pOpData); 928 qat_ocf_cookie_free(qat_instance, qat_cookie); 929 } 930 crp->crp_etype = rc; 931 crypto_done(crp); 932 933 return 0; 934 } 935 936 static void 937 qat_ocf_identify(driver_t *drv, device_t parent) 938 { 939 if (device_find_child(parent, "qat_ocf", -1) == NULL && 940 BUS_ADD_CHILD(parent, 200, "qat_ocf", -1) == 0) 941 device_printf(parent, "qat_ocf: could not attach!"); 942 } 943 944 static int 945 qat_ocf_probe(device_t dev) 946 { 947 device_set_desc(dev, "QAT engine"); 948 return (BUS_PROBE_NOWILDCARD); 949 } 950 951 static CpaStatus 952 qat_ocf_get_irq_instances(CpaInstanceHandle *cyInstHandles, 953 Cpa16U cyInstHandlesSize, 954 Cpa16U *foundInstances) 955 { 956 CpaStatus status = CPA_STATUS_SUCCESS; 957 icp_accel_dev_t **pAdfInsts = NULL; 958 icp_accel_dev_t *dev_addr = NULL; 959 sal_t *baseAddr = NULL; 960 sal_list_t *listTemp = NULL; 961 CpaInstanceHandle cyInstHandle; 962 CpaInstanceInfo2 info; 963 Cpa16U numDevices; 964 Cpa32U instCtr = 0; 965 Cpa32U i; 966 967 /* Get the number of devices */ 968 status = icp_amgr_getNumInstances(&numDevices); 969 if (CPA_STATUS_SUCCESS != status) 970 return status; 971 972 /* Allocate memory to store addr of accel_devs */ 973 pAdfInsts = 974 malloc(numDevices * sizeof(icp_accel_dev_t *), M_QAT_OCF, M_WAITOK); 975 976 /* Get ADF to return all accel_devs that support either 977 * symmetric or asymmetric crypto */ 978 status = icp_amgr_getAllAccelDevByCapabilities( 979 (ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC), pAdfInsts, &numDevices); 980 if (CPA_STATUS_SUCCESS != status) { 981 free(pAdfInsts, M_QAT_OCF); 982 return status; 983 } 984 985 for (i = 0; i < numDevices; i++) { 986 dev_addr = (icp_accel_dev_t *)pAdfInsts[i]; 987 baseAddr = dev_addr->pSalHandle; 988 if (NULL == baseAddr) 989 continue; 990 listTemp = baseAddr->sym_services; 991 while (NULL != listTemp) { 992 cyInstHandle = SalList_getObject(listTemp); 993 status = cpaCyInstanceGetInfo2(cyInstHandle, &info); 994 if (CPA_STATUS_SUCCESS != status) 995 continue; 996 listTemp = SalList_next(listTemp); 997 if (CPA_TRUE == info.isPolled) 998 continue; 999 if (instCtr >= cyInstHandlesSize) 1000 break; 1001 cyInstHandles[instCtr++] = cyInstHandle; 1002 } 1003 } 1004 free(pAdfInsts, M_QAT_OCF); 1005 *foundInstances = instCtr; 1006 1007 return CPA_STATUS_SUCCESS; 1008 } 1009 1010 static CpaStatus 1011 qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev) 1012 { 1013 CpaStatus status = CPA_STATUS_SUCCESS; 1014 Cpa16U numInstances = 0; 1015 CpaInstanceHandle cyInstHandles[QAT_OCF_MAX_INSTANCES] = { 0 }; 1016 CpaInstanceHandle cyInstHandle = NULL; 1017 Cpa32U startedInstances = 0; 1018 Cpa32U i; 1019 1020 qat_softc->numCyInstances = 0; 1021 status = qat_ocf_get_irq_instances(cyInstHandles, 1022 QAT_OCF_MAX_INSTANCES, 1023 &numInstances); 1024 if (CPA_STATUS_SUCCESS != status) 1025 return status; 1026 if (0 == numInstances) 1027 return CPA_STATUS_RESOURCE; 1028 1029 for (i = 0; i < numInstances; i++) { 1030 struct qat_ocf_instance *qat_ocf_instance; 1031 1032 cyInstHandle = cyInstHandles[i]; 1033 if (!cyInstHandle) 1034 continue; 1035 1036 /* Starting instance */ 1037 status = cpaCyStartInstance(cyInstHandle); 1038 if (CPA_STATUS_SUCCESS != status) { 1039 device_printf(qat_softc->sc_dev, 1040 "unable to get start instance\n"); 1041 continue; 1042 } 1043 1044 status = 1045 cpaCySetAddressTranslation(cyInstHandle, qatVirtToPhys); 1046 if (CPA_STATUS_SUCCESS != status) { 1047 device_printf(qat_softc->sc_dev, 1048 "unable to add virt to phys callback"); 1049 goto fail; 1050 } 1051 1052 status = cpaCySymDpRegCbFunc(cyInstHandle, symDpCallback); 1053 if (CPA_STATUS_SUCCESS != status) { 1054 device_printf(qat_softc->sc_dev, 1055 "unable to add user callback\n"); 1056 goto fail; 1057 } 1058 1059 qat_ocf_instance = &qat_softc->cyInstHandles[startedInstances]; 1060 qat_ocf_instance->cyInstHandle = cyInstHandle; 1061 mtx_init(&qat_ocf_instance->cyInstMtx, 1062 "Instance MTX", 1063 NULL, 1064 MTX_DEF); 1065 1066 /* Initialize cookie pool */ 1067 status = qat_ocf_cookie_pool_init(qat_ocf_instance, dev); 1068 if (CPA_STATUS_SUCCESS != status) { 1069 device_printf(qat_softc->sc_dev, 1070 "unable to create cookie pool\n"); 1071 goto fail; 1072 } 1073 1074 /* Disable forcing HW MAC validation for AEAD */ 1075 status = icp_sal_setForceAEADMACVerify(cyInstHandle, CPA_FALSE); 1076 if (CPA_STATUS_SUCCESS != status) { 1077 device_printf( 1078 qat_softc->sc_dev, 1079 "unable to disable AEAD HW MAC verification\n"); 1080 goto fail; 1081 } 1082 1083 qat_ocf_instance->driver_id = qat_softc->cryptodev_id; 1084 1085 startedInstances++; 1086 continue; 1087 fail: 1088 /* Stop instance */ 1089 status = cpaCyStopInstance(cyInstHandle); 1090 if (CPA_STATUS_SUCCESS != status) 1091 device_printf(qat_softc->sc_dev, 1092 "unable to stop the instance\n"); 1093 continue; 1094 } 1095 qat_softc->numCyInstances = startedInstances; 1096 1097 /* Success if at least one instance has been set */ 1098 if (!qat_softc->numCyInstances) 1099 return CPA_STATUS_FAIL; 1100 1101 return CPA_STATUS_SUCCESS; 1102 } 1103 1104 static CpaStatus 1105 qat_ocf_stop_instances(struct qat_ocf_softc *qat_softc) 1106 { 1107 CpaStatus status = CPA_STATUS_SUCCESS; 1108 int i; 1109 1110 for (i = 0; i < qat_softc->numCyInstances; i++) { 1111 struct qat_ocf_instance *qat_instance; 1112 1113 qat_instance = &qat_softc->cyInstHandles[i]; 1114 status = cpaCyStopInstance(qat_instance->cyInstHandle); 1115 if (CPA_STATUS_SUCCESS != status) { 1116 pr_err("QAT: stopping instance id: %d failed\n", i); 1117 mtx_unlock(&qat_instance->cyInstMtx); 1118 continue; 1119 } 1120 qat_ocf_cookie_pool_deinit(qat_instance); 1121 mtx_destroy(&qat_instance->cyInstMtx); 1122 } 1123 1124 return status; 1125 } 1126 1127 static int 1128 qat_ocf_attach(device_t dev) 1129 { 1130 int status; 1131 struct qat_ocf_softc *qat_softc; 1132 int32_t cryptodev_id; 1133 1134 qat_softc = device_get_softc(dev); 1135 qat_softc->sc_dev = dev; 1136 1137 cryptodev_id = crypto_get_driverid(dev, 1138 sizeof(struct qat_ocf_dsession), 1139 CRYPTOCAP_F_HARDWARE); 1140 if (cryptodev_id < 0) { 1141 device_printf(dev, "cannot initialize!\n"); 1142 goto fail; 1143 } 1144 qat_softc->cryptodev_id = cryptodev_id; 1145 1146 /* Starting instances for OCF */ 1147 status = qat_ocf_start_instances(qat_softc, dev); 1148 if (status) { 1149 device_printf(dev, "no QAT IRQ instances available\n"); 1150 goto fail; 1151 } 1152 1153 return 0; 1154 fail: 1155 qat_ocf_detach(dev); 1156 1157 return (ENXIO); 1158 } 1159 1160 static int 1161 qat_ocf_detach(device_t dev) 1162 { 1163 struct qat_ocf_softc *qat_softc = NULL; 1164 CpaStatus cpaStatus; 1165 int status = 0; 1166 1167 qat_softc = device_get_softc(dev); 1168 1169 if (qat_softc->cryptodev_id >= 0) { 1170 status = crypto_unregister_all(qat_softc->cryptodev_id); 1171 if (status) 1172 device_printf(dev, 1173 "unable to unregister QAt backend\n"); 1174 } 1175 1176 /* Stop QAT instances */ 1177 cpaStatus = qat_ocf_stop_instances(qat_softc); 1178 if (CPA_STATUS_SUCCESS != cpaStatus) { 1179 device_printf(dev, "unable to stop instances\n"); 1180 status = EIO; 1181 } 1182 1183 return status; 1184 } 1185 1186 static device_method_t qat_ocf_methods[] = 1187 { DEVMETHOD(device_identify, qat_ocf_identify), 1188 DEVMETHOD(device_probe, qat_ocf_probe), 1189 DEVMETHOD(device_attach, qat_ocf_attach), 1190 DEVMETHOD(device_detach, qat_ocf_detach), 1191 1192 /* Cryptodev interface */ 1193 DEVMETHOD(cryptodev_probesession, qat_ocf_probesession), 1194 DEVMETHOD(cryptodev_newsession, qat_ocf_newsession), 1195 DEVMETHOD(cryptodev_freesession, qat_ocf_freesession), 1196 DEVMETHOD(cryptodev_process, qat_ocf_process), 1197 1198 DEVMETHOD_END }; 1199 1200 static driver_t qat_ocf_driver = { 1201 .name = "qat_ocf", 1202 .methods = qat_ocf_methods, 1203 .size = sizeof(struct qat_ocf_softc), 1204 }; 1205 1206 1207 DRIVER_MODULE_ORDERED(qat, 1208 nexus, 1209 qat_ocf_driver, 1210 NULL, 1211 NULL, 1212 SI_ORDER_ANY); 1213 MODULE_VERSION(qat, 1); 1214 MODULE_DEPEND(qat, qat_c62x, 1, 1, 1); 1215 MODULE_DEPEND(qat, qat_200xx, 1, 1, 1); 1216 MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1); 1217 MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1); 1218 MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1); 1219 MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1); 1220 MODULE_DEPEND(qat, crypto, 1, 1, 1); 1221 MODULE_DEPEND(qat, qat_common, 1, 1, 1); 1222 MODULE_DEPEND(qat, qat_api, 1, 1, 1); 1223 MODULE_DEPEND(qat, linuxkpi, 1, 1, 1); 1224