1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /* System headers */ 5 #include <sys/param.h> 6 #include <sys/systm.h> 7 #include <sys/bus.h> 8 #include <sys/cpu.h> 9 #include <sys/kernel.h> 10 #include <sys/mbuf.h> 11 #include <sys/module.h> 12 #include <sys/mutex.h> 13 14 /* Cryptodev headers */ 15 #include <opencrypto/cryptodev.h> 16 #include "cryptodev_if.h" 17 18 /* QAT specific headers */ 19 #include "cpa.h" 20 #include "cpa_cy_im.h" 21 #include "cpa_cy_sym_dp.h" 22 #include "adf_accel_devices.h" 23 #include "adf_common_drv.h" 24 #include "lac_sym_hash_defs.h" 25 #include "lac_sym_qat_hash_defs_lookup.h" 26 27 /* To get only IRQ instances */ 28 #include "icp_accel_devices.h" 29 #include "icp_adf_accel_mgr.h" 30 #include "lac_sal_types.h" 31 32 /* QAT OCF specific headers */ 33 #include "qat_ocf_mem_pool.h" 34 #include "qat_ocf_utils.h" 35 36 #define QAT_OCF_MAX_INSTANCES (256) 37 #define QAT_OCF_SESSION_WAIT_TIMEOUT_MS (1000) 38 39 MALLOC_DEFINE(M_QAT_OCF, "qat_ocf", "qat_ocf(4) memory allocations"); 40 41 /* QAT OCF internal structures */ 42 struct qat_ocf_softc { 43 device_t sc_dev; 44 int32_t cryptodev_id; 45 struct qat_ocf_instance cyInstHandles[QAT_OCF_MAX_INSTANCES]; 46 int32_t numCyInstances; 47 }; 48 49 /* Function definitions */ 50 static void qat_ocf_freesession(device_t dev, crypto_session_t cses); 51 static int qat_ocf_probesession(device_t dev, 52 const struct crypto_session_params *csp); 53 static int qat_ocf_newsession(device_t dev, 54 crypto_session_t cses, 55 const struct crypto_session_params *csp); 56 static int qat_ocf_attach(device_t dev); 57 static int qat_ocf_detach(device_t dev); 58 59 static void 60 symDpCallback(CpaCySymDpOpData *pOpData, 61 CpaStatus result, 62 CpaBoolean verifyResult) 63 { 64 struct qat_ocf_cookie *qat_cookie; 65 struct cryptop *crp; 66 struct qat_ocf_dsession *qat_dsession = NULL; 67 struct qat_ocf_session *qat_session = NULL; 68 struct qat_ocf_instance *qat_instance = NULL; 69 CpaStatus status; 70 int rc = 0; 71 72 qat_cookie = (struct qat_ocf_cookie *)pOpData->pCallbackTag; 73 if (!qat_cookie) 74 return; 75 76 crp = qat_cookie->crp_op; 77 78 qat_dsession = crypto_get_driver_session(crp->crp_session); 79 qat_instance = qat_dsession->qatInstance; 80 81 status = qat_ocf_cookie_dma_post_sync(crp, pOpData); 82 if (CPA_STATUS_SUCCESS != status) { 83 rc = EIO; 84 goto exit; 85 } 86 87 status = qat_ocf_cookie_dma_unload(crp, pOpData); 88 if (CPA_STATUS_SUCCESS != status) { 89 rc = EIO; 90 goto exit; 91 } 92 93 /* Verify result */ 94 if (CPA_STATUS_SUCCESS != result) { 95 rc = EBADMSG; 96 goto exit; 97 } 98 99 /* Verify digest by FW (GCM and CCM only) */ 100 if (CPA_TRUE != verifyResult) { 101 rc = EBADMSG; 102 goto exit; 103 } 104 105 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 106 qat_session = &qat_dsession->encSession; 107 else 108 qat_session = &qat_dsession->decSession; 109 110 /* Copy back digest result if it's stored in separated buffer */ 111 if (pOpData->digestResult && qat_session->authLen > 0) { 112 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { 113 char icv[QAT_OCF_MAX_DIGEST] = { 0 }; 114 crypto_copydata(crp, 115 crp->crp_digest_start, 116 qat_session->authLen, 117 icv); 118 if (timingsafe_bcmp(icv, 119 qat_cookie->qat_ocf_digest, 120 qat_session->authLen) != 0) { 121 rc = EBADMSG; 122 goto exit; 123 } 124 } else { 125 crypto_copyback(crp, 126 crp->crp_digest_start, 127 qat_session->authLen, 128 qat_cookie->qat_ocf_digest); 129 } 130 } 131 132 exit: 133 qat_ocf_cookie_free(qat_instance, qat_cookie); 134 crp->crp_etype = rc; 135 crypto_done(crp); 136 137 return; 138 } 139 140 static inline CpaPhysicalAddr 141 qatVirtToPhys(void *virtAddr) 142 { 143 return (CpaPhysicalAddr)vtophys(virtAddr); 144 } 145 146 static int 147 qat_ocf_probesession(device_t dev, const struct crypto_session_params *csp) 148 { 149 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 150 0) { 151 return EINVAL; 152 } 153 154 switch (csp->csp_mode) { 155 case CSP_MODE_CIPHER: 156 switch (csp->csp_cipher_alg) { 157 case CRYPTO_AES_CBC: 158 case CRYPTO_AES_ICM: 159 if (csp->csp_ivlen != AES_BLOCK_LEN) 160 return EINVAL; 161 break; 162 case CRYPTO_AES_XTS: 163 if (csp->csp_ivlen != AES_XTS_IV_LEN) 164 return EINVAL; 165 break; 166 default: 167 return EINVAL; 168 } 169 break; 170 case CSP_MODE_DIGEST: 171 switch (csp->csp_auth_alg) { 172 case CRYPTO_SHA1: 173 case CRYPTO_SHA1_HMAC: 174 case CRYPTO_SHA2_256: 175 case CRYPTO_SHA2_256_HMAC: 176 case CRYPTO_SHA2_384: 177 case CRYPTO_SHA2_384_HMAC: 178 case CRYPTO_SHA2_512: 179 case CRYPTO_SHA2_512_HMAC: 180 break; 181 case CRYPTO_AES_NIST_GMAC: 182 if (csp->csp_ivlen != AES_GCM_IV_LEN) 183 return EINVAL; 184 break; 185 default: 186 return EINVAL; 187 } 188 break; 189 case CSP_MODE_AEAD: 190 switch (csp->csp_cipher_alg) { 191 case CRYPTO_AES_NIST_GCM_16: 192 if (csp->csp_ivlen != AES_GCM_IV_LEN) 193 return EINVAL; 194 break; 195 default: 196 return EINVAL; 197 } 198 break; 199 case CSP_MODE_ETA: 200 switch (csp->csp_auth_alg) { 201 case CRYPTO_SHA1_HMAC: 202 case CRYPTO_SHA2_256_HMAC: 203 case CRYPTO_SHA2_384_HMAC: 204 case CRYPTO_SHA2_512_HMAC: 205 switch (csp->csp_cipher_alg) { 206 case CRYPTO_AES_CBC: 207 case CRYPTO_AES_ICM: 208 if (csp->csp_ivlen != AES_BLOCK_LEN) 209 return EINVAL; 210 break; 211 case CRYPTO_AES_XTS: 212 if (csp->csp_ivlen != AES_XTS_IV_LEN) 213 return EINVAL; 214 break; 215 default: 216 return EINVAL; 217 } 218 break; 219 default: 220 return EINVAL; 221 } 222 break; 223 default: 224 return EINVAL; 225 } 226 227 return CRYPTODEV_PROBE_HARDWARE; 228 } 229 230 static CpaStatus 231 qat_ocf_session_init(device_t dev, 232 struct cryptop *crp, 233 struct qat_ocf_instance *qat_instance, 234 struct qat_ocf_session *qat_ssession) 235 { 236 CpaStatus status = CPA_STATUS_SUCCESS; 237 /* Crytpodev structures */ 238 crypto_session_t cses; 239 const struct crypto_session_params *csp; 240 /* DP API Session configuration */ 241 CpaCySymSessionSetupData sessionSetupData = { 0 }; 242 CpaCySymSessionCtx sessionCtx = NULL; 243 Cpa32U sessionCtxSize = 0; 244 245 cses = crp->crp_session; 246 if (NULL == cses) { 247 device_printf(dev, "no crypto session in cryptodev request\n"); 248 return CPA_STATUS_FAIL; 249 } 250 251 csp = crypto_get_params(cses); 252 if (NULL == csp) { 253 device_printf(dev, "no session in cryptodev session\n"); 254 return CPA_STATUS_FAIL; 255 } 256 257 /* Common fields */ 258 sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH; 259 /* Cipher key */ 260 if (crp->crp_cipher_key) 261 sessionSetupData.cipherSetupData.pCipherKey = 262 crp->crp_cipher_key; 263 else 264 sessionSetupData.cipherSetupData.pCipherKey = 265 csp->csp_cipher_key; 266 sessionSetupData.cipherSetupData.cipherKeyLenInBytes = 267 csp->csp_cipher_klen; 268 269 /* Auth key */ 270 if (crp->crp_auth_key) 271 sessionSetupData.hashSetupData.authModeSetupData.authKey = 272 crp->crp_auth_key; 273 else 274 sessionSetupData.hashSetupData.authModeSetupData.authKey = 275 csp->csp_auth_key; 276 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = 277 csp->csp_auth_klen; 278 279 qat_ssession->aadLen = crp->crp_aad_length; 280 if (CPA_TRUE == is_sep_aad_supported(csp)) 281 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes = 282 crp->crp_aad_length; 283 else 284 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes = 285 0; 286 287 /* Just setup algorithm - regardless of mode */ 288 if (csp->csp_cipher_alg) { 289 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; 290 291 switch (csp->csp_cipher_alg) { 292 case CRYPTO_AES_CBC: 293 sessionSetupData.cipherSetupData.cipherAlgorithm = 294 CPA_CY_SYM_CIPHER_AES_CBC; 295 break; 296 case CRYPTO_AES_ICM: 297 sessionSetupData.cipherSetupData.cipherAlgorithm = 298 CPA_CY_SYM_CIPHER_AES_CTR; 299 break; 300 case CRYPTO_AES_XTS: 301 sessionSetupData.cipherSetupData.cipherAlgorithm = 302 CPA_CY_SYM_CIPHER_AES_XTS; 303 break; 304 case CRYPTO_AES_NIST_GCM_16: 305 sessionSetupData.cipherSetupData.cipherAlgorithm = 306 CPA_CY_SYM_CIPHER_AES_GCM; 307 sessionSetupData.hashSetupData.hashAlgorithm = 308 CPA_CY_SYM_HASH_AES_GCM; 309 sessionSetupData.hashSetupData.hashMode = 310 CPA_CY_SYM_HASH_MODE_AUTH; 311 break; 312 default: 313 device_printf(dev, 314 "cipher_alg: %d not supported\n", 315 csp->csp_cipher_alg); 316 status = CPA_STATUS_UNSUPPORTED; 317 goto fail; 318 } 319 } 320 321 if (csp->csp_auth_alg) { 322 switch (csp->csp_auth_alg) { 323 case CRYPTO_SHA1_HMAC: 324 sessionSetupData.hashSetupData.hashAlgorithm = 325 CPA_CY_SYM_HASH_SHA1; 326 sessionSetupData.hashSetupData.hashMode = 327 CPA_CY_SYM_HASH_MODE_AUTH; 328 break; 329 case CRYPTO_SHA1: 330 sessionSetupData.hashSetupData.hashAlgorithm = 331 CPA_CY_SYM_HASH_SHA1; 332 sessionSetupData.hashSetupData.hashMode = 333 CPA_CY_SYM_HASH_MODE_PLAIN; 334 break; 335 336 case CRYPTO_SHA2_256_HMAC: 337 sessionSetupData.hashSetupData.hashAlgorithm = 338 CPA_CY_SYM_HASH_SHA256; 339 sessionSetupData.hashSetupData.hashMode = 340 CPA_CY_SYM_HASH_MODE_AUTH; 341 break; 342 case CRYPTO_SHA2_256: 343 sessionSetupData.hashSetupData.hashAlgorithm = 344 CPA_CY_SYM_HASH_SHA256; 345 sessionSetupData.hashSetupData.hashMode = 346 CPA_CY_SYM_HASH_MODE_PLAIN; 347 break; 348 349 case CRYPTO_SHA2_224_HMAC: 350 sessionSetupData.hashSetupData.hashAlgorithm = 351 CPA_CY_SYM_HASH_SHA224; 352 sessionSetupData.hashSetupData.hashMode = 353 CPA_CY_SYM_HASH_MODE_AUTH; 354 break; 355 case CRYPTO_SHA2_224: 356 sessionSetupData.hashSetupData.hashAlgorithm = 357 CPA_CY_SYM_HASH_SHA224; 358 sessionSetupData.hashSetupData.hashMode = 359 CPA_CY_SYM_HASH_MODE_PLAIN; 360 break; 361 362 case CRYPTO_SHA2_384_HMAC: 363 sessionSetupData.hashSetupData.hashAlgorithm = 364 CPA_CY_SYM_HASH_SHA384; 365 sessionSetupData.hashSetupData.hashMode = 366 CPA_CY_SYM_HASH_MODE_AUTH; 367 break; 368 case CRYPTO_SHA2_384: 369 sessionSetupData.hashSetupData.hashAlgorithm = 370 CPA_CY_SYM_HASH_SHA384; 371 sessionSetupData.hashSetupData.hashMode = 372 CPA_CY_SYM_HASH_MODE_PLAIN; 373 break; 374 375 case CRYPTO_SHA2_512_HMAC: 376 sessionSetupData.hashSetupData.hashAlgorithm = 377 CPA_CY_SYM_HASH_SHA512; 378 sessionSetupData.hashSetupData.hashMode = 379 CPA_CY_SYM_HASH_MODE_AUTH; 380 break; 381 case CRYPTO_SHA2_512: 382 sessionSetupData.hashSetupData.hashAlgorithm = 383 CPA_CY_SYM_HASH_SHA512; 384 sessionSetupData.hashSetupData.hashMode = 385 CPA_CY_SYM_HASH_MODE_PLAIN; 386 break; 387 case CRYPTO_AES_NIST_GMAC: 388 sessionSetupData.hashSetupData.hashAlgorithm = 389 CPA_CY_SYM_HASH_AES_GMAC; 390 break; 391 default: 392 status = CPA_STATUS_UNSUPPORTED; 393 goto fail; 394 } 395 } /* csp->csp_auth_alg */ 396 397 /* Setting digest-length if no cipher-only mode is set */ 398 if (csp->csp_mode != CSP_MODE_CIPHER) { 399 lac_sym_qat_hash_defs_t *pHashDefsInfo = NULL; 400 if (csp->csp_auth_mlen) { 401 sessionSetupData.hashSetupData.digestResultLenInBytes = 402 csp->csp_auth_mlen; 403 qat_ssession->authLen = csp->csp_auth_mlen; 404 } else { 405 LacSymQat_HashDefsLookupGet( 406 qat_instance->cyInstHandle, 407 sessionSetupData.hashSetupData.hashAlgorithm, 408 &pHashDefsInfo); 409 if (NULL == pHashDefsInfo) { 410 device_printf( 411 dev, 412 "unable to find corresponding hash data\n"); 413 status = CPA_STATUS_UNSUPPORTED; 414 goto fail; 415 } 416 sessionSetupData.hashSetupData.digestResultLenInBytes = 417 pHashDefsInfo->algInfo->digestLength; 418 qat_ssession->authLen = 419 pHashDefsInfo->algInfo->digestLength; 420 } 421 sessionSetupData.verifyDigest = CPA_FALSE; 422 } 423 424 switch (csp->csp_mode) { 425 case CSP_MODE_AEAD: 426 sessionSetupData.symOperation = 427 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 428 /* Place the digest result in a buffer unrelated to srcBuffer */ 429 sessionSetupData.digestIsAppended = CPA_TRUE; 430 /* For GCM and CCM driver forces to verify digest on HW */ 431 sessionSetupData.verifyDigest = CPA_TRUE; 432 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 433 sessionSetupData.cipherSetupData.cipherDirection = 434 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 435 sessionSetupData.algChainOrder = 436 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 437 } else { 438 sessionSetupData.cipherSetupData.cipherDirection = 439 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 440 sessionSetupData.algChainOrder = 441 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 442 } 443 break; 444 case CSP_MODE_ETA: 445 sessionSetupData.symOperation = 446 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 447 /* Place the digest result in a buffer unrelated to srcBuffer */ 448 sessionSetupData.digestIsAppended = CPA_FALSE; 449 /* Due to FW limitation to verify only appended MACs */ 450 sessionSetupData.verifyDigest = CPA_FALSE; 451 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 452 sessionSetupData.cipherSetupData.cipherDirection = 453 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 454 sessionSetupData.algChainOrder = 455 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 456 } else { 457 sessionSetupData.cipherSetupData.cipherDirection = 458 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 459 sessionSetupData.algChainOrder = 460 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 461 } 462 break; 463 case CSP_MODE_CIPHER: 464 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 465 sessionSetupData.cipherSetupData.cipherDirection = 466 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 467 } else { 468 sessionSetupData.cipherSetupData.cipherDirection = 469 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 470 } 471 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; 472 break; 473 case CSP_MODE_DIGEST: 474 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; 475 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { 476 sessionSetupData.symOperation = 477 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 478 /* GMAC is always encrypt */ 479 sessionSetupData.cipherSetupData.cipherDirection = 480 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 481 sessionSetupData.algChainOrder = 482 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 483 sessionSetupData.cipherSetupData.cipherAlgorithm = 484 CPA_CY_SYM_CIPHER_AES_GCM; 485 sessionSetupData.hashSetupData.hashAlgorithm = 486 CPA_CY_SYM_HASH_AES_GMAC; 487 sessionSetupData.hashSetupData.hashMode = 488 CPA_CY_SYM_HASH_MODE_AUTH; 489 /* Same key for cipher and auth */ 490 sessionSetupData.cipherSetupData.pCipherKey = 491 csp->csp_auth_key; 492 sessionSetupData.cipherSetupData.cipherKeyLenInBytes = 493 csp->csp_auth_klen; 494 /* Generated GMAC stored in separated buffer */ 495 sessionSetupData.digestIsAppended = CPA_FALSE; 496 /* Digest verification not allowed in GMAC case */ 497 sessionSetupData.verifyDigest = CPA_FALSE; 498 /* No AAD allowed */ 499 sessionSetupData.hashSetupData.authModeSetupData 500 .aadLenInBytes = 0; 501 } else { 502 sessionSetupData.cipherSetupData.cipherDirection = 503 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 504 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; 505 sessionSetupData.digestIsAppended = CPA_FALSE; 506 } 507 break; 508 default: 509 device_printf(dev, 510 "%s: unhandled crypto algorithm %d, %d\n", 511 __func__, 512 csp->csp_cipher_alg, 513 csp->csp_auth_alg); 514 status = CPA_STATUS_FAIL; 515 goto fail; 516 } 517 518 /* Extracting session size */ 519 status = cpaCySymSessionCtxGetSize(qat_instance->cyInstHandle, 520 &sessionSetupData, 521 &sessionCtxSize); 522 if (CPA_STATUS_SUCCESS != status) { 523 device_printf(dev, "unable to get session size\n"); 524 goto fail; 525 } 526 527 /* Allocating contiguous memory for session */ 528 sessionCtx = contigmalloc(sessionCtxSize, 529 M_QAT_OCF, 530 M_NOWAIT, 531 0, 532 ~1UL, 533 1 << (bsrl(sessionCtxSize - 1) + 1), 534 0); 535 if (NULL == sessionCtx) { 536 device_printf(dev, "unable to allocate memory for session\n"); 537 status = CPA_STATUS_RESOURCE; 538 goto fail; 539 } 540 541 status = cpaCySymDpInitSession(qat_instance->cyInstHandle, 542 &sessionSetupData, 543 sessionCtx); 544 if (CPA_STATUS_SUCCESS != status) { 545 device_printf(dev, "session initialization failed\n"); 546 goto fail; 547 } 548 549 /* NOTE: lets keep double session (both directions) approach to overcome 550 * lack of direction update in FBSD QAT. 551 */ 552 qat_ssession->sessionCtx = sessionCtx; 553 qat_ssession->sessionCtxSize = sessionCtxSize; 554 555 return CPA_STATUS_SUCCESS; 556 557 fail: 558 /* Release resources if any */ 559 if (sessionCtx) 560 contigfree(sessionCtx, sessionCtxSize, M_QAT_OCF); 561 562 return status; 563 } 564 565 static int 566 qat_ocf_newsession(device_t dev, 567 crypto_session_t cses, 568 const struct crypto_session_params *csp) 569 { 570 /* Cryptodev QAT structures */ 571 struct qat_ocf_softc *qat_softc; 572 struct qat_ocf_dsession *qat_dsession; 573 struct qat_ocf_instance *qat_instance; 574 u_int cpu_id = PCPU_GET(cpuid); 575 576 /* Create cryptodev session */ 577 qat_softc = device_get_softc(dev); 578 qat_instance = 579 &qat_softc->cyInstHandles[cpu_id % qat_softc->numCyInstances]; 580 qat_dsession = crypto_get_driver_session(cses); 581 if (NULL == qat_dsession) { 582 device_printf(dev, "Unable to create new session\n"); 583 return (EINVAL); 584 } 585 586 /* Add only instance at this point remaining operations moved to 587 * lazy session init */ 588 qat_dsession->qatInstance = qat_instance; 589 590 return 0; 591 } 592 593 static CpaStatus 594 qat_ocf_remove_session(device_t dev, 595 CpaInstanceHandle cyInstHandle, 596 struct qat_ocf_session *qat_session) 597 { 598 CpaStatus status = CPA_STATUS_SUCCESS; 599 600 if (NULL == qat_session->sessionCtx) 601 return CPA_STATUS_SUCCESS; 602 603 /* User callback is executed right before decrementing pending 604 * callback atomic counter. To avoid removing session rejection 605 * we have to wait a very short while for counter update 606 * after call back execution. */ 607 status = qat_ocf_wait_for_session(qat_session->sessionCtx, 608 QAT_OCF_SESSION_WAIT_TIMEOUT_MS); 609 if (CPA_STATUS_SUCCESS != status) { 610 device_printf(dev, "waiting for session un-busy failed\n"); 611 return CPA_STATUS_FAIL; 612 } 613 614 status = cpaCySymDpRemoveSession(cyInstHandle, qat_session->sessionCtx); 615 if (CPA_STATUS_SUCCESS != status) { 616 device_printf(dev, "error while removing session\n"); 617 return CPA_STATUS_FAIL; 618 } 619 620 explicit_bzero(qat_session->sessionCtx, qat_session->sessionCtxSize); 621 contigfree(qat_session->sessionCtx, 622 qat_session->sessionCtxSize, 623 M_QAT_OCF); 624 qat_session->sessionCtx = NULL; 625 qat_session->sessionCtxSize = 0; 626 627 return CPA_STATUS_SUCCESS; 628 } 629 630 static void 631 qat_ocf_freesession(device_t dev, crypto_session_t cses) 632 { 633 CpaStatus status = CPA_STATUS_SUCCESS; 634 struct qat_ocf_dsession *qat_dsession = NULL; 635 struct qat_ocf_instance *qat_instance = NULL; 636 637 qat_dsession = crypto_get_driver_session(cses); 638 qat_instance = qat_dsession->qatInstance; 639 mtx_lock(&qat_instance->cyInstMtx); 640 status = qat_ocf_remove_session(dev, 641 qat_dsession->qatInstance->cyInstHandle, 642 &qat_dsession->encSession); 643 if (CPA_STATUS_SUCCESS != status) 644 device_printf(dev, "unable to remove encrypt session\n"); 645 status = qat_ocf_remove_session(dev, 646 qat_dsession->qatInstance->cyInstHandle, 647 &qat_dsession->decSession); 648 if (CPA_STATUS_SUCCESS != status) 649 device_printf(dev, "unable to remove decrypt session\n"); 650 mtx_unlock(&qat_instance->cyInstMtx); 651 } 652 653 /* QAT GCM/CCM FW API are only algorithms which support separated AAD. */ 654 static CpaStatus 655 qat_ocf_load_aad_gcm(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 656 { 657 CpaCySymDpOpData *pOpData; 658 659 pOpData = &qat_cookie->pOpdata; 660 661 if (NULL != crp->crp_aad) 662 memcpy(qat_cookie->qat_ocf_gcm_aad, 663 crp->crp_aad, 664 crp->crp_aad_length); 665 else 666 crypto_copydata(crp, 667 crp->crp_aad_start, 668 crp->crp_aad_length, 669 qat_cookie->qat_ocf_gcm_aad); 670 671 pOpData->pAdditionalAuthData = qat_cookie->qat_ocf_gcm_aad; 672 pOpData->additionalAuthData = qat_cookie->qat_ocf_gcm_aad_paddr; 673 674 return CPA_STATUS_SUCCESS; 675 } 676 677 static CpaStatus 678 qat_ocf_load_aad(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 679 { 680 CpaStatus status = CPA_STATUS_SUCCESS; 681 const struct crypto_session_params *csp; 682 CpaCySymDpOpData *pOpData; 683 struct qat_ocf_load_cb_arg args; 684 685 pOpData = &qat_cookie->pOpdata; 686 pOpData->pAdditionalAuthData = NULL; 687 pOpData->additionalAuthData = 0UL; 688 689 if (crp->crp_aad_length == 0) 690 return CPA_STATUS_SUCCESS; 691 692 if (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX) 693 return CPA_STATUS_FAIL; 694 695 csp = crypto_get_params(crp->crp_session); 696 697 /* Handle GCM/CCM case */ 698 if (CPA_TRUE == is_sep_aad_supported(csp)) 699 return qat_ocf_load_aad_gcm(crp, qat_cookie); 700 701 if (NULL == crp->crp_aad) { 702 /* AAD already embedded in source buffer */ 703 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 704 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start; 705 706 pOpData->messageLenToHashInBytes = 707 crp->crp_aad_length + crp->crp_payload_length; 708 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start; 709 710 return CPA_STATUS_SUCCESS; 711 } 712 713 /* Separated AAD not supported by QAT - lets place the content 714 * of ADD buffer at the very beginning of source SGL */ 715 args.crp_op = crp; 716 args.qat_cookie = qat_cookie; 717 args.pOpData = pOpData; 718 args.error = 0; 719 status = bus_dmamap_load(qat_cookie->gcm_aad_dma_mem.dma_tag, 720 qat_cookie->gcm_aad_dma_mem.dma_map, 721 crp->crp_aad, 722 crp->crp_aad_length, 723 qat_ocf_crypto_load_aadbuf_cb, 724 &args, 725 BUS_DMA_NOWAIT); 726 qat_cookie->is_sep_aad_used = CPA_TRUE; 727 728 /* Right after this step we have AAD placed in the first flat buffer 729 * in source SGL */ 730 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 731 pOpData->cryptoStartSrcOffsetInBytes = 732 crp->crp_aad_length + crp->crp_aad_start + crp->crp_payload_start; 733 734 pOpData->messageLenToHashInBytes = 735 crp->crp_aad_length + crp->crp_payload_length; 736 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start; 737 738 return status; 739 } 740 741 static CpaStatus 742 qat_ocf_load(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 743 { 744 CpaStatus status = CPA_STATUS_SUCCESS; 745 CpaCySymDpOpData *pOpData; 746 struct qat_ocf_load_cb_arg args; 747 /* cryptodev internals */ 748 const struct crypto_session_params *csp; 749 750 pOpData = &qat_cookie->pOpdata; 751 752 csp = crypto_get_params(crp->crp_session); 753 754 /* Load IV buffer if present */ 755 if (csp->csp_ivlen > 0) { 756 memset(qat_cookie->qat_ocf_iv_buf, 757 0, 758 sizeof(qat_cookie->qat_ocf_iv_buf)); 759 crypto_read_iv(crp, qat_cookie->qat_ocf_iv_buf); 760 pOpData->iv = qat_cookie->qat_ocf_iv_buf_paddr; 761 pOpData->pIv = qat_cookie->qat_ocf_iv_buf; 762 pOpData->ivLenInBytes = csp->csp_ivlen; 763 } 764 765 /* GCM/CCM - load AAD to separated buffer 766 * AES+SHA - load AAD to first flat in SGL */ 767 status = qat_ocf_load_aad(crp, qat_cookie); 768 if (CPA_STATUS_SUCCESS != status) 769 goto fail; 770 771 /* Load source buffer */ 772 args.crp_op = crp; 773 args.qat_cookie = qat_cookie; 774 args.pOpData = pOpData; 775 args.error = 0; 776 status = bus_dmamap_load_crp_buffer(qat_cookie->src_dma_mem.dma_tag, 777 qat_cookie->src_dma_mem.dma_map, 778 &crp->crp_buf, 779 qat_ocf_crypto_load_buf_cb, 780 &args, 781 BUS_DMA_NOWAIT); 782 if (CPA_STATUS_SUCCESS != status) 783 goto fail; 784 pOpData->srcBuffer = qat_cookie->src_buffer_list_paddr; 785 pOpData->srcBufferLen = CPA_DP_BUFLIST; 786 787 /* Load destination buffer */ 788 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 789 status = 790 bus_dmamap_load_crp_buffer(qat_cookie->dst_dma_mem.dma_tag, 791 qat_cookie->dst_dma_mem.dma_map, 792 &crp->crp_obuf, 793 qat_ocf_crypto_load_obuf_cb, 794 &args, 795 BUS_DMA_NOWAIT); 796 if (CPA_STATUS_SUCCESS != status) 797 goto fail; 798 pOpData->dstBuffer = qat_cookie->dst_buffer_list_paddr; 799 pOpData->dstBufferLen = CPA_DP_BUFLIST; 800 } else { 801 pOpData->dstBuffer = pOpData->srcBuffer; 802 pOpData->dstBufferLen = pOpData->srcBufferLen; 803 } 804 805 if (CPA_TRUE == is_use_sep_digest(csp)) 806 pOpData->digestResult = qat_cookie->qat_ocf_digest_paddr; 807 else 808 pOpData->digestResult = 0UL; 809 810 /* GMAC - aka zero length buffer */ 811 if (CPA_TRUE == is_gmac_exception(csp)) 812 pOpData->messageLenToCipherInBytes = 0; 813 814 fail: 815 return status; 816 } 817 818 static int 819 qat_ocf_check_input(device_t dev, struct cryptop *crp) 820 { 821 const struct crypto_session_params *csp; 822 csp = crypto_get_params(crp->crp_session); 823 824 if (crypto_buffer_len(&crp->crp_buf) > QAT_OCF_MAX_LEN) 825 return E2BIG; 826 827 if (CPA_TRUE == is_sep_aad_supported(csp) && 828 (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX)) 829 return EBADMSG; 830 831 return 0; 832 } 833 834 static int 835 qat_ocf_process(device_t dev, struct cryptop *crp, int hint) 836 { 837 CpaStatus status = CPA_STATUS_SUCCESS; 838 int rc = 0; 839 struct qat_ocf_dsession *qat_dsession = NULL; 840 struct qat_ocf_session *qat_session = NULL; 841 struct qat_ocf_instance *qat_instance = NULL; 842 CpaCySymDpOpData *pOpData = NULL; 843 struct qat_ocf_cookie *qat_cookie = NULL; 844 CpaBoolean memLoaded = CPA_FALSE; 845 846 rc = qat_ocf_check_input(dev, crp); 847 if (rc) 848 goto fail; 849 850 qat_dsession = crypto_get_driver_session(crp->crp_session); 851 852 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 853 qat_session = &qat_dsession->encSession; 854 else 855 qat_session = &qat_dsession->decSession; 856 qat_instance = qat_dsession->qatInstance; 857 858 status = qat_ocf_cookie_alloc(qat_instance, &qat_cookie); 859 if (CPA_STATUS_SUCCESS != status) { 860 rc = EAGAIN; 861 goto fail; 862 } 863 864 qat_cookie->crp_op = crp; 865 866 /* Common request fields */ 867 pOpData = &qat_cookie->pOpdata; 868 pOpData->instanceHandle = qat_instance->cyInstHandle; 869 pOpData->sessionCtx = NULL; 870 871 /* Cipher fields */ 872 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start; 873 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 874 /* Digest fields - any exceptions from this basic rules are covered 875 * in qat_ocf_load */ 876 pOpData->hashStartSrcOffsetInBytes = crp->crp_payload_start; 877 pOpData->messageLenToHashInBytes = crp->crp_payload_length; 878 879 status = qat_ocf_load(crp, qat_cookie); 880 if (CPA_STATUS_SUCCESS != status) { 881 device_printf(dev, 882 "unable to load OCF buffers to QAT DMA " 883 "transaction\n"); 884 rc = EIO; 885 goto fail; 886 } 887 memLoaded = CPA_TRUE; 888 889 status = qat_ocf_cookie_dma_pre_sync(crp, pOpData); 890 if (CPA_STATUS_SUCCESS != status) { 891 device_printf(dev, "unable to sync DMA buffers\n"); 892 rc = EIO; 893 goto fail; 894 } 895 896 mtx_lock(&qat_instance->cyInstMtx); 897 /* Session initialization at the first request. It's done 898 * in such way to overcome missing QAT specific session data 899 * such like AAD length and limited possibility to update 900 * QAT session while handling traffic. 901 */ 902 if (NULL == qat_session->sessionCtx) { 903 status = 904 qat_ocf_session_init(dev, crp, qat_instance, qat_session); 905 if (CPA_STATUS_SUCCESS != status) { 906 mtx_unlock(&qat_instance->cyInstMtx); 907 device_printf(dev, "unable to init session\n"); 908 rc = EIO; 909 goto fail; 910 } 911 } else { 912 status = qat_ocf_handle_session_update(qat_dsession, crp); 913 if (CPA_STATUS_RESOURCE == status) { 914 mtx_unlock(&qat_instance->cyInstMtx); 915 rc = EAGAIN; 916 goto fail; 917 } else if (CPA_STATUS_SUCCESS != status) { 918 mtx_unlock(&qat_instance->cyInstMtx); 919 rc = EIO; 920 goto fail; 921 } 922 } 923 pOpData->sessionCtx = qat_session->sessionCtx; 924 status = cpaCySymDpEnqueueOp(pOpData, CPA_TRUE); 925 mtx_unlock(&qat_instance->cyInstMtx); 926 if (CPA_STATUS_SUCCESS != status) { 927 if (CPA_STATUS_RETRY == status) { 928 rc = EAGAIN; 929 goto fail; 930 } 931 device_printf(dev, 932 "unable to send request. Status: %d\n", 933 status); 934 rc = EIO; 935 goto fail; 936 } 937 938 return 0; 939 fail: 940 if (qat_cookie) { 941 if (memLoaded) 942 qat_ocf_cookie_dma_unload(crp, pOpData); 943 qat_ocf_cookie_free(qat_instance, qat_cookie); 944 } 945 crp->crp_etype = rc; 946 crypto_done(crp); 947 948 return 0; 949 } 950 951 static void 952 qat_ocf_identify(driver_t *drv, device_t parent) 953 { 954 if (device_find_child(parent, "qat_ocf", -1) == NULL && 955 BUS_ADD_CHILD(parent, 200, "qat_ocf", -1) == 0) 956 device_printf(parent, "qat_ocf: could not attach!"); 957 } 958 959 static int 960 qat_ocf_probe(device_t dev) 961 { 962 device_set_desc(dev, "QAT engine"); 963 return (BUS_PROBE_NOWILDCARD); 964 } 965 966 static CpaStatus 967 qat_ocf_get_irq_instances(CpaInstanceHandle *cyInstHandles, 968 Cpa16U cyInstHandlesSize, 969 Cpa16U *foundInstances) 970 { 971 CpaStatus status = CPA_STATUS_SUCCESS; 972 icp_accel_dev_t **pAdfInsts = NULL; 973 icp_accel_dev_t *dev_addr = NULL; 974 sal_t *baseAddr = NULL; 975 sal_list_t *listTemp = NULL; 976 CpaInstanceHandle cyInstHandle; 977 CpaInstanceInfo2 info; 978 Cpa16U numDevices; 979 Cpa32U instCtr = 0; 980 Cpa32U i; 981 982 /* Get the number of devices */ 983 status = icp_amgr_getNumInstances(&numDevices); 984 if (CPA_STATUS_SUCCESS != status) 985 return status; 986 987 /* Allocate memory to store addr of accel_devs */ 988 pAdfInsts = 989 malloc(numDevices * sizeof(icp_accel_dev_t *), M_QAT_OCF, M_WAITOK); 990 991 /* Get ADF to return all accel_devs that support either 992 * symmetric or asymmetric crypto */ 993 status = icp_amgr_getAllAccelDevByCapabilities( 994 (ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC), pAdfInsts, &numDevices); 995 if (CPA_STATUS_SUCCESS != status) { 996 free(pAdfInsts, M_QAT_OCF); 997 return status; 998 } 999 1000 for (i = 0; i < numDevices; i++) { 1001 dev_addr = (icp_accel_dev_t *)pAdfInsts[i]; 1002 baseAddr = dev_addr->pSalHandle; 1003 if (NULL == baseAddr) 1004 continue; 1005 listTemp = baseAddr->sym_services; 1006 while (NULL != listTemp) { 1007 cyInstHandle = SalList_getObject(listTemp); 1008 status = cpaCyInstanceGetInfo2(cyInstHandle, &info); 1009 if (CPA_STATUS_SUCCESS != status) 1010 continue; 1011 listTemp = SalList_next(listTemp); 1012 if (CPA_TRUE == info.isPolled) 1013 continue; 1014 if (instCtr >= cyInstHandlesSize) 1015 break; 1016 cyInstHandles[instCtr++] = cyInstHandle; 1017 } 1018 } 1019 free(pAdfInsts, M_QAT_OCF); 1020 *foundInstances = instCtr; 1021 1022 return CPA_STATUS_SUCCESS; 1023 } 1024 1025 static CpaStatus 1026 qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev) 1027 { 1028 CpaStatus status = CPA_STATUS_SUCCESS; 1029 Cpa16U numInstances = 0; 1030 CpaInstanceHandle cyInstHandles[QAT_OCF_MAX_INSTANCES] = { 0 }; 1031 CpaInstanceHandle cyInstHandle = NULL; 1032 Cpa32U startedInstances = 0; 1033 Cpa32U i; 1034 1035 qat_softc->numCyInstances = 0; 1036 status = qat_ocf_get_irq_instances(cyInstHandles, 1037 QAT_OCF_MAX_INSTANCES, 1038 &numInstances); 1039 if (CPA_STATUS_SUCCESS != status) 1040 return status; 1041 if (0 == numInstances) 1042 return CPA_STATUS_RESOURCE; 1043 1044 for (i = 0; i < numInstances; i++) { 1045 struct qat_ocf_instance *qat_ocf_instance; 1046 1047 cyInstHandle = cyInstHandles[i]; 1048 if (!cyInstHandle) 1049 continue; 1050 1051 /* Starting instance */ 1052 status = cpaCyStartInstance(cyInstHandle); 1053 if (CPA_STATUS_SUCCESS != status) { 1054 device_printf(qat_softc->sc_dev, 1055 "unable to get start instance\n"); 1056 continue; 1057 } 1058 1059 status = 1060 cpaCySetAddressTranslation(cyInstHandle, qatVirtToPhys); 1061 if (CPA_STATUS_SUCCESS != status) { 1062 device_printf(qat_softc->sc_dev, 1063 "unable to add virt to phys callback"); 1064 goto fail; 1065 } 1066 1067 status = cpaCySymDpRegCbFunc(cyInstHandle, symDpCallback); 1068 if (CPA_STATUS_SUCCESS != status) { 1069 device_printf(qat_softc->sc_dev, 1070 "unable to add user callback\n"); 1071 goto fail; 1072 } 1073 1074 qat_ocf_instance = &qat_softc->cyInstHandles[startedInstances]; 1075 qat_ocf_instance->cyInstHandle = cyInstHandle; 1076 mtx_init(&qat_ocf_instance->cyInstMtx, 1077 "Instance MTX", 1078 NULL, 1079 MTX_DEF); 1080 1081 /* Initialize cookie pool */ 1082 status = qat_ocf_cookie_pool_init(qat_ocf_instance, dev); 1083 if (CPA_STATUS_SUCCESS != status) { 1084 device_printf(qat_softc->sc_dev, 1085 "unable to create cookie pool\n"); 1086 goto fail; 1087 } 1088 1089 qat_ocf_instance->driver_id = qat_softc->cryptodev_id; 1090 1091 startedInstances++; 1092 continue; 1093 fail: 1094 /* Stop instance */ 1095 status = cpaCyStopInstance(cyInstHandle); 1096 if (CPA_STATUS_SUCCESS != status) 1097 device_printf(qat_softc->sc_dev, 1098 "unable to stop the instance\n"); 1099 continue; 1100 } 1101 qat_softc->numCyInstances = startedInstances; 1102 1103 /* Success if at least one instance has been set */ 1104 if (!qat_softc->numCyInstances) 1105 return CPA_STATUS_FAIL; 1106 1107 return CPA_STATUS_SUCCESS; 1108 } 1109 1110 static CpaStatus 1111 qat_ocf_stop_instances(struct qat_ocf_softc *qat_softc) 1112 { 1113 CpaStatus status = CPA_STATUS_SUCCESS; 1114 int i; 1115 1116 for (i = 0; i < qat_softc->numCyInstances; i++) { 1117 struct qat_ocf_instance *qat_instance; 1118 1119 qat_instance = &qat_softc->cyInstHandles[i]; 1120 status = cpaCyStopInstance(qat_instance->cyInstHandle); 1121 if (CPA_STATUS_SUCCESS != status) { 1122 pr_err("QAT: stopping instance id: %d failed\n", i); 1123 mtx_unlock(&qat_instance->cyInstMtx); 1124 continue; 1125 } 1126 qat_ocf_cookie_pool_deinit(qat_instance); 1127 mtx_destroy(&qat_instance->cyInstMtx); 1128 } 1129 1130 return status; 1131 } 1132 1133 static int 1134 qat_ocf_attach(device_t dev) 1135 { 1136 int status; 1137 struct qat_ocf_softc *qat_softc; 1138 int32_t cryptodev_id; 1139 1140 qat_softc = device_get_softc(dev); 1141 qat_softc->sc_dev = dev; 1142 1143 cryptodev_id = crypto_get_driverid(dev, 1144 sizeof(struct qat_ocf_dsession), 1145 CRYPTOCAP_F_HARDWARE); 1146 if (cryptodev_id < 0) { 1147 device_printf(dev, "cannot initialize!\n"); 1148 goto fail; 1149 } 1150 qat_softc->cryptodev_id = cryptodev_id; 1151 1152 /* Starting instances for OCF */ 1153 status = qat_ocf_start_instances(qat_softc, dev); 1154 if (status) { 1155 device_printf(dev, "no QAT IRQ instances available\n"); 1156 goto fail; 1157 } 1158 1159 return 0; 1160 fail: 1161 qat_ocf_detach(dev); 1162 1163 return (ENXIO); 1164 } 1165 1166 static int 1167 qat_ocf_detach(device_t dev) 1168 { 1169 struct qat_ocf_softc *qat_softc = NULL; 1170 CpaStatus cpaStatus; 1171 int status = 0; 1172 1173 qat_softc = device_get_softc(dev); 1174 1175 if (qat_softc->cryptodev_id >= 0) { 1176 status = crypto_unregister_all(qat_softc->cryptodev_id); 1177 if (status) 1178 device_printf(dev, 1179 "unable to unregister QAt backend\n"); 1180 } 1181 1182 /* Stop QAT instances */ 1183 cpaStatus = qat_ocf_stop_instances(qat_softc); 1184 if (CPA_STATUS_SUCCESS != cpaStatus) { 1185 device_printf(dev, "unable to stop instances\n"); 1186 status = EIO; 1187 } 1188 1189 return status; 1190 } 1191 1192 static device_method_t qat_ocf_methods[] = 1193 { DEVMETHOD(device_identify, qat_ocf_identify), 1194 DEVMETHOD(device_probe, qat_ocf_probe), 1195 DEVMETHOD(device_attach, qat_ocf_attach), 1196 DEVMETHOD(device_detach, qat_ocf_detach), 1197 1198 /* Cryptodev interface */ 1199 DEVMETHOD(cryptodev_probesession, qat_ocf_probesession), 1200 DEVMETHOD(cryptodev_newsession, qat_ocf_newsession), 1201 DEVMETHOD(cryptodev_freesession, qat_ocf_freesession), 1202 DEVMETHOD(cryptodev_process, qat_ocf_process), 1203 1204 DEVMETHOD_END }; 1205 1206 static driver_t qat_ocf_driver = { 1207 .name = "qat_ocf", 1208 .methods = qat_ocf_methods, 1209 .size = sizeof(struct qat_ocf_softc), 1210 }; 1211 1212 1213 DRIVER_MODULE_ORDERED(qat, 1214 nexus, 1215 qat_ocf_driver, 1216 NULL, 1217 NULL, 1218 SI_ORDER_ANY); 1219 MODULE_VERSION(qat, 1); 1220 MODULE_DEPEND(qat, qat_c62x, 1, 1, 1); 1221 MODULE_DEPEND(qat, qat_200xx, 1, 1, 1); 1222 MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1); 1223 MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1); 1224 MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1); 1225 MODULE_DEPEND(qat, crypto, 1, 1, 1); 1226 MODULE_DEPEND(qat, qat_common, 1, 1, 1); 1227 MODULE_DEPEND(qat, qat_api, 1, 1, 1); 1228 MODULE_DEPEND(qat, linuxkpi, 1, 1, 1); 1229