1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 /* System headers */
4 #include <sys/param.h>
5 #include <sys/systm.h>
6 #include <sys/bus.h>
7 #include <sys/cpu.h>
8 #include <sys/kernel.h>
9 #include <sys/mbuf.h>
10 #include <sys/module.h>
11 #include <sys/mutex.h>
12 #include <sys/sysctl.h>
13
14 /* Cryptodev headers */
15 #include <opencrypto/cryptodev.h>
16 #include "cryptodev_if.h"
17
18 /* QAT specific headers */
19 #include "cpa.h"
20 #include "cpa_cy_im.h"
21 #include "cpa_cy_sym_dp.h"
22 #include "adf_accel_devices.h"
23 #include "adf_common_drv.h"
24 #include "lac_sym_hash_defs.h"
25 #include "lac_sym_qat_hash_defs_lookup.h"
26
27 /* To get only IRQ instances */
28 #include "icp_accel_devices.h"
29 #include "icp_adf_accel_mgr.h"
30 #include "lac_sal_types.h"
31
32 /* To disable AEAD HW MAC verification */
33 #include "icp_sal_user.h"
34
35 /* QAT OCF specific headers */
36 #include "qat_ocf_mem_pool.h"
37 #include "qat_ocf_utils.h"
38
39 #define QAT_OCF_MAX_INSTANCES (256)
40 #define QAT_OCF_SESSION_WAIT_TIMEOUT_MS (1000)
41
42 MALLOC_DEFINE(M_QAT_OCF, "qat_ocf", "qat_ocf(4) memory allocations");
43
44 /* QAT OCF internal structures */
45 struct qat_ocf_softc {
46 device_t sc_dev;
47 struct sysctl_oid *rc;
48 uint32_t enabled;
49 int32_t cryptodev_id;
50 struct qat_ocf_instance cyInstHandles[QAT_OCF_MAX_INSTANCES];
51 int32_t numCyInstances;
52 };
53
54 /* Function definitions */
55 static void qat_ocf_freesession(device_t dev, crypto_session_t cses);
56 static int qat_ocf_probesession(device_t dev,
57 const struct crypto_session_params *csp);
58 static int qat_ocf_newsession(device_t dev,
59 crypto_session_t cses,
60 const struct crypto_session_params *csp);
61 static int qat_ocf_attach(device_t dev);
62 static int qat_ocf_detach(device_t dev);
63
64 static void
symDpCallback(CpaCySymDpOpData * pOpData,CpaStatus result,CpaBoolean verifyResult)65 symDpCallback(CpaCySymDpOpData *pOpData,
66 CpaStatus result,
67 CpaBoolean verifyResult)
68 {
69 struct qat_ocf_cookie *qat_cookie;
70 struct cryptop *crp;
71 struct qat_ocf_dsession *qat_dsession = NULL;
72 struct qat_ocf_session *qat_session = NULL;
73 struct qat_ocf_instance *qat_instance = NULL;
74 CpaStatus status;
75 int rc = 0;
76
77 qat_cookie = (struct qat_ocf_cookie *)pOpData->pCallbackTag;
78 if (!qat_cookie)
79 return;
80
81 crp = qat_cookie->crp_op;
82
83 qat_dsession = crypto_get_driver_session(crp->crp_session);
84 qat_instance = qat_dsession->qatInstance;
85
86 status = qat_ocf_cookie_dma_post_sync(crp, pOpData);
87 if (CPA_STATUS_SUCCESS != status) {
88 rc = EIO;
89 goto exit;
90 }
91
92 status = qat_ocf_cookie_dma_unload(crp, pOpData);
93 if (CPA_STATUS_SUCCESS != status) {
94 rc = EIO;
95 goto exit;
96 }
97
98 /* Verify result */
99 if (CPA_STATUS_SUCCESS != result) {
100 rc = EBADMSG;
101 goto exit;
102 }
103
104 /* Verify digest by FW (GCM and CCM only) */
105 if (CPA_TRUE != verifyResult) {
106 rc = EBADMSG;
107 goto exit;
108 }
109
110 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
111 qat_session = &qat_dsession->encSession;
112 else
113 qat_session = &qat_dsession->decSession;
114
115 /* Copy back digest result if it's stored in separated buffer */
116 if (pOpData->digestResult && qat_session->authLen > 0) {
117 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
118 char icv[QAT_OCF_MAX_DIGEST] = { 0 };
119 crypto_copydata(crp,
120 crp->crp_digest_start,
121 qat_session->authLen,
122 icv);
123 if (timingsafe_bcmp(icv,
124 qat_cookie->qat_ocf_digest,
125 qat_session->authLen) != 0) {
126 rc = EBADMSG;
127 goto exit;
128 }
129 } else {
130 crypto_copyback(crp,
131 crp->crp_digest_start,
132 qat_session->authLen,
133 qat_cookie->qat_ocf_digest);
134 }
135 }
136
137 exit:
138 qat_ocf_cookie_free(qat_instance, qat_cookie);
139 crp->crp_etype = rc;
140 crypto_done(crp);
141
142 return;
143 }
144
145 static inline CpaPhysicalAddr
qatVirtToPhys(void * virtAddr)146 qatVirtToPhys(void *virtAddr)
147 {
148 return (CpaPhysicalAddr)vtophys(virtAddr);
149 }
150
151 static int
qat_ocf_probesession(device_t dev,const struct crypto_session_params * csp)152 qat_ocf_probesession(device_t dev, const struct crypto_session_params *csp)
153 {
154 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
155 0) {
156 return EINVAL;
157 }
158
159 switch (csp->csp_mode) {
160 case CSP_MODE_CIPHER:
161 switch (csp->csp_cipher_alg) {
162 case CRYPTO_AES_CBC:
163 case CRYPTO_AES_ICM:
164 if (csp->csp_ivlen != AES_BLOCK_LEN)
165 return EINVAL;
166 break;
167 case CRYPTO_AES_XTS:
168 if (csp->csp_ivlen != AES_XTS_IV_LEN)
169 return EINVAL;
170 break;
171 default:
172 return EINVAL;
173 }
174 break;
175 case CSP_MODE_DIGEST:
176 switch (csp->csp_auth_alg) {
177 case CRYPTO_SHA1:
178 case CRYPTO_SHA1_HMAC:
179 case CRYPTO_SHA2_256:
180 case CRYPTO_SHA2_256_HMAC:
181 case CRYPTO_SHA2_384:
182 case CRYPTO_SHA2_384_HMAC:
183 case CRYPTO_SHA2_512:
184 case CRYPTO_SHA2_512_HMAC:
185 break;
186 case CRYPTO_AES_NIST_GMAC:
187 if (csp->csp_ivlen != AES_GCM_IV_LEN)
188 return EINVAL;
189 break;
190 default:
191 return EINVAL;
192 }
193 break;
194 case CSP_MODE_AEAD:
195 switch (csp->csp_cipher_alg) {
196 case CRYPTO_AES_NIST_GCM_16:
197 if (csp->csp_ivlen != AES_GCM_IV_LEN)
198 return EINVAL;
199 break;
200 default:
201 return EINVAL;
202 }
203 break;
204 case CSP_MODE_ETA:
205 switch (csp->csp_auth_alg) {
206 case CRYPTO_SHA1_HMAC:
207 case CRYPTO_SHA2_256_HMAC:
208 case CRYPTO_SHA2_384_HMAC:
209 case CRYPTO_SHA2_512_HMAC:
210 switch (csp->csp_cipher_alg) {
211 case CRYPTO_AES_CBC:
212 case CRYPTO_AES_ICM:
213 if (csp->csp_ivlen != AES_BLOCK_LEN)
214 return EINVAL;
215 break;
216 case CRYPTO_AES_XTS:
217 if (csp->csp_ivlen != AES_XTS_IV_LEN)
218 return EINVAL;
219 break;
220 default:
221 return EINVAL;
222 }
223 break;
224 default:
225 return EINVAL;
226 }
227 break;
228 default:
229 return EINVAL;
230 }
231
232 return CRYPTODEV_PROBE_HARDWARE;
233 }
234
235 static CpaStatus
qat_ocf_session_init(device_t dev,struct cryptop * crp,struct qat_ocf_instance * qat_instance,struct qat_ocf_session * qat_ssession)236 qat_ocf_session_init(device_t dev,
237 struct cryptop *crp,
238 struct qat_ocf_instance *qat_instance,
239 struct qat_ocf_session *qat_ssession)
240 {
241 CpaStatus status = CPA_STATUS_SUCCESS;
242 /* Crytpodev structures */
243 crypto_session_t cses;
244 const struct crypto_session_params *csp;
245 /* DP API Session configuration */
246 CpaCySymSessionSetupData sessionSetupData = { 0 };
247 CpaCySymSessionCtx sessionCtx = NULL;
248 Cpa32U sessionCtxSize = 0;
249
250 cses = crp->crp_session;
251 if (NULL == cses) {
252 device_printf(dev, "no crypto session in cryptodev request\n");
253 return CPA_STATUS_FAIL;
254 }
255
256 csp = crypto_get_params(cses);
257 if (NULL == csp) {
258 device_printf(dev, "no session in cryptodev session\n");
259 return CPA_STATUS_FAIL;
260 }
261
262 /* Common fields */
263 sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
264 /* Cipher key */
265 if (crp->crp_cipher_key)
266 sessionSetupData.cipherSetupData.pCipherKey =
267 crp->crp_cipher_key;
268 else
269 sessionSetupData.cipherSetupData.pCipherKey =
270 csp->csp_cipher_key;
271 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
272 csp->csp_cipher_klen;
273
274 /* Auth key */
275 if (crp->crp_auth_key)
276 sessionSetupData.hashSetupData.authModeSetupData.authKey =
277 crp->crp_auth_key;
278 else
279 sessionSetupData.hashSetupData.authModeSetupData.authKey =
280 csp->csp_auth_key;
281 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
282 csp->csp_auth_klen;
283
284 qat_ssession->aadLen = crp->crp_aad_length;
285 if (CPA_TRUE == is_sep_aad_supported(csp))
286 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes =
287 crp->crp_aad_length;
288 else
289 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes =
290 0;
291
292 /* Just setup algorithm - regardless of mode */
293 if (csp->csp_cipher_alg) {
294 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
295
296 switch (csp->csp_cipher_alg) {
297 case CRYPTO_AES_CBC:
298 sessionSetupData.cipherSetupData.cipherAlgorithm =
299 CPA_CY_SYM_CIPHER_AES_CBC;
300 break;
301 case CRYPTO_AES_ICM:
302 sessionSetupData.cipherSetupData.cipherAlgorithm =
303 CPA_CY_SYM_CIPHER_AES_CTR;
304 break;
305 case CRYPTO_AES_XTS:
306 sessionSetupData.cipherSetupData.cipherAlgorithm =
307 CPA_CY_SYM_CIPHER_AES_XTS;
308 break;
309 case CRYPTO_AES_NIST_GCM_16:
310 sessionSetupData.cipherSetupData.cipherAlgorithm =
311 CPA_CY_SYM_CIPHER_AES_GCM;
312 sessionSetupData.hashSetupData.hashAlgorithm =
313 CPA_CY_SYM_HASH_AES_GCM;
314 sessionSetupData.hashSetupData.hashMode =
315 CPA_CY_SYM_HASH_MODE_AUTH;
316 break;
317 default:
318 device_printf(dev,
319 "cipher_alg: %d not supported\n",
320 csp->csp_cipher_alg);
321 status = CPA_STATUS_UNSUPPORTED;
322 goto fail;
323 }
324 }
325
326 if (csp->csp_auth_alg) {
327 switch (csp->csp_auth_alg) {
328 case CRYPTO_SHA1_HMAC:
329 sessionSetupData.hashSetupData.hashAlgorithm =
330 CPA_CY_SYM_HASH_SHA1;
331 sessionSetupData.hashSetupData.hashMode =
332 CPA_CY_SYM_HASH_MODE_AUTH;
333 break;
334 case CRYPTO_SHA1:
335 sessionSetupData.hashSetupData.hashAlgorithm =
336 CPA_CY_SYM_HASH_SHA1;
337 sessionSetupData.hashSetupData.hashMode =
338 CPA_CY_SYM_HASH_MODE_PLAIN;
339 break;
340
341 case CRYPTO_SHA2_256_HMAC:
342 sessionSetupData.hashSetupData.hashAlgorithm =
343 CPA_CY_SYM_HASH_SHA256;
344 sessionSetupData.hashSetupData.hashMode =
345 CPA_CY_SYM_HASH_MODE_AUTH;
346 break;
347 case CRYPTO_SHA2_256:
348 sessionSetupData.hashSetupData.hashAlgorithm =
349 CPA_CY_SYM_HASH_SHA256;
350 sessionSetupData.hashSetupData.hashMode =
351 CPA_CY_SYM_HASH_MODE_PLAIN;
352 break;
353
354 case CRYPTO_SHA2_224_HMAC:
355 sessionSetupData.hashSetupData.hashAlgorithm =
356 CPA_CY_SYM_HASH_SHA224;
357 sessionSetupData.hashSetupData.hashMode =
358 CPA_CY_SYM_HASH_MODE_AUTH;
359 break;
360 case CRYPTO_SHA2_224:
361 sessionSetupData.hashSetupData.hashAlgorithm =
362 CPA_CY_SYM_HASH_SHA224;
363 sessionSetupData.hashSetupData.hashMode =
364 CPA_CY_SYM_HASH_MODE_PLAIN;
365 break;
366
367 case CRYPTO_SHA2_384_HMAC:
368 sessionSetupData.hashSetupData.hashAlgorithm =
369 CPA_CY_SYM_HASH_SHA384;
370 sessionSetupData.hashSetupData.hashMode =
371 CPA_CY_SYM_HASH_MODE_AUTH;
372 break;
373 case CRYPTO_SHA2_384:
374 sessionSetupData.hashSetupData.hashAlgorithm =
375 CPA_CY_SYM_HASH_SHA384;
376 sessionSetupData.hashSetupData.hashMode =
377 CPA_CY_SYM_HASH_MODE_PLAIN;
378 break;
379
380 case CRYPTO_SHA2_512_HMAC:
381 sessionSetupData.hashSetupData.hashAlgorithm =
382 CPA_CY_SYM_HASH_SHA512;
383 sessionSetupData.hashSetupData.hashMode =
384 CPA_CY_SYM_HASH_MODE_AUTH;
385 break;
386 case CRYPTO_SHA2_512:
387 sessionSetupData.hashSetupData.hashAlgorithm =
388 CPA_CY_SYM_HASH_SHA512;
389 sessionSetupData.hashSetupData.hashMode =
390 CPA_CY_SYM_HASH_MODE_PLAIN;
391 break;
392 case CRYPTO_AES_NIST_GMAC:
393 sessionSetupData.hashSetupData.hashAlgorithm =
394 CPA_CY_SYM_HASH_AES_GMAC;
395 break;
396 default:
397 status = CPA_STATUS_UNSUPPORTED;
398 goto fail;
399 }
400 } /* csp->csp_auth_alg */
401
402 /* Setting digest-length if no cipher-only mode is set */
403 if (csp->csp_mode != CSP_MODE_CIPHER) {
404 lac_sym_qat_hash_defs_t *pHashDefsInfo = NULL;
405 if (csp->csp_auth_mlen) {
406 sessionSetupData.hashSetupData.digestResultLenInBytes =
407 csp->csp_auth_mlen;
408 qat_ssession->authLen = csp->csp_auth_mlen;
409 } else {
410 LacSymQat_HashDefsLookupGet(
411 qat_instance->cyInstHandle,
412 sessionSetupData.hashSetupData.hashAlgorithm,
413 &pHashDefsInfo);
414 if (NULL == pHashDefsInfo) {
415 device_printf(
416 dev,
417 "unable to find corresponding hash data\n");
418 status = CPA_STATUS_UNSUPPORTED;
419 goto fail;
420 }
421 sessionSetupData.hashSetupData.digestResultLenInBytes =
422 pHashDefsInfo->algInfo->digestLength;
423 qat_ssession->authLen =
424 pHashDefsInfo->algInfo->digestLength;
425 }
426 sessionSetupData.verifyDigest = CPA_FALSE;
427 }
428
429 switch (csp->csp_mode) {
430 case CSP_MODE_AEAD:
431 case CSP_MODE_ETA:
432 sessionSetupData.symOperation =
433 CPA_CY_SYM_OP_ALGORITHM_CHAINING;
434 /* Place the digest result in a buffer unrelated to srcBuffer */
435 sessionSetupData.digestIsAppended = CPA_FALSE;
436 /* Due to FW limitation to verify only appended MACs */
437 sessionSetupData.verifyDigest = CPA_FALSE;
438 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
439 sessionSetupData.cipherSetupData.cipherDirection =
440 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
441 sessionSetupData.algChainOrder =
442 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
443 } else {
444 sessionSetupData.cipherSetupData.cipherDirection =
445 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
446 sessionSetupData.algChainOrder =
447 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
448 }
449 break;
450 case CSP_MODE_CIPHER:
451 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
452 sessionSetupData.cipherSetupData.cipherDirection =
453 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
454 } else {
455 sessionSetupData.cipherSetupData.cipherDirection =
456 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
457 }
458 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
459 break;
460 case CSP_MODE_DIGEST:
461 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
462 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
463 sessionSetupData.symOperation =
464 CPA_CY_SYM_OP_ALGORITHM_CHAINING;
465 /* GMAC is always encrypt */
466 sessionSetupData.cipherSetupData.cipherDirection =
467 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
468 sessionSetupData.algChainOrder =
469 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
470 sessionSetupData.cipherSetupData.cipherAlgorithm =
471 CPA_CY_SYM_CIPHER_AES_GCM;
472 sessionSetupData.hashSetupData.hashAlgorithm =
473 CPA_CY_SYM_HASH_AES_GMAC;
474 sessionSetupData.hashSetupData.hashMode =
475 CPA_CY_SYM_HASH_MODE_AUTH;
476 /* Same key for cipher and auth */
477 sessionSetupData.cipherSetupData.pCipherKey =
478 csp->csp_auth_key;
479 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
480 csp->csp_auth_klen;
481 /* Generated GMAC stored in separated buffer */
482 sessionSetupData.digestIsAppended = CPA_FALSE;
483 /* Digest verification not allowed in GMAC case */
484 sessionSetupData.verifyDigest = CPA_FALSE;
485 /* No AAD allowed */
486 sessionSetupData.hashSetupData.authModeSetupData
487 .aadLenInBytes = 0;
488 } else {
489 sessionSetupData.cipherSetupData.cipherDirection =
490 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
491 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
492 sessionSetupData.digestIsAppended = CPA_FALSE;
493 }
494 break;
495 default:
496 device_printf(dev,
497 "%s: unhandled crypto algorithm %d, %d\n",
498 __func__,
499 csp->csp_cipher_alg,
500 csp->csp_auth_alg);
501 status = CPA_STATUS_FAIL;
502 goto fail;
503 }
504
505 /* Extracting session size */
506 status = cpaCySymSessionCtxGetSize(qat_instance->cyInstHandle,
507 &sessionSetupData,
508 &sessionCtxSize);
509 if (CPA_STATUS_SUCCESS != status) {
510 device_printf(dev, "unable to get session size\n");
511 goto fail;
512 }
513
514 /* Allocating contiguous memory for session */
515 sessionCtx = contigmalloc(sessionCtxSize,
516 M_QAT_OCF,
517 M_NOWAIT,
518 0,
519 ~1UL,
520 1 << (ilog2(sessionCtxSize - 1) + 1),
521 0);
522 if (NULL == sessionCtx) {
523 device_printf(dev, "unable to allocate memory for session\n");
524 status = CPA_STATUS_RESOURCE;
525 goto fail;
526 }
527
528 status = cpaCySymDpInitSession(qat_instance->cyInstHandle,
529 &sessionSetupData,
530 sessionCtx);
531 if (CPA_STATUS_SUCCESS != status) {
532 device_printf(dev, "session initialization failed\n");
533 goto fail;
534 }
535
536 /* NOTE: lets keep double session (both directions) approach to overcome
537 * lack of direction update in FBSD QAT.
538 */
539 qat_ssession->sessionCtx = sessionCtx;
540 qat_ssession->sessionCtxSize = sessionCtxSize;
541
542 return CPA_STATUS_SUCCESS;
543
544 fail:
545 /* Release resources if any */
546 if (sessionCtx)
547 contigfree(sessionCtx, sessionCtxSize, M_QAT_OCF);
548
549 return status;
550 }
551
552 static int
qat_ocf_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)553 qat_ocf_newsession(device_t dev,
554 crypto_session_t cses,
555 const struct crypto_session_params *csp)
556 {
557 /* Cryptodev QAT structures */
558 struct qat_ocf_softc *qat_softc;
559 struct qat_ocf_dsession *qat_dsession;
560 struct qat_ocf_instance *qat_instance;
561 u_int cpu_id = PCPU_GET(cpuid);
562
563 /* Create cryptodev session */
564 qat_softc = device_get_softc(dev);
565 if (qat_softc->numCyInstances > 0) {
566 qat_instance =
567 &qat_softc
568 ->cyInstHandles[cpu_id % qat_softc->numCyInstances];
569 qat_dsession = crypto_get_driver_session(cses);
570 if (NULL == qat_dsession) {
571 device_printf(dev, "Unable to create new session\n");
572 return (EINVAL);
573 }
574
575 /* Add only instance at this point remaining operations moved to
576 * lazy session init */
577 qat_dsession->qatInstance = qat_instance;
578 } else {
579 return ENXIO;
580 }
581
582 return 0;
583 }
584
585 static CpaStatus
qat_ocf_remove_session(device_t dev,CpaInstanceHandle cyInstHandle,struct qat_ocf_session * qat_session)586 qat_ocf_remove_session(device_t dev,
587 CpaInstanceHandle cyInstHandle,
588 struct qat_ocf_session *qat_session)
589 {
590 CpaStatus status = CPA_STATUS_SUCCESS;
591
592 if (NULL == qat_session->sessionCtx)
593 return CPA_STATUS_SUCCESS;
594
595 /* User callback is executed right before decrementing pending
596 * callback atomic counter. To avoid removing session rejection
597 * we have to wait a very short while for counter update
598 * after call back execution. */
599 status = qat_ocf_wait_for_session(qat_session->sessionCtx,
600 QAT_OCF_SESSION_WAIT_TIMEOUT_MS);
601 if (CPA_STATUS_SUCCESS != status) {
602 device_printf(dev, "waiting for session un-busy failed\n");
603 return CPA_STATUS_FAIL;
604 }
605
606 status = cpaCySymDpRemoveSession(cyInstHandle, qat_session->sessionCtx);
607 if (CPA_STATUS_SUCCESS != status) {
608 device_printf(dev, "error while removing session\n");
609 return CPA_STATUS_FAIL;
610 }
611
612 explicit_bzero(qat_session->sessionCtx, qat_session->sessionCtxSize);
613 contigfree(qat_session->sessionCtx,
614 qat_session->sessionCtxSize,
615 M_QAT_OCF);
616 qat_session->sessionCtx = NULL;
617 qat_session->sessionCtxSize = 0;
618
619 return CPA_STATUS_SUCCESS;
620 }
621
622 static void
qat_ocf_freesession(device_t dev,crypto_session_t cses)623 qat_ocf_freesession(device_t dev, crypto_session_t cses)
624 {
625 CpaStatus status = CPA_STATUS_SUCCESS;
626 struct qat_ocf_dsession *qat_dsession = NULL;
627 struct qat_ocf_instance *qat_instance = NULL;
628
629 qat_dsession = crypto_get_driver_session(cses);
630 qat_instance = qat_dsession->qatInstance;
631 mtx_lock(&qat_instance->cyInstMtx);
632 status = qat_ocf_remove_session(dev,
633 qat_dsession->qatInstance->cyInstHandle,
634 &qat_dsession->encSession);
635 if (CPA_STATUS_SUCCESS != status)
636 device_printf(dev, "unable to remove encrypt session\n");
637 status = qat_ocf_remove_session(dev,
638 qat_dsession->qatInstance->cyInstHandle,
639 &qat_dsession->decSession);
640 if (CPA_STATUS_SUCCESS != status)
641 device_printf(dev, "unable to remove decrypt session\n");
642 mtx_unlock(&qat_instance->cyInstMtx);
643 }
644
645 /* QAT GCM/CCM FW API are only algorithms which support separated AAD. */
646 static CpaStatus
qat_ocf_load_aad_gcm(struct cryptop * crp,struct qat_ocf_cookie * qat_cookie)647 qat_ocf_load_aad_gcm(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
648 {
649 CpaCySymDpOpData *pOpData;
650
651 pOpData = &qat_cookie->pOpdata;
652
653 if (NULL != crp->crp_aad)
654 memcpy(qat_cookie->qat_ocf_gcm_aad,
655 crp->crp_aad,
656 crp->crp_aad_length);
657 else
658 crypto_copydata(crp,
659 crp->crp_aad_start,
660 crp->crp_aad_length,
661 qat_cookie->qat_ocf_gcm_aad);
662
663 pOpData->pAdditionalAuthData = qat_cookie->qat_ocf_gcm_aad;
664 pOpData->additionalAuthData = qat_cookie->qat_ocf_gcm_aad_paddr;
665
666 return CPA_STATUS_SUCCESS;
667 }
668
669 static CpaStatus
qat_ocf_load_aad(struct cryptop * crp,struct qat_ocf_cookie * qat_cookie)670 qat_ocf_load_aad(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
671 {
672 CpaStatus status = CPA_STATUS_SUCCESS;
673 const struct crypto_session_params *csp;
674 CpaCySymDpOpData *pOpData;
675 struct qat_ocf_load_cb_arg args;
676
677 pOpData = &qat_cookie->pOpdata;
678 pOpData->pAdditionalAuthData = NULL;
679 pOpData->additionalAuthData = 0UL;
680
681 if (crp->crp_aad_length == 0)
682 return CPA_STATUS_SUCCESS;
683
684 if (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX)
685 return CPA_STATUS_FAIL;
686
687 csp = crypto_get_params(crp->crp_session);
688
689 /* Handle GCM/CCM case */
690 if (CPA_TRUE == is_sep_aad_supported(csp))
691 return qat_ocf_load_aad_gcm(crp, qat_cookie);
692
693 if (NULL == crp->crp_aad) {
694 /* AAD already embedded in source buffer */
695 pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
696 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start;
697
698 pOpData->messageLenToHashInBytes =
699 crp->crp_aad_length + crp->crp_payload_length;
700 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start;
701
702 return CPA_STATUS_SUCCESS;
703 }
704
705 /* Separated AAD not supported by QAT - lets place the content
706 * of ADD buffer at the very beginning of source SGL */
707 args.crp_op = crp;
708 args.qat_cookie = qat_cookie;
709 args.pOpData = pOpData;
710 args.error = 0;
711 status = bus_dmamap_load(qat_cookie->gcm_aad_dma_mem.dma_tag,
712 qat_cookie->gcm_aad_dma_mem.dma_map,
713 crp->crp_aad,
714 crp->crp_aad_length,
715 qat_ocf_crypto_load_aadbuf_cb,
716 &args,
717 BUS_DMA_NOWAIT);
718 qat_cookie->is_sep_aad_used = CPA_TRUE;
719
720 /* Right after this step we have AAD placed in the first flat buffer
721 * in source SGL */
722 pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
723 pOpData->cryptoStartSrcOffsetInBytes =
724 crp->crp_aad_length + crp->crp_aad_start + crp->crp_payload_start;
725
726 pOpData->messageLenToHashInBytes =
727 crp->crp_aad_length + crp->crp_payload_length;
728 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start;
729
730 return status;
731 }
732
733 static CpaStatus
qat_ocf_load(struct cryptop * crp,struct qat_ocf_cookie * qat_cookie)734 qat_ocf_load(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
735 {
736 CpaStatus status = CPA_STATUS_SUCCESS;
737 CpaCySymDpOpData *pOpData;
738 struct qat_ocf_load_cb_arg args;
739 /* cryptodev internals */
740 const struct crypto_session_params *csp;
741
742 pOpData = &qat_cookie->pOpdata;
743
744 csp = crypto_get_params(crp->crp_session);
745
746 /* Load IV buffer if present */
747 if (csp->csp_ivlen > 0) {
748 memset(qat_cookie->qat_ocf_iv_buf,
749 0,
750 sizeof(qat_cookie->qat_ocf_iv_buf));
751 crypto_read_iv(crp, qat_cookie->qat_ocf_iv_buf);
752 pOpData->iv = qat_cookie->qat_ocf_iv_buf_paddr;
753 pOpData->pIv = qat_cookie->qat_ocf_iv_buf;
754 pOpData->ivLenInBytes = csp->csp_ivlen;
755 }
756
757 /* GCM/CCM - load AAD to separated buffer
758 * AES+SHA - load AAD to first flat in SGL */
759 status = qat_ocf_load_aad(crp, qat_cookie);
760 if (CPA_STATUS_SUCCESS != status)
761 goto fail;
762
763 /* Load source buffer */
764 args.crp_op = crp;
765 args.qat_cookie = qat_cookie;
766 args.pOpData = pOpData;
767 args.error = 0;
768 status = bus_dmamap_load_crp_buffer(qat_cookie->src_dma_mem.dma_tag,
769 qat_cookie->src_dma_mem.dma_map,
770 &crp->crp_buf,
771 qat_ocf_crypto_load_buf_cb,
772 &args,
773 BUS_DMA_NOWAIT);
774 if (CPA_STATUS_SUCCESS != status)
775 goto fail;
776 pOpData->srcBuffer = qat_cookie->src_buffer_list_paddr;
777 pOpData->srcBufferLen = CPA_DP_BUFLIST;
778
779 /* Load destination buffer */
780 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
781 status =
782 bus_dmamap_load_crp_buffer(qat_cookie->dst_dma_mem.dma_tag,
783 qat_cookie->dst_dma_mem.dma_map,
784 &crp->crp_obuf,
785 qat_ocf_crypto_load_obuf_cb,
786 &args,
787 BUS_DMA_NOWAIT);
788 if (CPA_STATUS_SUCCESS != status)
789 goto fail;
790 pOpData->dstBuffer = qat_cookie->dst_buffer_list_paddr;
791 pOpData->dstBufferLen = CPA_DP_BUFLIST;
792 } else {
793 pOpData->dstBuffer = pOpData->srcBuffer;
794 pOpData->dstBufferLen = pOpData->srcBufferLen;
795 }
796
797 if (CPA_TRUE == is_use_sep_digest(csp))
798 pOpData->digestResult = qat_cookie->qat_ocf_digest_paddr;
799 else
800 pOpData->digestResult = 0UL;
801
802 /* GMAC - aka zero length buffer */
803 if (CPA_TRUE == is_gmac_exception(csp))
804 pOpData->messageLenToCipherInBytes = 0;
805
806 fail:
807 return status;
808 }
809
810 static int
qat_ocf_check_input(device_t dev,struct cryptop * crp)811 qat_ocf_check_input(device_t dev, struct cryptop *crp)
812 {
813 const struct crypto_session_params *csp;
814 csp = crypto_get_params(crp->crp_session);
815
816 if (crypto_buffer_len(&crp->crp_buf) > QAT_OCF_MAX_LEN)
817 return E2BIG;
818
819 if (CPA_TRUE == is_sep_aad_supported(csp) &&
820 (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX))
821 return EBADMSG;
822
823 return 0;
824 }
825
826 static int
qat_ocf_process(device_t dev,struct cryptop * crp,int hint)827 qat_ocf_process(device_t dev, struct cryptop *crp, int hint)
828 {
829 CpaStatus status = CPA_STATUS_SUCCESS;
830 int rc = 0;
831 struct qat_ocf_dsession *qat_dsession = NULL;
832 struct qat_ocf_session *qat_session = NULL;
833 struct qat_ocf_instance *qat_instance = NULL;
834 CpaCySymDpOpData *pOpData = NULL;
835 struct qat_ocf_cookie *qat_cookie = NULL;
836 CpaBoolean memLoaded = CPA_FALSE;
837
838 rc = qat_ocf_check_input(dev, crp);
839 if (rc)
840 goto fail;
841
842 qat_dsession = crypto_get_driver_session(crp->crp_session);
843
844 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
845 qat_session = &qat_dsession->encSession;
846 else
847 qat_session = &qat_dsession->decSession;
848 qat_instance = qat_dsession->qatInstance;
849
850 status = qat_ocf_cookie_alloc(qat_instance, &qat_cookie);
851 if (CPA_STATUS_SUCCESS != status) {
852 rc = EAGAIN;
853 goto fail;
854 }
855
856 qat_cookie->crp_op = crp;
857
858 /* Common request fields */
859 pOpData = &qat_cookie->pOpdata;
860 pOpData->instanceHandle = qat_instance->cyInstHandle;
861 pOpData->sessionCtx = NULL;
862
863 /* Cipher fields */
864 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start;
865 pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
866 /* Digest fields - any exceptions from this basic rules are covered
867 * in qat_ocf_load */
868 pOpData->hashStartSrcOffsetInBytes = crp->crp_payload_start;
869 pOpData->messageLenToHashInBytes = crp->crp_payload_length;
870
871 status = qat_ocf_load(crp, qat_cookie);
872 if (CPA_STATUS_SUCCESS != status) {
873 device_printf(dev,
874 "unable to load OCF buffers to QAT DMA "
875 "transaction\n");
876 rc = EIO;
877 goto fail;
878 }
879 memLoaded = CPA_TRUE;
880
881 status = qat_ocf_cookie_dma_pre_sync(crp, pOpData);
882 if (CPA_STATUS_SUCCESS != status) {
883 device_printf(dev, "unable to sync DMA buffers\n");
884 rc = EIO;
885 goto fail;
886 }
887
888 mtx_lock(&qat_instance->cyInstMtx);
889 /* Session initialization at the first request. It's done
890 * in such way to overcome missing QAT specific session data
891 * such like AAD length and limited possibility to update
892 * QAT session while handling traffic.
893 */
894 if (NULL == qat_session->sessionCtx) {
895 status =
896 qat_ocf_session_init(dev, crp, qat_instance, qat_session);
897 if (CPA_STATUS_SUCCESS != status) {
898 mtx_unlock(&qat_instance->cyInstMtx);
899 device_printf(dev, "unable to init session\n");
900 rc = EIO;
901 goto fail;
902 }
903 } else {
904 status = qat_ocf_handle_session_update(qat_dsession, crp);
905 if (CPA_STATUS_RESOURCE == status) {
906 mtx_unlock(&qat_instance->cyInstMtx);
907 rc = EAGAIN;
908 goto fail;
909 } else if (CPA_STATUS_SUCCESS != status) {
910 mtx_unlock(&qat_instance->cyInstMtx);
911 rc = EIO;
912 goto fail;
913 }
914 }
915 pOpData->sessionCtx = qat_session->sessionCtx;
916 status = cpaCySymDpEnqueueOp(pOpData, CPA_TRUE);
917 mtx_unlock(&qat_instance->cyInstMtx);
918 if (CPA_STATUS_SUCCESS != status) {
919 if (CPA_STATUS_RETRY == status) {
920 rc = EAGAIN;
921 goto fail;
922 }
923 device_printf(dev,
924 "unable to send request. Status: %d\n",
925 status);
926 rc = EIO;
927 goto fail;
928 }
929
930 return 0;
931 fail:
932 if (qat_cookie) {
933 if (memLoaded)
934 qat_ocf_cookie_dma_unload(crp, pOpData);
935 qat_ocf_cookie_free(qat_instance, qat_cookie);
936 }
937 crp->crp_etype = rc;
938 crypto_done(crp);
939
940 return 0;
941 }
942
943 static void
qat_ocf_identify(driver_t * drv,device_t parent)944 qat_ocf_identify(driver_t *drv, device_t parent)
945 {
946 if (device_find_child(parent, "qat_ocf", -1) == NULL &&
947 BUS_ADD_CHILD(parent, 200, "qat_ocf", -1) == 0)
948 device_printf(parent, "qat_ocf: could not attach!");
949 }
950
951 static int
qat_ocf_probe(device_t dev)952 qat_ocf_probe(device_t dev)
953 {
954 device_set_desc(dev, "QAT engine");
955 return (BUS_PROBE_NOWILDCARD);
956 }
957
958 static CpaStatus
qat_ocf_get_irq_instances(CpaInstanceHandle * cyInstHandles,Cpa16U cyInstHandlesSize,Cpa16U * foundInstances)959 qat_ocf_get_irq_instances(CpaInstanceHandle *cyInstHandles,
960 Cpa16U cyInstHandlesSize,
961 Cpa16U *foundInstances)
962 {
963 CpaStatus status = CPA_STATUS_SUCCESS;
964 icp_accel_dev_t **pAdfInsts = NULL;
965 icp_accel_dev_t *dev_addr = NULL;
966 sal_t *baseAddr = NULL;
967 sal_list_t *listTemp = NULL;
968 CpaInstanceHandle cyInstHandle;
969 CpaInstanceInfo2 info;
970 Cpa16U numDevices;
971 Cpa32U instCtr = 0;
972 Cpa32U i;
973
974 /* Get the number of devices */
975 status = icp_amgr_getNumInstances(&numDevices);
976 if (CPA_STATUS_SUCCESS != status)
977 return status;
978
979 /* Allocate memory to store addr of accel_devs */
980 pAdfInsts =
981 malloc(numDevices * sizeof(icp_accel_dev_t *), M_QAT_OCF, M_WAITOK);
982
983 /* Get ADF to return all accel_devs that support either
984 * symmetric or asymmetric crypto */
985 status = icp_amgr_getAllAccelDevByCapabilities(
986 (ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC), pAdfInsts, &numDevices);
987 if (CPA_STATUS_SUCCESS != status) {
988 free(pAdfInsts, M_QAT_OCF);
989 return status;
990 }
991
992 for (i = 0; i < numDevices; i++) {
993 dev_addr = (icp_accel_dev_t *)pAdfInsts[i];
994 baseAddr = dev_addr->pSalHandle;
995 if (NULL == baseAddr)
996 continue;
997 listTemp = baseAddr->sym_services;
998 if (NULL == listTemp) {
999 listTemp = baseAddr->crypto_services;
1000 }
1001
1002 while (NULL != listTemp) {
1003 cyInstHandle = SalList_getObject(listTemp);
1004 status = cpaCyInstanceGetInfo2(cyInstHandle, &info);
1005 if (CPA_STATUS_SUCCESS != status)
1006 continue;
1007 listTemp = SalList_next(listTemp);
1008 if (CPA_TRUE == info.isPolled)
1009 continue;
1010 if (instCtr >= cyInstHandlesSize)
1011 break;
1012 cyInstHandles[instCtr++] = cyInstHandle;
1013 }
1014 }
1015 free(pAdfInsts, M_QAT_OCF);
1016 *foundInstances = instCtr;
1017
1018 return CPA_STATUS_SUCCESS;
1019 }
1020
1021 static CpaStatus
qat_ocf_start_instances(struct qat_ocf_softc * qat_softc,device_t dev)1022 qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev)
1023 {
1024 CpaStatus status = CPA_STATUS_SUCCESS;
1025 Cpa16U numInstances = 0;
1026 CpaInstanceHandle cyInstHandles[QAT_OCF_MAX_INSTANCES] = { 0 };
1027 CpaInstanceHandle cyInstHandle = NULL;
1028 Cpa32U startedInstances = 0;
1029 Cpa32U i;
1030
1031 qat_softc->numCyInstances = 0;
1032 status = qat_ocf_get_irq_instances(cyInstHandles,
1033 QAT_OCF_MAX_INSTANCES,
1034 &numInstances);
1035 if (CPA_STATUS_SUCCESS != status)
1036 return status;
1037
1038 for (i = 0; i < numInstances; i++) {
1039 struct qat_ocf_instance *qat_ocf_instance;
1040
1041 cyInstHandle = cyInstHandles[i];
1042 if (!cyInstHandle)
1043 continue;
1044
1045 /* Starting instance */
1046 status = cpaCyStartInstance(cyInstHandle);
1047 if (CPA_STATUS_SUCCESS != status) {
1048 device_printf(qat_softc->sc_dev,
1049 "unable to get start instance\n");
1050 continue;
1051 }
1052
1053 qat_ocf_instance = &qat_softc->cyInstHandles[startedInstances];
1054 qat_ocf_instance->cyInstHandle = cyInstHandle;
1055 mtx_init(&qat_ocf_instance->cyInstMtx,
1056 "Instance MTX",
1057 NULL,
1058 MTX_DEF);
1059
1060 status =
1061 cpaCySetAddressTranslation(cyInstHandle, qatVirtToPhys);
1062 if (CPA_STATUS_SUCCESS != status) {
1063 device_printf(qat_softc->sc_dev,
1064 "unable to add virt to phys callback\n");
1065 goto fail;
1066 }
1067
1068 status = cpaCySymDpRegCbFunc(cyInstHandle, symDpCallback);
1069 if (CPA_STATUS_SUCCESS != status) {
1070 device_printf(qat_softc->sc_dev,
1071 "unable to add user callback\n");
1072 goto fail;
1073 }
1074
1075 /* Initialize cookie pool */
1076 status = qat_ocf_cookie_pool_init(qat_ocf_instance, dev);
1077 if (CPA_STATUS_SUCCESS != status) {
1078 device_printf(qat_softc->sc_dev,
1079 "unable to create cookie pool\n");
1080 goto fail;
1081 }
1082
1083 /* Disable forcing HW MAC validation for AEAD */
1084 status = icp_sal_setForceAEADMACVerify(cyInstHandle, CPA_FALSE);
1085 if (CPA_STATUS_SUCCESS != status) {
1086 device_printf(
1087 qat_softc->sc_dev,
1088 "unable to disable AEAD HW MAC verification\n");
1089 goto fail;
1090 }
1091
1092 qat_ocf_instance->driver_id = qat_softc->cryptodev_id;
1093
1094 startedInstances++;
1095 continue;
1096 fail:
1097 mtx_destroy(&qat_ocf_instance->cyInstMtx);
1098
1099 /* Stop instance */
1100 status = cpaCyStopInstance(cyInstHandle);
1101 if (CPA_STATUS_SUCCESS != status)
1102 device_printf(qat_softc->sc_dev,
1103 "unable to stop the instance\n");
1104 }
1105 qat_softc->numCyInstances = startedInstances;
1106
1107 return CPA_STATUS_SUCCESS;
1108 }
1109
1110 static CpaStatus
qat_ocf_stop_instances(struct qat_ocf_softc * qat_softc)1111 qat_ocf_stop_instances(struct qat_ocf_softc *qat_softc)
1112 {
1113 CpaStatus status = CPA_STATUS_SUCCESS;
1114 int i;
1115
1116 for (i = 0; i < qat_softc->numCyInstances; i++) {
1117 struct qat_ocf_instance *qat_instance;
1118
1119 qat_instance = &qat_softc->cyInstHandles[i];
1120 status = cpaCyStopInstance(qat_instance->cyInstHandle);
1121 if (CPA_STATUS_SUCCESS != status) {
1122 pr_err("QAT: stopping instance id: %d failed\n", i);
1123 continue;
1124 }
1125 qat_ocf_cookie_pool_deinit(qat_instance);
1126 mtx_destroy(&qat_instance->cyInstMtx);
1127 }
1128
1129 qat_softc->numCyInstances = 0;
1130
1131 return status;
1132 }
1133
1134 static int
qat_ocf_deinit(struct qat_ocf_softc * qat_softc)1135 qat_ocf_deinit(struct qat_ocf_softc *qat_softc)
1136 {
1137 int status = 0;
1138 CpaStatus cpaStatus;
1139
1140 if (qat_softc->cryptodev_id >= 0) {
1141 crypto_unregister_all(qat_softc->cryptodev_id);
1142 qat_softc->cryptodev_id = -1;
1143 }
1144
1145 /* Stop QAT instances */
1146 cpaStatus = qat_ocf_stop_instances(qat_softc);
1147 if (CPA_STATUS_SUCCESS != cpaStatus) {
1148 device_printf(qat_softc->sc_dev, "unable to stop instances\n");
1149 status = EIO;
1150 }
1151
1152 return status;
1153 }
1154
1155 static int
qat_ocf_init(struct qat_ocf_softc * qat_softc)1156 qat_ocf_init(struct qat_ocf_softc *qat_softc)
1157 {
1158 int32_t cryptodev_id;
1159
1160 /* Starting instances for OCF */
1161 if (qat_ocf_start_instances(qat_softc, qat_softc->sc_dev)) {
1162 device_printf(qat_softc->sc_dev,
1163 "unable to get QAT IRQ instances\n");
1164 goto fail;
1165 }
1166
1167 /* Register only if instances available */
1168 if (qat_softc->numCyInstances) {
1169 cryptodev_id =
1170 crypto_get_driverid(qat_softc->sc_dev,
1171 sizeof(struct qat_ocf_dsession),
1172 CRYPTOCAP_F_HARDWARE);
1173 if (cryptodev_id < 0) {
1174 device_printf(qat_softc->sc_dev,
1175 "cannot initialize!\n");
1176 goto fail;
1177 }
1178 qat_softc->cryptodev_id = cryptodev_id;
1179 }
1180
1181 return 0;
1182 fail:
1183 qat_ocf_deinit(qat_softc);
1184
1185 return ENXIO;
1186 }
1187
qat_ocf_sysctl_handle(SYSCTL_HANDLER_ARGS)1188 static int qat_ocf_sysctl_handle(SYSCTL_HANDLER_ARGS)
1189 {
1190 struct qat_ocf_softc *qat_softc = NULL;
1191 int ret = 0;
1192 device_t dev = arg1;
1193 u_int enabled;
1194
1195 qat_softc = device_get_softc(dev);
1196 enabled = qat_softc->enabled;
1197
1198 ret = sysctl_handle_int(oidp, &enabled, 0, req);
1199 if (ret || !req->newptr)
1200 return (ret);
1201
1202 if (qat_softc->enabled != enabled) {
1203 if (enabled) {
1204 ret = qat_ocf_init(qat_softc);
1205
1206 } else {
1207 ret = qat_ocf_deinit(qat_softc);
1208 }
1209
1210 if (!ret)
1211 qat_softc->enabled = enabled;
1212 }
1213
1214 return ret;
1215 }
1216
1217 static int
qat_ocf_attach(device_t dev)1218 qat_ocf_attach(device_t dev)
1219 {
1220 int status;
1221 struct qat_ocf_softc *qat_softc;
1222
1223 qat_softc = device_get_softc(dev);
1224 qat_softc->sc_dev = dev;
1225 qat_softc->cryptodev_id = -1;
1226 qat_softc->enabled = 1;
1227
1228 qat_softc->rc =
1229 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
1230 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
1231 OID_AUTO,
1232 "enable",
1233 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
1234 dev,
1235 0,
1236 qat_ocf_sysctl_handle,
1237 "I",
1238 "QAT OCF support enablement");
1239
1240 if (!qat_softc->rc)
1241 return ENOMEM;
1242 if (qat_softc->enabled) {
1243 status = qat_ocf_init(qat_softc);
1244 if (status) {
1245 device_printf(dev, "qat_ocf init failed\n");
1246 goto fail;
1247 }
1248 }
1249
1250 return 0;
1251 fail:
1252 qat_ocf_deinit(qat_softc);
1253
1254 return (ENXIO);
1255 }
1256
1257 static int
qat_ocf_detach(device_t dev)1258 qat_ocf_detach(device_t dev)
1259 {
1260 struct qat_ocf_softc *qat_softc = device_get_softc(dev);
1261
1262 return qat_ocf_deinit(qat_softc);
1263 }
1264
1265 static device_method_t qat_ocf_methods[] =
1266 { DEVMETHOD(device_identify, qat_ocf_identify),
1267 DEVMETHOD(device_probe, qat_ocf_probe),
1268 DEVMETHOD(device_attach, qat_ocf_attach),
1269 DEVMETHOD(device_detach, qat_ocf_detach),
1270
1271 /* Cryptodev interface */
1272 DEVMETHOD(cryptodev_probesession, qat_ocf_probesession),
1273 DEVMETHOD(cryptodev_newsession, qat_ocf_newsession),
1274 DEVMETHOD(cryptodev_freesession, qat_ocf_freesession),
1275 DEVMETHOD(cryptodev_process, qat_ocf_process),
1276
1277 DEVMETHOD_END };
1278
1279 static driver_t qat_ocf_driver = {
1280 .name = "qat_ocf",
1281 .methods = qat_ocf_methods,
1282 .size = sizeof(struct qat_ocf_softc),
1283 };
1284
1285
1286 DRIVER_MODULE_ORDERED(qat,
1287 nexus,
1288 qat_ocf_driver,
1289 NULL,
1290 NULL,
1291 SI_ORDER_ANY);
1292 MODULE_VERSION(qat, 1);
1293 MODULE_DEPEND(qat, qat_c62x, 1, 1, 1);
1294 MODULE_DEPEND(qat, qat_200xx, 1, 1, 1);
1295 MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1);
1296 MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1);
1297 MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1);
1298 MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1);
1299 MODULE_DEPEND(qat, crypto, 1, 1, 1);
1300 MODULE_DEPEND(qat, qat_common, 1, 1, 1);
1301 MODULE_DEPEND(qat, qat_api, 1, 1, 1);
1302 MODULE_DEPEND(qat, linuxkpi, 1, 1, 1);
1303