1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * This file contains the core framework routines for the 28 * kernel cryptographic framework. These routines are at the 29 * layer, between the kernel API/ioctls and the SPI. 30 */ 31 32 #include <sys/types.h> 33 #include <sys/errno.h> 34 #include <sys/kmem.h> 35 #include <sys/proc.h> 36 #include <sys/cpuvar.h> 37 #include <sys/cpupart.h> 38 #include <sys/ksynch.h> 39 #include <sys/callb.h> 40 #include <sys/cmn_err.h> 41 #include <sys/systm.h> 42 #include <sys/sysmacros.h> 43 #include <sys/kstat.h> 44 #include <sys/crypto/common.h> 45 #include <sys/crypto/impl.h> 46 #include <sys/crypto/sched_impl.h> 47 #include <sys/crypto/api.h> 48 #include <sys/crypto/spi.h> 49 #include <sys/taskq_impl.h> 50 #include <sys/ddi.h> 51 #include <sys/sunddi.h> 52 53 54 kcf_global_swq_t *gswq; /* Global software queue */ 55 56 /* Thread pool related variables */ 57 static kcf_pool_t *kcfpool; /* Thread pool of kcfd LWPs */ 58 int kcf_maxthreads = 2; 59 int kcf_minthreads = 1; 60 int kcf_thr_multiple = 2; /* Boot-time tunable for experimentation */ 61 static ulong_t kcf_idlethr_timeout; 62 static boolean_t kcf_sched_running = B_FALSE; 63 #define KCF_DEFAULT_THRTIMEOUT 60000000 /* 60 seconds */ 64 65 /* kmem caches used by the scheduler */ 66 static struct kmem_cache *kcf_sreq_cache; 67 static struct kmem_cache *kcf_areq_cache; 68 static struct kmem_cache *kcf_context_cache; 69 70 /* Global request ID table */ 71 static kcf_reqid_table_t *kcf_reqid_table[REQID_TABLES]; 72 73 /* KCF stats. Not protected. */ 74 static kcf_stats_t kcf_ksdata = { 75 { "total threads in pool", KSTAT_DATA_UINT32}, 76 { "idle threads in pool", KSTAT_DATA_UINT32}, 77 { "min threads in pool", KSTAT_DATA_UINT32}, 78 { "max threads in pool", KSTAT_DATA_UINT32}, 79 { "requests in gswq", KSTAT_DATA_UINT32}, 80 { "max requests in gswq", KSTAT_DATA_UINT32}, 81 { "threads for HW taskq", KSTAT_DATA_UINT32}, 82 { "minalloc for HW taskq", KSTAT_DATA_UINT32}, 83 { "maxalloc for HW taskq", KSTAT_DATA_UINT32} 84 }; 85 86 static kstat_t *kcf_misc_kstat = NULL; 87 ulong_t kcf_swprov_hndl = 0; 88 89 static kcf_areq_node_t *kcf_areqnode_alloc(kcf_provider_desc_t *, 90 kcf_context_t *, crypto_call_req_t *, kcf_req_params_t *, boolean_t); 91 static int kcf_disp_sw_request(kcf_areq_node_t *); 92 static void process_req_hwp(void *); 93 static kcf_areq_node_t *kcf_dequeue(); 94 static int kcf_enqueue(kcf_areq_node_t *); 95 static void kcf_failover_thread(); 96 static void kcfpool_alloc(); 97 static void kcf_reqid_delete(kcf_areq_node_t *areq); 98 static crypto_req_id_t kcf_reqid_insert(kcf_areq_node_t *areq); 99 static int kcf_misc_kstat_update(kstat_t *ksp, int rw); 100 static void compute_min_max_threads(); 101 102 103 /* 104 * Create a new context. 105 */ 106 crypto_ctx_t * 107 kcf_new_ctx(crypto_call_req_t *crq, kcf_provider_desc_t *pd, 108 crypto_session_id_t sid) 109 { 110 crypto_ctx_t *ctx; 111 kcf_context_t *kcf_ctx; 112 113 kcf_ctx = kmem_cache_alloc(kcf_context_cache, 114 (crq == NULL) ? KM_SLEEP : KM_NOSLEEP); 115 if (kcf_ctx == NULL) 116 return (NULL); 117 118 /* initialize the context for the consumer */ 119 kcf_ctx->kc_refcnt = 1; 120 kcf_ctx->kc_req_chain_first = NULL; 121 kcf_ctx->kc_req_chain_last = NULL; 122 kcf_ctx->kc_secondctx = NULL; 123 KCF_PROV_REFHOLD(pd); 124 kcf_ctx->kc_prov_desc = pd; 125 kcf_ctx->kc_sw_prov_desc = NULL; 126 kcf_ctx->kc_mech = NULL; 127 128 ctx = &kcf_ctx->kc_glbl_ctx; 129 ctx->cc_provider = pd->pd_prov_handle; 130 ctx->cc_session = sid; 131 ctx->cc_provider_private = NULL; 132 ctx->cc_framework_private = (void *)kcf_ctx; 133 ctx->cc_flags = 0; 134 ctx->cc_opstate = NULL; 135 136 return (ctx); 137 } 138 139 /* 140 * Allocate a new async request node. 141 * 142 * ictx - Framework private context pointer 143 * crq - Has callback function and argument. Should be non NULL. 144 * req - The parameters to pass to the SPI 145 */ 146 static kcf_areq_node_t * 147 kcf_areqnode_alloc(kcf_provider_desc_t *pd, kcf_context_t *ictx, 148 crypto_call_req_t *crq, kcf_req_params_t *req, boolean_t isdual) 149 { 150 kcf_areq_node_t *arptr, *areq; 151 152 ASSERT(crq != NULL); 153 arptr = kmem_cache_alloc(kcf_areq_cache, KM_NOSLEEP); 154 if (arptr == NULL) 155 return (NULL); 156 157 arptr->an_state = REQ_ALLOCATED; 158 arptr->an_reqarg = *crq; 159 arptr->an_params = *req; 160 arptr->an_context = ictx; 161 arptr->an_isdual = isdual; 162 163 arptr->an_next = arptr->an_prev = NULL; 164 KCF_PROV_REFHOLD(pd); 165 arptr->an_provider = pd; 166 arptr->an_tried_plist = NULL; 167 arptr->an_refcnt = 1; 168 arptr->an_idnext = arptr->an_idprev = NULL; 169 170 /* 171 * Requests for context-less operations do not use the 172 * fields - an_is_my_turn, and an_ctxchain_next. 173 */ 174 if (ictx == NULL) 175 return (arptr); 176 177 KCF_CONTEXT_REFHOLD(ictx); 178 /* 179 * Chain this request to the context. 180 */ 181 mutex_enter(&ictx->kc_in_use_lock); 182 arptr->an_ctxchain_next = NULL; 183 if ((areq = ictx->kc_req_chain_last) == NULL) { 184 arptr->an_is_my_turn = B_TRUE; 185 ictx->kc_req_chain_last = 186 ictx->kc_req_chain_first = arptr; 187 } else { 188 ASSERT(ictx->kc_req_chain_first != NULL); 189 arptr->an_is_my_turn = B_FALSE; 190 /* Insert the new request to the end of the chain. */ 191 areq->an_ctxchain_next = arptr; 192 ictx->kc_req_chain_last = arptr; 193 } 194 mutex_exit(&ictx->kc_in_use_lock); 195 196 return (arptr); 197 } 198 199 /* 200 * Queue the request node and do one of the following: 201 * - If there is an idle thread signal it to run. 202 * - If there is no idle thread and max running threads is not 203 * reached, signal the creator thread for more threads. 204 * 205 * If the two conditions above are not met, we don't need to do 206 * any thing. The request will be picked up by one of the 207 * worker threads when it becomes available. 208 */ 209 static int 210 kcf_disp_sw_request(kcf_areq_node_t *areq) 211 { 212 int err; 213 int cnt = 0; 214 215 if ((err = kcf_enqueue(areq)) != 0) 216 return (err); 217 218 if (kcfpool->kp_idlethreads > 0) { 219 /* Signal an idle thread to run */ 220 mutex_enter(&gswq->gs_lock); 221 cv_signal(&gswq->gs_cv); 222 mutex_exit(&gswq->gs_lock); 223 224 return (CRYPTO_QUEUED); 225 } 226 227 /* 228 * We keep the number of running threads to be at 229 * kcf_minthreads to reduce gs_lock contention. 230 */ 231 cnt = kcf_minthreads - 232 (kcfpool->kp_threads - kcfpool->kp_blockedthreads); 233 if (cnt > 0) { 234 /* 235 * The following ensures the number of threads in pool 236 * does not exceed kcf_maxthreads. 237 */ 238 cnt = min(cnt, kcf_maxthreads - kcfpool->kp_threads); 239 if (cnt > 0) { 240 /* Signal the creator thread for more threads */ 241 mutex_enter(&kcfpool->kp_user_lock); 242 if (!kcfpool->kp_signal_create_thread) { 243 kcfpool->kp_signal_create_thread = B_TRUE; 244 kcfpool->kp_nthrs = cnt; 245 cv_signal(&kcfpool->kp_user_cv); 246 } 247 mutex_exit(&kcfpool->kp_user_lock); 248 } 249 } 250 251 return (CRYPTO_QUEUED); 252 } 253 254 /* 255 * This routine is called by the taskq associated with 256 * each hardware provider. We notify the kernel consumer 257 * via the callback routine in case of CRYPTO_SUCCESS or 258 * a failure. 259 * 260 * A request can be of type kcf_areq_node_t or of type 261 * kcf_sreq_node_t. 262 */ 263 static void 264 process_req_hwp(void *ireq) 265 { 266 int error = 0; 267 crypto_ctx_t *ctx; 268 kcf_call_type_t ctype; 269 kcf_provider_desc_t *pd; 270 kcf_areq_node_t *areq = (kcf_areq_node_t *)ireq; 271 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)ireq; 272 kcf_prov_cpu_t *mp; 273 274 pd = ((ctype = GET_REQ_TYPE(ireq)) == CRYPTO_SYNCH) ? 275 sreq->sn_provider : areq->an_provider; 276 277 /* 278 * Wait if flow control is in effect for the provider. A 279 * CRYPTO_PROVIDER_READY or CRYPTO_PROVIDER_FAILED 280 * notification will signal us. We also get signaled if 281 * the provider is unregistering. 282 */ 283 if (pd->pd_state == KCF_PROV_BUSY) { 284 mutex_enter(&pd->pd_lock); 285 while (pd->pd_state == KCF_PROV_BUSY) 286 cv_wait(&pd->pd_resume_cv, &pd->pd_lock); 287 mutex_exit(&pd->pd_lock); 288 } 289 290 /* 291 * Bump the internal reference count while the request is being 292 * processed. This is how we know when it's safe to unregister 293 * a provider. This step must precede the pd_state check below. 294 */ 295 mp = &(pd->pd_percpu_bins[CPU_SEQID]); 296 KCF_PROV_JOB_HOLD(mp); 297 298 /* 299 * Fail the request if the provider has failed. We return a 300 * recoverable error and the notified clients attempt any 301 * recovery. For async clients this is done in kcf_aop_done() 302 * and for sync clients it is done in the k-api routines. 303 */ 304 if (pd->pd_state >= KCF_PROV_FAILED) { 305 error = CRYPTO_DEVICE_ERROR; 306 goto bail; 307 } 308 309 if (ctype == CRYPTO_SYNCH) { 310 mutex_enter(&sreq->sn_lock); 311 sreq->sn_state = REQ_INPROGRESS; 312 sreq->sn_mp = mp; 313 mutex_exit(&sreq->sn_lock); 314 315 ctx = sreq->sn_context ? &sreq->sn_context->kc_glbl_ctx : NULL; 316 error = common_submit_request(sreq->sn_provider, ctx, 317 sreq->sn_params, sreq); 318 } else { 319 kcf_context_t *ictx; 320 ASSERT(ctype == CRYPTO_ASYNCH); 321 322 /* 323 * We are in the per-hardware provider thread context and 324 * hence can sleep. Note that the caller would have done 325 * a taskq_dispatch(..., TQ_NOSLEEP) and would have returned. 326 */ 327 ctx = (ictx = areq->an_context) ? &ictx->kc_glbl_ctx : NULL; 328 329 mutex_enter(&areq->an_lock); 330 /* 331 * We need to maintain ordering for multi-part requests. 332 * an_is_my_turn is set to B_TRUE initially for a request 333 * when it is enqueued and there are no other requests 334 * for that context. It is set later from kcf_aop_done() when 335 * the request before us in the chain of requests for the 336 * context completes. We get signaled at that point. 337 */ 338 if (ictx != NULL) { 339 ASSERT(ictx->kc_prov_desc == areq->an_provider); 340 341 while (areq->an_is_my_turn == B_FALSE) { 342 cv_wait(&areq->an_turn_cv, &areq->an_lock); 343 } 344 } 345 areq->an_state = REQ_INPROGRESS; 346 areq->an_mp = mp; 347 mutex_exit(&areq->an_lock); 348 349 error = common_submit_request(areq->an_provider, ctx, 350 &areq->an_params, areq); 351 } 352 353 bail: 354 if (error == CRYPTO_QUEUED) { 355 /* 356 * The request is queued by the provider and we should 357 * get a crypto_op_notification() from the provider later. 358 * We notify the consumer at that time. 359 */ 360 return; 361 } else { /* CRYPTO_SUCCESS or other failure */ 362 KCF_PROV_JOB_RELE(mp); 363 if (ctype == CRYPTO_SYNCH) 364 kcf_sop_done(sreq, error); 365 else 366 kcf_aop_done(areq, error); 367 } 368 } 369 370 /* 371 * This routine checks if a request can be retried on another 372 * provider. If true, mech1 is initialized to point to the mechanism 373 * structure. mech2 is also initialized in case of a dual operation. fg 374 * is initialized to the correct crypto_func_group_t bit flag. They are 375 * initialized by this routine, so that the caller can pass them to a 376 * kcf_get_mech_provider() or kcf_get_dual_provider() with no further change. 377 * 378 * We check that the request is for a init or atomic routine and that 379 * it is for one of the operation groups used from k-api . 380 */ 381 static boolean_t 382 can_resubmit(kcf_areq_node_t *areq, crypto_mechanism_t **mech1, 383 crypto_mechanism_t **mech2, crypto_func_group_t *fg) 384 { 385 kcf_req_params_t *params; 386 kcf_op_type_t optype; 387 388 params = &areq->an_params; 389 optype = params->rp_optype; 390 391 if (!(IS_INIT_OP(optype) || IS_ATOMIC_OP(optype))) 392 return (B_FALSE); 393 394 switch (params->rp_opgrp) { 395 case KCF_OG_DIGEST: { 396 kcf_digest_ops_params_t *dops = ¶ms->rp_u.digest_params; 397 398 dops->do_mech.cm_type = dops->do_framework_mechtype; 399 *mech1 = &dops->do_mech; 400 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DIGEST : 401 CRYPTO_FG_DIGEST_ATOMIC; 402 break; 403 } 404 405 case KCF_OG_MAC: { 406 kcf_mac_ops_params_t *mops = ¶ms->rp_u.mac_params; 407 408 mops->mo_mech.cm_type = mops->mo_framework_mechtype; 409 *mech1 = &mops->mo_mech; 410 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC : 411 CRYPTO_FG_MAC_ATOMIC; 412 break; 413 } 414 415 case KCF_OG_SIGN: { 416 kcf_sign_ops_params_t *sops = ¶ms->rp_u.sign_params; 417 418 sops->so_mech.cm_type = sops->so_framework_mechtype; 419 *mech1 = &sops->so_mech; 420 switch (optype) { 421 case KCF_OP_INIT: 422 *fg = CRYPTO_FG_SIGN; 423 break; 424 case KCF_OP_ATOMIC: 425 *fg = CRYPTO_FG_SIGN_ATOMIC; 426 break; 427 default: 428 ASSERT(optype == KCF_OP_SIGN_RECOVER_ATOMIC); 429 *fg = CRYPTO_FG_SIGN_RECOVER_ATOMIC; 430 } 431 break; 432 } 433 434 case KCF_OG_VERIFY: { 435 kcf_verify_ops_params_t *vops = ¶ms->rp_u.verify_params; 436 437 vops->vo_mech.cm_type = vops->vo_framework_mechtype; 438 *mech1 = &vops->vo_mech; 439 switch (optype) { 440 case KCF_OP_INIT: 441 *fg = CRYPTO_FG_VERIFY; 442 break; 443 case KCF_OP_ATOMIC: 444 *fg = CRYPTO_FG_VERIFY_ATOMIC; 445 break; 446 default: 447 ASSERT(optype == KCF_OP_VERIFY_RECOVER_ATOMIC); 448 *fg = CRYPTO_FG_VERIFY_RECOVER_ATOMIC; 449 } 450 break; 451 } 452 453 case KCF_OG_ENCRYPT: { 454 kcf_encrypt_ops_params_t *eops = ¶ms->rp_u.encrypt_params; 455 456 eops->eo_mech.cm_type = eops->eo_framework_mechtype; 457 *mech1 = &eops->eo_mech; 458 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT : 459 CRYPTO_FG_ENCRYPT_ATOMIC; 460 break; 461 } 462 463 case KCF_OG_DECRYPT: { 464 kcf_decrypt_ops_params_t *dcrops = ¶ms->rp_u.decrypt_params; 465 466 dcrops->dop_mech.cm_type = dcrops->dop_framework_mechtype; 467 *mech1 = &dcrops->dop_mech; 468 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DECRYPT : 469 CRYPTO_FG_DECRYPT_ATOMIC; 470 break; 471 } 472 473 case KCF_OG_ENCRYPT_MAC: { 474 kcf_encrypt_mac_ops_params_t *eops = 475 ¶ms->rp_u.encrypt_mac_params; 476 477 eops->em_encr_mech.cm_type = eops->em_framework_encr_mechtype; 478 *mech1 = &eops->em_encr_mech; 479 eops->em_mac_mech.cm_type = eops->em_framework_mac_mechtype; 480 *mech2 = &eops->em_mac_mech; 481 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT_MAC : 482 CRYPTO_FG_ENCRYPT_MAC_ATOMIC; 483 break; 484 } 485 486 case KCF_OG_MAC_DECRYPT: { 487 kcf_mac_decrypt_ops_params_t *dops = 488 ¶ms->rp_u.mac_decrypt_params; 489 490 dops->md_mac_mech.cm_type = dops->md_framework_mac_mechtype; 491 *mech1 = &dops->md_mac_mech; 492 dops->md_decr_mech.cm_type = dops->md_framework_decr_mechtype; 493 *mech2 = &dops->md_decr_mech; 494 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC_DECRYPT : 495 CRYPTO_FG_MAC_DECRYPT_ATOMIC; 496 break; 497 } 498 499 default: 500 return (B_FALSE); 501 } 502 503 return (B_TRUE); 504 } 505 506 /* 507 * This routine is called when a request to a provider has failed 508 * with a recoverable error. This routine tries to find another provider 509 * and dispatches the request to the new provider, if one is available. 510 * We reuse the request structure. 511 * 512 * A return value of NULL from kcf_get_mech_provider() indicates 513 * we have tried the last provider. 514 */ 515 static int 516 kcf_resubmit_request(kcf_areq_node_t *areq) 517 { 518 int error = CRYPTO_FAILED; 519 kcf_context_t *ictx; 520 kcf_provider_desc_t *old_pd; 521 kcf_provider_desc_t *new_pd; 522 crypto_mechanism_t *mech1 = NULL, *mech2 = NULL; 523 crypto_mech_type_t prov_mt1, prov_mt2; 524 crypto_func_group_t fg; 525 526 if (!can_resubmit(areq, &mech1, &mech2, &fg)) 527 return (error); 528 529 old_pd = areq->an_provider; 530 /* 531 * Add old_pd to the list of providers already tried. 532 * We release the new hold on old_pd in kcf_free_triedlist(). 533 */ 534 if (kcf_insert_triedlist(&areq->an_tried_plist, old_pd, 535 KM_NOSLEEP | KCF_HOLD_PROV) == NULL) 536 return (error); 537 538 if (mech1 && !mech2) { 539 new_pd = kcf_get_mech_provider(mech1->cm_type, NULL, &error, 540 areq->an_tried_plist, fg, 541 (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), 0); 542 } else { 543 ASSERT(mech1 != NULL && mech2 != NULL); 544 545 new_pd = kcf_get_dual_provider(mech1, mech2, NULL, &prov_mt1, 546 &prov_mt2, &error, areq->an_tried_plist, fg, fg, 547 (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), 0); 548 } 549 550 if (new_pd == NULL) 551 return (error); 552 553 /* 554 * We reuse the old context by resetting provider specific 555 * fields in it. 556 */ 557 if ((ictx = areq->an_context) != NULL) { 558 crypto_ctx_t *ctx; 559 560 ASSERT(old_pd == ictx->kc_prov_desc); 561 KCF_PROV_REFRELE(ictx->kc_prov_desc); 562 KCF_PROV_REFHOLD(new_pd); 563 ictx->kc_prov_desc = new_pd; 564 565 ctx = &ictx->kc_glbl_ctx; 566 ctx->cc_provider = new_pd->pd_prov_handle; 567 ctx->cc_session = new_pd->pd_sid; 568 ctx->cc_provider_private = NULL; 569 } 570 571 /* We reuse areq. by resetting the provider and context fields. */ 572 KCF_PROV_REFRELE(old_pd); 573 KCF_PROV_REFHOLD(new_pd); 574 areq->an_provider = new_pd; 575 mutex_enter(&areq->an_lock); 576 areq->an_state = REQ_WAITING; 577 mutex_exit(&areq->an_lock); 578 579 switch (new_pd->pd_prov_type) { 580 case CRYPTO_SW_PROVIDER: 581 error = kcf_disp_sw_request(areq); 582 break; 583 584 case CRYPTO_HW_PROVIDER: { 585 taskq_t *taskq = new_pd->pd_taskq; 586 587 if (taskq_dispatch(taskq, process_req_hwp, areq, TQ_NOSLEEP) == 588 (taskqid_t)0) { 589 error = CRYPTO_HOST_MEMORY; 590 } else { 591 error = CRYPTO_QUEUED; 592 } 593 594 break; 595 } 596 } 597 598 KCF_PROV_REFRELE(new_pd); 599 return (error); 600 } 601 602 #define EMPTY_TASKQ(tq) ((tq)->tq_task.tqent_next == &(tq)->tq_task) 603 604 /* 605 * Routine called by both ioctl and k-api. The consumer should 606 * bundle the parameters into a kcf_req_params_t structure. A bunch 607 * of macros are available in ops_impl.h for this bundling. They are: 608 * 609 * KCF_WRAP_DIGEST_OPS_PARAMS() 610 * KCF_WRAP_MAC_OPS_PARAMS() 611 * KCF_WRAP_ENCRYPT_OPS_PARAMS() 612 * KCF_WRAP_DECRYPT_OPS_PARAMS() ... etc. 613 * 614 * It is the caller's responsibility to free the ctx argument when 615 * appropriate. See the KCF_CONTEXT_COND_RELEASE macro for details. 616 */ 617 int 618 kcf_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx, 619 crypto_call_req_t *crq, kcf_req_params_t *params, boolean_t cont) 620 { 621 int error; 622 kcf_areq_node_t *areq; 623 kcf_sreq_node_t *sreq; 624 kcf_context_t *kcf_ctx; 625 taskq_t *taskq; 626 kcf_prov_cpu_t *mp; 627 628 kcf_ctx = ctx ? (kcf_context_t *)ctx->cc_framework_private : NULL; 629 630 /* Synchronous cases */ 631 if (crq == NULL) { 632 switch (pd->pd_prov_type) { 633 case CRYPTO_SW_PROVIDER: 634 error = common_submit_request(pd, ctx, params, 635 KCF_RHNDL(KM_SLEEP)); 636 break; 637 638 case CRYPTO_HW_PROVIDER: 639 taskq = pd->pd_taskq; 640 641 /* 642 * Special case for CRYPTO_SYNCHRONOUS providers that 643 * never return a CRYPTO_QUEUED error. We skip any 644 * request allocation and call the SPI directly. 645 */ 646 if ((pd->pd_flags & CRYPTO_SYNCHRONOUS) && 647 EMPTY_TASKQ(taskq)) { 648 mp = &(pd->pd_percpu_bins[CPU_SEQID]); 649 KCF_PROV_JOB_HOLD(mp); 650 651 if (pd->pd_state == KCF_PROV_READY) { 652 error = common_submit_request(pd, ctx, 653 params, KCF_RHNDL(KM_SLEEP)); 654 KCF_PROV_JOB_RELE(mp); 655 ASSERT(error != CRYPTO_QUEUED); 656 break; 657 } 658 KCF_PROV_JOB_RELE(mp); 659 } 660 661 sreq = kmem_cache_alloc(kcf_sreq_cache, KM_SLEEP); 662 sreq->sn_state = REQ_ALLOCATED; 663 sreq->sn_rv = CRYPTO_FAILED; 664 sreq->sn_params = params; 665 666 /* 667 * Note that we do not need to hold the context 668 * for synchronous case as the context will never 669 * become invalid underneath us. We do not need to hold 670 * the provider here either as the caller has a hold. 671 */ 672 sreq->sn_context = kcf_ctx; 673 ASSERT(KCF_PROV_REFHELD(pd)); 674 sreq->sn_provider = pd; 675 676 ASSERT(taskq != NULL); 677 /* 678 * Call the SPI directly if the taskq is empty and the 679 * provider is not busy, else dispatch to the taskq. 680 * Calling directly is fine as this is the synchronous 681 * case. This is unlike the asynchronous case where we 682 * must always dispatch to the taskq. 683 */ 684 if (EMPTY_TASKQ(taskq) && 685 pd->pd_state == KCF_PROV_READY) { 686 process_req_hwp(sreq); 687 } else { 688 /* 689 * We can not tell from taskq_dispatch() return 690 * value if we exceeded maxalloc. Hence the 691 * check here. Since we are allowed to wait in 692 * the synchronous case, we wait for the taskq 693 * to become empty. 694 */ 695 if (taskq->tq_nalloc >= crypto_taskq_maxalloc) { 696 taskq_wait(taskq); 697 } 698 699 (void) taskq_dispatch(taskq, process_req_hwp, 700 sreq, TQ_SLEEP); 701 } 702 703 /* 704 * Wait for the notification to arrive, 705 * if the operation is not done yet. 706 * Bug# 4722589 will make the wait a cv_wait_sig(). 707 */ 708 mutex_enter(&sreq->sn_lock); 709 while (sreq->sn_state < REQ_DONE) 710 cv_wait(&sreq->sn_cv, &sreq->sn_lock); 711 mutex_exit(&sreq->sn_lock); 712 713 error = sreq->sn_rv; 714 kmem_cache_free(kcf_sreq_cache, sreq); 715 716 break; 717 718 default: 719 error = CRYPTO_FAILED; 720 break; 721 } 722 723 } else { /* Asynchronous cases */ 724 switch (pd->pd_prov_type) { 725 case CRYPTO_SW_PROVIDER: 726 if (!(crq->cr_flag & CRYPTO_ALWAYS_QUEUE)) { 727 /* 728 * This case has less overhead since there is 729 * no switching of context. 730 */ 731 error = common_submit_request(pd, ctx, params, 732 KCF_RHNDL(KM_NOSLEEP)); 733 } else { 734 /* 735 * CRYPTO_ALWAYS_QUEUE is set. We need to 736 * queue the request and return. 737 */ 738 areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, 739 params, cont); 740 if (areq == NULL) 741 error = CRYPTO_HOST_MEMORY; 742 else { 743 if (!(crq->cr_flag 744 & CRYPTO_SKIP_REQID)) { 745 /* 746 * Set the request handle. This handle 747 * is used for any crypto_cancel_req(9f) 748 * calls from the consumer. We have to 749 * do this before dispatching the 750 * request. 751 */ 752 crq->cr_reqid = kcf_reqid_insert(areq); 753 } 754 755 error = kcf_disp_sw_request(areq); 756 /* 757 * There is an error processing this 758 * request. Remove the handle and 759 * release the request structure. 760 */ 761 if (error != CRYPTO_QUEUED) { 762 if (!(crq->cr_flag 763 & CRYPTO_SKIP_REQID)) 764 kcf_reqid_delete(areq); 765 KCF_AREQ_REFRELE(areq); 766 } 767 } 768 } 769 break; 770 771 case CRYPTO_HW_PROVIDER: 772 /* 773 * We need to queue the request and return. 774 */ 775 areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, params, 776 cont); 777 if (areq == NULL) { 778 error = CRYPTO_HOST_MEMORY; 779 goto done; 780 } 781 782 taskq = pd->pd_taskq; 783 ASSERT(taskq != NULL); 784 /* 785 * We can not tell from taskq_dispatch() return 786 * value if we exceeded maxalloc. Hence the check 787 * here. 788 */ 789 if (taskq->tq_nalloc >= crypto_taskq_maxalloc) { 790 error = CRYPTO_BUSY; 791 KCF_AREQ_REFRELE(areq); 792 goto done; 793 } 794 795 if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) { 796 /* 797 * Set the request handle. This handle is used 798 * for any crypto_cancel_req(9f) calls from the 799 * consumer. We have to do this before dispatching 800 * the request. 801 */ 802 crq->cr_reqid = kcf_reqid_insert(areq); 803 } 804 805 if (taskq_dispatch(taskq, 806 process_req_hwp, areq, TQ_NOSLEEP) == 807 (taskqid_t)0) { 808 error = CRYPTO_HOST_MEMORY; 809 if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) 810 kcf_reqid_delete(areq); 811 KCF_AREQ_REFRELE(areq); 812 } else { 813 error = CRYPTO_QUEUED; 814 } 815 break; 816 817 default: 818 error = CRYPTO_FAILED; 819 break; 820 } 821 } 822 823 done: 824 return (error); 825 } 826 827 /* 828 * We're done with this framework context, so free it. Note that freeing 829 * framework context (kcf_context) frees the global context (crypto_ctx). 830 * 831 * The provider is responsible for freeing provider private context after a 832 * final or single operation and resetting the cc_provider_private field 833 * to NULL. It should do this before it notifies the framework of the 834 * completion. We still need to call KCF_PROV_FREE_CONTEXT to handle cases 835 * like crypto_cancel_ctx(9f). 836 */ 837 void 838 kcf_free_context(kcf_context_t *kcf_ctx) 839 { 840 kcf_provider_desc_t *pd = kcf_ctx->kc_prov_desc; 841 crypto_ctx_t *gctx = &kcf_ctx->kc_glbl_ctx; 842 kcf_context_t *kcf_secondctx = kcf_ctx->kc_secondctx; 843 kcf_prov_cpu_t *mp; 844 845 /* Release the second context, if any */ 846 847 if (kcf_secondctx != NULL) 848 KCF_CONTEXT_REFRELE(kcf_secondctx); 849 850 if (gctx->cc_provider_private != NULL) { 851 mutex_enter(&pd->pd_lock); 852 if (!KCF_IS_PROV_REMOVED(pd)) { 853 /* 854 * Increment the provider's internal refcnt so it 855 * doesn't unregister from the framework while 856 * we're calling the entry point. 857 */ 858 mp = &(pd->pd_percpu_bins[CPU_SEQID]); 859 KCF_PROV_JOB_HOLD(mp); 860 mutex_exit(&pd->pd_lock); 861 (void) KCF_PROV_FREE_CONTEXT(pd, gctx); 862 KCF_PROV_JOB_RELE(mp); 863 } else { 864 mutex_exit(&pd->pd_lock); 865 } 866 } 867 868 /* kcf_ctx->kc_prov_desc has a hold on pd */ 869 KCF_PROV_REFRELE(kcf_ctx->kc_prov_desc); 870 871 /* check if this context is shared with a software provider */ 872 if ((gctx->cc_flags & CRYPTO_INIT_OPSTATE) && 873 kcf_ctx->kc_sw_prov_desc != NULL) { 874 KCF_PROV_REFRELE(kcf_ctx->kc_sw_prov_desc); 875 } 876 877 kmem_cache_free(kcf_context_cache, kcf_ctx); 878 } 879 880 /* 881 * Free the request after releasing all the holds. 882 */ 883 void 884 kcf_free_req(kcf_areq_node_t *areq) 885 { 886 KCF_PROV_REFRELE(areq->an_provider); 887 if (areq->an_context != NULL) 888 KCF_CONTEXT_REFRELE(areq->an_context); 889 890 if (areq->an_tried_plist != NULL) 891 kcf_free_triedlist(areq->an_tried_plist); 892 kmem_cache_free(kcf_areq_cache, areq); 893 } 894 895 /* 896 * Utility routine to remove a request from the chain of requests 897 * hanging off a context. 898 */ 899 void 900 kcf_removereq_in_ctxchain(kcf_context_t *ictx, kcf_areq_node_t *areq) 901 { 902 kcf_areq_node_t *cur, *prev; 903 904 /* 905 * Get context lock, search for areq in the chain and remove it. 906 */ 907 ASSERT(ictx != NULL); 908 mutex_enter(&ictx->kc_in_use_lock); 909 prev = cur = ictx->kc_req_chain_first; 910 911 while (cur != NULL) { 912 if (cur == areq) { 913 if (prev == cur) { 914 if ((ictx->kc_req_chain_first = 915 cur->an_ctxchain_next) == NULL) 916 ictx->kc_req_chain_last = NULL; 917 } else { 918 if (cur == ictx->kc_req_chain_last) 919 ictx->kc_req_chain_last = prev; 920 prev->an_ctxchain_next = cur->an_ctxchain_next; 921 } 922 923 break; 924 } 925 prev = cur; 926 cur = cur->an_ctxchain_next; 927 } 928 mutex_exit(&ictx->kc_in_use_lock); 929 } 930 931 /* 932 * Remove the specified node from the global software queue. 933 * 934 * The caller must hold the queue lock and request lock (an_lock). 935 */ 936 void 937 kcf_remove_node(kcf_areq_node_t *node) 938 { 939 kcf_areq_node_t *nextp = node->an_next; 940 kcf_areq_node_t *prevp = node->an_prev; 941 942 ASSERT(mutex_owned(&gswq->gs_lock)); 943 944 if (nextp != NULL) 945 nextp->an_prev = prevp; 946 else 947 gswq->gs_last = prevp; 948 949 if (prevp != NULL) 950 prevp->an_next = nextp; 951 else 952 gswq->gs_first = nextp; 953 954 ASSERT(mutex_owned(&node->an_lock)); 955 node->an_state = REQ_CANCELED; 956 } 957 958 /* 959 * Remove and return the first node in the global software queue. 960 * 961 * The caller must hold the queue lock. 962 */ 963 static kcf_areq_node_t * 964 kcf_dequeue() 965 { 966 kcf_areq_node_t *tnode = NULL; 967 968 ASSERT(mutex_owned(&gswq->gs_lock)); 969 if ((tnode = gswq->gs_first) == NULL) { 970 return (NULL); 971 } else { 972 ASSERT(gswq->gs_first->an_prev == NULL); 973 gswq->gs_first = tnode->an_next; 974 if (tnode->an_next == NULL) 975 gswq->gs_last = NULL; 976 else 977 tnode->an_next->an_prev = NULL; 978 } 979 980 gswq->gs_njobs--; 981 return (tnode); 982 } 983 984 /* 985 * Add the request node to the end of the global software queue. 986 * 987 * The caller should not hold the queue lock. Returns 0 if the 988 * request is successfully queued. Returns CRYPTO_BUSY if the limit 989 * on the number of jobs is exceeded. 990 */ 991 static int 992 kcf_enqueue(kcf_areq_node_t *node) 993 { 994 kcf_areq_node_t *tnode; 995 996 mutex_enter(&gswq->gs_lock); 997 998 if (gswq->gs_njobs >= gswq->gs_maxjobs) { 999 mutex_exit(&gswq->gs_lock); 1000 return (CRYPTO_BUSY); 1001 } 1002 1003 if (gswq->gs_last == NULL) { 1004 gswq->gs_first = gswq->gs_last = node; 1005 } else { 1006 ASSERT(gswq->gs_last->an_next == NULL); 1007 tnode = gswq->gs_last; 1008 tnode->an_next = node; 1009 gswq->gs_last = node; 1010 node->an_prev = tnode; 1011 } 1012 1013 gswq->gs_njobs++; 1014 1015 /* an_lock not needed here as we hold gs_lock */ 1016 node->an_state = REQ_WAITING; 1017 1018 mutex_exit(&gswq->gs_lock); 1019 1020 return (0); 1021 } 1022 1023 /* 1024 * Decrement the thread pool count and signal the failover 1025 * thread if we are the last one out. 1026 */ 1027 static void 1028 kcf_decrcnt_andsignal() 1029 { 1030 KCF_ATOMIC_DECR(kcfpool->kp_threads); 1031 1032 mutex_enter(&kcfpool->kp_thread_lock); 1033 if (kcfpool->kp_threads == 0) 1034 cv_signal(&kcfpool->kp_nothr_cv); 1035 mutex_exit(&kcfpool->kp_thread_lock); 1036 } 1037 1038 /* 1039 * Function run by a thread from kcfpool to work on global software queue. 1040 * It is called from ioctl(CRYPTO_POOL_RUN, ...). 1041 */ 1042 int 1043 kcf_svc_do_run(void) 1044 { 1045 int error = 0; 1046 clock_t rv; 1047 clock_t timeout_val; 1048 kcf_areq_node_t *req; 1049 kcf_context_t *ictx; 1050 kcf_provider_desc_t *pd; 1051 1052 KCF_ATOMIC_INCR(kcfpool->kp_threads); 1053 1054 for (;;) { 1055 mutex_enter(&gswq->gs_lock); 1056 1057 while ((req = kcf_dequeue()) == NULL) { 1058 timeout_val = ddi_get_lbolt() + 1059 drv_usectohz(kcf_idlethr_timeout); 1060 1061 KCF_ATOMIC_INCR(kcfpool->kp_idlethreads); 1062 rv = cv_timedwait_sig(&gswq->gs_cv, &gswq->gs_lock, 1063 timeout_val); 1064 KCF_ATOMIC_DECR(kcfpool->kp_idlethreads); 1065 1066 switch (rv) { 1067 case 0: 1068 /* 1069 * A signal (as in kill(2)) is pending. We did 1070 * not get any cv_signal(). 1071 */ 1072 kcf_decrcnt_andsignal(); 1073 mutex_exit(&gswq->gs_lock); 1074 return (EINTR); 1075 1076 case -1: 1077 /* 1078 * Timed out and we are not signaled. Let us 1079 * see if this thread should exit. We should 1080 * keep at least kcf_minthreads. 1081 */ 1082 if (kcfpool->kp_threads > kcf_minthreads) { 1083 kcf_decrcnt_andsignal(); 1084 mutex_exit(&gswq->gs_lock); 1085 return (0); 1086 } 1087 1088 /* Resume the wait for work */ 1089 break; 1090 1091 default: 1092 /* 1093 * We are signaled to work on the queue. 1094 */ 1095 break; 1096 } 1097 } 1098 1099 mutex_exit(&gswq->gs_lock); 1100 1101 ictx = req->an_context; 1102 if (ictx == NULL) { /* Context-less operation */ 1103 pd = req->an_provider; 1104 error = common_submit_request(pd, NULL, 1105 &req->an_params, req); 1106 kcf_aop_done(req, error); 1107 continue; 1108 } 1109 1110 /* 1111 * We check if we can work on the request now. 1112 * Solaris does not guarantee any order on how the threads 1113 * are scheduled or how the waiters on a mutex are chosen. 1114 * So, we need to maintain our own order. 1115 * 1116 * is_my_turn is set to B_TRUE initially for a request when 1117 * it is enqueued and there are no other requests 1118 * for that context. Note that a thread sleeping on 1119 * an_turn_cv is not counted as an idle thread. This is 1120 * because we define an idle thread as one that sleeps on the 1121 * global queue waiting for new requests. 1122 */ 1123 mutex_enter(&req->an_lock); 1124 while (req->an_is_my_turn == B_FALSE) { 1125 KCF_ATOMIC_INCR(kcfpool->kp_blockedthreads); 1126 cv_wait(&req->an_turn_cv, &req->an_lock); 1127 KCF_ATOMIC_DECR(kcfpool->kp_blockedthreads); 1128 } 1129 1130 req->an_state = REQ_INPROGRESS; 1131 mutex_exit(&req->an_lock); 1132 1133 pd = ictx->kc_prov_desc; 1134 ASSERT(pd == req->an_provider); 1135 error = common_submit_request(pd, &ictx->kc_glbl_ctx, 1136 &req->an_params, req); 1137 1138 kcf_aop_done(req, error); 1139 } 1140 } 1141 1142 /* 1143 * kmem_cache_alloc constructor for sync request structure. 1144 */ 1145 /* ARGSUSED */ 1146 static int 1147 kcf_sreq_cache_constructor(void *buf, void *cdrarg, int kmflags) 1148 { 1149 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf; 1150 1151 sreq->sn_type = CRYPTO_SYNCH; 1152 cv_init(&sreq->sn_cv, NULL, CV_DEFAULT, NULL); 1153 mutex_init(&sreq->sn_lock, NULL, MUTEX_DEFAULT, NULL); 1154 1155 return (0); 1156 } 1157 1158 /* ARGSUSED */ 1159 static void 1160 kcf_sreq_cache_destructor(void *buf, void *cdrarg) 1161 { 1162 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf; 1163 1164 mutex_destroy(&sreq->sn_lock); 1165 cv_destroy(&sreq->sn_cv); 1166 } 1167 1168 /* 1169 * kmem_cache_alloc constructor for async request structure. 1170 */ 1171 /* ARGSUSED */ 1172 static int 1173 kcf_areq_cache_constructor(void *buf, void *cdrarg, int kmflags) 1174 { 1175 kcf_areq_node_t *areq = (kcf_areq_node_t *)buf; 1176 1177 areq->an_type = CRYPTO_ASYNCH; 1178 areq->an_refcnt = 0; 1179 mutex_init(&areq->an_lock, NULL, MUTEX_DEFAULT, NULL); 1180 cv_init(&areq->an_done, NULL, CV_DEFAULT, NULL); 1181 cv_init(&areq->an_turn_cv, NULL, CV_DEFAULT, NULL); 1182 1183 return (0); 1184 } 1185 1186 /* ARGSUSED */ 1187 static void 1188 kcf_areq_cache_destructor(void *buf, void *cdrarg) 1189 { 1190 kcf_areq_node_t *areq = (kcf_areq_node_t *)buf; 1191 1192 ASSERT(areq->an_refcnt == 0); 1193 mutex_destroy(&areq->an_lock); 1194 cv_destroy(&areq->an_done); 1195 cv_destroy(&areq->an_turn_cv); 1196 } 1197 1198 /* 1199 * kmem_cache_alloc constructor for kcf_context structure. 1200 */ 1201 /* ARGSUSED */ 1202 static int 1203 kcf_context_cache_constructor(void *buf, void *cdrarg, int kmflags) 1204 { 1205 kcf_context_t *kctx = (kcf_context_t *)buf; 1206 1207 kctx->kc_refcnt = 0; 1208 mutex_init(&kctx->kc_in_use_lock, NULL, MUTEX_DEFAULT, NULL); 1209 1210 return (0); 1211 } 1212 1213 /* ARGSUSED */ 1214 static void 1215 kcf_context_cache_destructor(void *buf, void *cdrarg) 1216 { 1217 kcf_context_t *kctx = (kcf_context_t *)buf; 1218 1219 ASSERT(kctx->kc_refcnt == 0); 1220 mutex_destroy(&kctx->kc_in_use_lock); 1221 } 1222 1223 /* 1224 * Creates and initializes all the structures needed by the framework. 1225 */ 1226 void 1227 kcf_sched_init(void) 1228 { 1229 int i; 1230 kcf_reqid_table_t *rt; 1231 1232 /* 1233 * Create all the kmem caches needed by the framework. We set the 1234 * align argument to 64, to get a slab aligned to 64-byte as well as 1235 * have the objects (cache_chunksize) to be a 64-byte multiple. 1236 * This helps to avoid false sharing as this is the size of the 1237 * CPU cache line. 1238 */ 1239 kcf_sreq_cache = kmem_cache_create("kcf_sreq_cache", 1240 sizeof (struct kcf_sreq_node), 64, kcf_sreq_cache_constructor, 1241 kcf_sreq_cache_destructor, NULL, NULL, NULL, 0); 1242 1243 kcf_areq_cache = kmem_cache_create("kcf_areq_cache", 1244 sizeof (struct kcf_areq_node), 64, kcf_areq_cache_constructor, 1245 kcf_areq_cache_destructor, NULL, NULL, NULL, 0); 1246 1247 kcf_context_cache = kmem_cache_create("kcf_context_cache", 1248 sizeof (struct kcf_context), 64, kcf_context_cache_constructor, 1249 kcf_context_cache_destructor, NULL, NULL, NULL, 0); 1250 1251 mutex_init(&kcf_dh_lock, NULL, MUTEX_DEFAULT, NULL); 1252 1253 gswq = kmem_alloc(sizeof (kcf_global_swq_t), KM_SLEEP); 1254 1255 mutex_init(&gswq->gs_lock, NULL, MUTEX_DEFAULT, NULL); 1256 cv_init(&gswq->gs_cv, NULL, CV_DEFAULT, NULL); 1257 gswq->gs_njobs = 0; 1258 gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc; 1259 gswq->gs_first = gswq->gs_last = NULL; 1260 1261 /* Initialize the global reqid table */ 1262 for (i = 0; i < REQID_TABLES; i++) { 1263 rt = kmem_zalloc(sizeof (kcf_reqid_table_t), KM_SLEEP); 1264 kcf_reqid_table[i] = rt; 1265 mutex_init(&rt->rt_lock, NULL, MUTEX_DEFAULT, NULL); 1266 rt->rt_curid = i; 1267 } 1268 1269 /* Allocate and initialize the thread pool */ 1270 kcfpool_alloc(); 1271 1272 /* Initialize the event notification list variables */ 1273 mutex_init(&ntfy_list_lock, NULL, MUTEX_DEFAULT, NULL); 1274 cv_init(&ntfy_list_cv, NULL, CV_DEFAULT, NULL); 1275 1276 /* Initialize the crypto_bufcall list variables */ 1277 mutex_init(&cbuf_list_lock, NULL, MUTEX_DEFAULT, NULL); 1278 cv_init(&cbuf_list_cv, NULL, CV_DEFAULT, NULL); 1279 1280 /* Create the kcf kstat */ 1281 kcf_misc_kstat = kstat_create("kcf", 0, "framework_stats", "crypto", 1282 KSTAT_TYPE_NAMED, sizeof (kcf_stats_t) / sizeof (kstat_named_t), 1283 KSTAT_FLAG_VIRTUAL); 1284 1285 if (kcf_misc_kstat != NULL) { 1286 kcf_misc_kstat->ks_data = &kcf_ksdata; 1287 kcf_misc_kstat->ks_update = kcf_misc_kstat_update; 1288 kstat_install(kcf_misc_kstat); 1289 } 1290 } 1291 1292 /* 1293 * This routine should only be called by drv/cryptoadm. 1294 * 1295 * kcf_sched_running flag isn't protected by a lock. But, we are safe because 1296 * the first thread ("cryptoadm refresh") calling this routine during 1297 * boot time completes before any other thread that can call this routine. 1298 */ 1299 void 1300 kcf_sched_start(void) 1301 { 1302 if (kcf_sched_running) 1303 return; 1304 1305 /* Start the failover kernel thread for now */ 1306 (void) thread_create(NULL, 0, &kcf_failover_thread, 0, 0, &p0, 1307 TS_RUN, minclsyspri); 1308 1309 /* Start the background processing thread. */ 1310 (void) thread_create(NULL, 0, &crypto_bufcall_service, 0, 0, &p0, 1311 TS_RUN, minclsyspri); 1312 1313 kcf_sched_running = B_TRUE; 1314 } 1315 1316 /* 1317 * Signal the waiting sync client. 1318 */ 1319 void 1320 kcf_sop_done(kcf_sreq_node_t *sreq, int error) 1321 { 1322 mutex_enter(&sreq->sn_lock); 1323 sreq->sn_state = REQ_DONE; 1324 sreq->sn_rv = error; 1325 cv_signal(&sreq->sn_cv); 1326 mutex_exit(&sreq->sn_lock); 1327 } 1328 1329 /* 1330 * Callback the async client with the operation status. 1331 * We free the async request node and possibly the context. 1332 * We also handle any chain of requests hanging off of 1333 * the context. 1334 */ 1335 void 1336 kcf_aop_done(kcf_areq_node_t *areq, int error) 1337 { 1338 kcf_op_type_t optype; 1339 boolean_t skip_notify = B_FALSE; 1340 kcf_context_t *ictx; 1341 kcf_areq_node_t *nextreq; 1342 1343 /* 1344 * Handle recoverable errors. This has to be done first 1345 * before doing any thing else in this routine so that 1346 * we do not change the state of the request. 1347 */ 1348 if (error != CRYPTO_SUCCESS && IS_RECOVERABLE(error)) { 1349 /* 1350 * We try another provider, if one is available. Else 1351 * we continue with the failure notification to the 1352 * client. 1353 */ 1354 if (kcf_resubmit_request(areq) == CRYPTO_QUEUED) 1355 return; 1356 } 1357 1358 mutex_enter(&areq->an_lock); 1359 areq->an_state = REQ_DONE; 1360 mutex_exit(&areq->an_lock); 1361 1362 optype = (&areq->an_params)->rp_optype; 1363 if ((ictx = areq->an_context) != NULL) { 1364 /* 1365 * A request after it is removed from the request 1366 * queue, still stays on a chain of requests hanging 1367 * of its context structure. It needs to be removed 1368 * from this chain at this point. 1369 */ 1370 mutex_enter(&ictx->kc_in_use_lock); 1371 nextreq = areq->an_ctxchain_next; 1372 if (nextreq != NULL) { 1373 mutex_enter(&nextreq->an_lock); 1374 nextreq->an_is_my_turn = B_TRUE; 1375 cv_signal(&nextreq->an_turn_cv); 1376 mutex_exit(&nextreq->an_lock); 1377 } 1378 1379 ictx->kc_req_chain_first = nextreq; 1380 if (nextreq == NULL) 1381 ictx->kc_req_chain_last = NULL; 1382 mutex_exit(&ictx->kc_in_use_lock); 1383 1384 if (IS_SINGLE_OP(optype) || IS_FINAL_OP(optype)) { 1385 ASSERT(nextreq == NULL); 1386 KCF_CONTEXT_REFRELE(ictx); 1387 } else if (error != CRYPTO_SUCCESS && IS_INIT_OP(optype)) { 1388 /* 1389 * NOTE - We do not release the context in case of update 1390 * operations. We require the consumer to free it explicitly, 1391 * in case it wants to abandon an update operation. This is done 1392 * as there may be mechanisms in ECB mode that can continue 1393 * even if an operation on a block fails. 1394 */ 1395 KCF_CONTEXT_REFRELE(ictx); 1396 } 1397 } 1398 1399 /* Deal with the internal continuation to this request first */ 1400 1401 if (areq->an_isdual) { 1402 kcf_dual_req_t *next_arg; 1403 next_arg = (kcf_dual_req_t *)areq->an_reqarg.cr_callback_arg; 1404 next_arg->kr_areq = areq; 1405 KCF_AREQ_REFHOLD(areq); 1406 areq->an_isdual = B_FALSE; 1407 1408 NOTIFY_CLIENT(areq, error); 1409 return; 1410 } 1411 1412 /* 1413 * If CRYPTO_NOTIFY_OPDONE flag is set, we should notify 1414 * always. If this flag is clear, we skip the notification 1415 * provided there are no errors. We check this flag for only 1416 * init or update operations. It is ignored for single, final or 1417 * atomic operations. 1418 */ 1419 skip_notify = (IS_UPDATE_OP(optype) || IS_INIT_OP(optype)) && 1420 (!(areq->an_reqarg.cr_flag & CRYPTO_NOTIFY_OPDONE)) && 1421 (error == CRYPTO_SUCCESS); 1422 1423 if (!skip_notify) { 1424 NOTIFY_CLIENT(areq, error); 1425 } 1426 1427 if (!(areq->an_reqarg.cr_flag & CRYPTO_SKIP_REQID)) 1428 kcf_reqid_delete(areq); 1429 1430 KCF_AREQ_REFRELE(areq); 1431 } 1432 1433 /* 1434 * Allocate the thread pool and initialize all the fields. 1435 */ 1436 static void 1437 kcfpool_alloc() 1438 { 1439 kcfpool = kmem_alloc(sizeof (kcf_pool_t), KM_SLEEP); 1440 1441 kcfpool->kp_threads = kcfpool->kp_idlethreads = 0; 1442 kcfpool->kp_blockedthreads = 0; 1443 kcfpool->kp_signal_create_thread = B_FALSE; 1444 kcfpool->kp_nthrs = 0; 1445 kcfpool->kp_user_waiting = B_FALSE; 1446 1447 mutex_init(&kcfpool->kp_thread_lock, NULL, MUTEX_DEFAULT, NULL); 1448 cv_init(&kcfpool->kp_nothr_cv, NULL, CV_DEFAULT, NULL); 1449 1450 mutex_init(&kcfpool->kp_user_lock, NULL, MUTEX_DEFAULT, NULL); 1451 cv_init(&kcfpool->kp_user_cv, NULL, CV_DEFAULT, NULL); 1452 1453 kcf_idlethr_timeout = KCF_DEFAULT_THRTIMEOUT; 1454 } 1455 1456 /* 1457 * This function is run by the 'creator' thread in the pool. 1458 * It is called from ioctl(CRYPTO_POOL_WAIT, ...). 1459 */ 1460 int 1461 kcf_svc_wait(int *nthrs) 1462 { 1463 clock_t rv; 1464 clock_t timeout_val; 1465 1466 if (kcfpool == NULL) 1467 return (ENOENT); 1468 1469 mutex_enter(&kcfpool->kp_user_lock); 1470 /* Check if there's already a user thread waiting on this kcfpool */ 1471 if (kcfpool->kp_user_waiting) { 1472 mutex_exit(&kcfpool->kp_user_lock); 1473 *nthrs = 0; 1474 return (EBUSY); 1475 } 1476 1477 kcfpool->kp_user_waiting = B_TRUE; 1478 1479 /* Go to sleep, waiting for the signaled flag. */ 1480 while (!kcfpool->kp_signal_create_thread) { 1481 timeout_val = ddi_get_lbolt() + 1482 drv_usectohz(kcf_idlethr_timeout); 1483 1484 rv = cv_timedwait_sig(&kcfpool->kp_user_cv, 1485 &kcfpool->kp_user_lock, timeout_val); 1486 switch (rv) { 1487 case 0: 1488 /* Interrupted, return to handle exit or signal */ 1489 kcfpool->kp_user_waiting = B_FALSE; 1490 kcfpool->kp_signal_create_thread = B_FALSE; 1491 mutex_exit(&kcfpool->kp_user_lock); 1492 /* 1493 * kcfd is exiting. Release the door and 1494 * invalidate it. 1495 */ 1496 mutex_enter(&kcf_dh_lock); 1497 if (kcf_dh != NULL) { 1498 door_ki_rele(kcf_dh); 1499 kcf_dh = NULL; 1500 } 1501 mutex_exit(&kcf_dh_lock); 1502 return (EINTR); 1503 1504 case -1: 1505 /* Timed out. Recalculate the min/max threads */ 1506 compute_min_max_threads(); 1507 if (kcf_need_provtab_walk) 1508 kcf_free_unregistered_provs(); 1509 break; 1510 1511 default: 1512 /* Worker thread did a cv_signal() */ 1513 break; 1514 } 1515 } 1516 1517 kcfpool->kp_signal_create_thread = B_FALSE; 1518 kcfpool->kp_user_waiting = B_FALSE; 1519 1520 *nthrs = kcfpool->kp_nthrs; 1521 mutex_exit(&kcfpool->kp_user_lock); 1522 1523 /* Return to userland for possible thread creation. */ 1524 return (0); 1525 } 1526 1527 1528 /* 1529 * This routine introduces a locking order for gswq->gs_lock followed 1530 * by cpu_lock. 1531 * This means that no consumer of the k-api should hold cpu_lock when calling 1532 * k-api routines. 1533 */ 1534 static void 1535 compute_min_max_threads() 1536 { 1537 mutex_enter(&gswq->gs_lock); 1538 mutex_enter(&cpu_lock); 1539 kcf_minthreads = curthread->t_cpupart->cp_ncpus; 1540 mutex_exit(&cpu_lock); 1541 kcf_maxthreads = kcf_thr_multiple * kcf_minthreads; 1542 gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc; 1543 mutex_exit(&gswq->gs_lock); 1544 } 1545 1546 /* 1547 * This is the main routine of the failover kernel thread. 1548 * If there are any threads in the pool we sleep. The last thread in the 1549 * pool to exit will signal us to get to work. We get back to sleep 1550 * once we detect that the pool has threads. 1551 * 1552 * Note that in the hand-off from us to a pool thread we get to run once. 1553 * Since this hand-off is a rare event this should be fine. 1554 */ 1555 static void 1556 kcf_failover_thread() 1557 { 1558 int error = 0; 1559 kcf_context_t *ictx; 1560 kcf_areq_node_t *req; 1561 callb_cpr_t cpr_info; 1562 kmutex_t cpr_lock; 1563 static boolean_t is_logged = B_FALSE; 1564 1565 mutex_init(&cpr_lock, NULL, MUTEX_DEFAULT, NULL); 1566 CALLB_CPR_INIT(&cpr_info, &cpr_lock, callb_generic_cpr, 1567 "kcf_failover_thread"); 1568 1569 for (;;) { 1570 /* 1571 * Wait if there are any threads are in the pool. 1572 */ 1573 if (kcfpool->kp_threads > 0) { 1574 mutex_enter(&cpr_lock); 1575 CALLB_CPR_SAFE_BEGIN(&cpr_info); 1576 mutex_exit(&cpr_lock); 1577 1578 mutex_enter(&kcfpool->kp_thread_lock); 1579 cv_wait(&kcfpool->kp_nothr_cv, 1580 &kcfpool->kp_thread_lock); 1581 mutex_exit(&kcfpool->kp_thread_lock); 1582 1583 mutex_enter(&cpr_lock); 1584 CALLB_CPR_SAFE_END(&cpr_info, &cpr_lock); 1585 mutex_exit(&cpr_lock); 1586 is_logged = B_FALSE; 1587 } 1588 1589 /* 1590 * Get the requests from the queue and wait if needed. 1591 */ 1592 mutex_enter(&gswq->gs_lock); 1593 1594 while ((req = kcf_dequeue()) == NULL) { 1595 mutex_enter(&cpr_lock); 1596 CALLB_CPR_SAFE_BEGIN(&cpr_info); 1597 mutex_exit(&cpr_lock); 1598 1599 KCF_ATOMIC_INCR(kcfpool->kp_idlethreads); 1600 cv_wait(&gswq->gs_cv, &gswq->gs_lock); 1601 KCF_ATOMIC_DECR(kcfpool->kp_idlethreads); 1602 1603 mutex_enter(&cpr_lock); 1604 CALLB_CPR_SAFE_END(&cpr_info, &cpr_lock); 1605 mutex_exit(&cpr_lock); 1606 } 1607 1608 mutex_exit(&gswq->gs_lock); 1609 1610 /* 1611 * We check the kp_threads since kcfd could have started 1612 * while we are waiting on the global software queue. 1613 */ 1614 if (kcfpool->kp_threads <= 0 && !is_logged) { 1615 cmn_err(CE_WARN, "kcfd is not running. Please check " 1616 "and restart kcfd. Using the failover kernel " 1617 "thread for now.\n"); 1618 is_logged = B_TRUE; 1619 } 1620 1621 /* 1622 * Get to work on the request. 1623 */ 1624 ictx = req->an_context; 1625 mutex_enter(&req->an_lock); 1626 req->an_state = REQ_INPROGRESS; 1627 mutex_exit(&req->an_lock); 1628 1629 error = common_submit_request(req->an_provider, ictx ? 1630 &ictx->kc_glbl_ctx : NULL, &req->an_params, req); 1631 1632 kcf_aop_done(req, error); 1633 } 1634 } 1635 1636 /* 1637 * Insert the async request in the hash table after assigning it 1638 * an ID. Returns the ID. 1639 * 1640 * The ID is used by the caller to pass as an argument to a 1641 * cancel_req() routine later. 1642 */ 1643 static crypto_req_id_t 1644 kcf_reqid_insert(kcf_areq_node_t *areq) 1645 { 1646 int indx; 1647 crypto_req_id_t id; 1648 kcf_areq_node_t *headp; 1649 kcf_reqid_table_t *rt = 1650 kcf_reqid_table[CPU->cpu_seqid & REQID_TABLE_MASK]; 1651 1652 mutex_enter(&rt->rt_lock); 1653 1654 rt->rt_curid = id = 1655 (rt->rt_curid - REQID_COUNTER_LOW) | REQID_COUNTER_HIGH; 1656 SET_REQID(areq, id); 1657 indx = REQID_HASH(id); 1658 headp = areq->an_idnext = rt->rt_idhash[indx]; 1659 areq->an_idprev = NULL; 1660 if (headp != NULL) 1661 headp->an_idprev = areq; 1662 1663 rt->rt_idhash[indx] = areq; 1664 mutex_exit(&rt->rt_lock); 1665 1666 return (id); 1667 } 1668 1669 /* 1670 * Delete the async request from the hash table. 1671 */ 1672 static void 1673 kcf_reqid_delete(kcf_areq_node_t *areq) 1674 { 1675 int indx; 1676 kcf_areq_node_t *nextp, *prevp; 1677 crypto_req_id_t id = GET_REQID(areq); 1678 kcf_reqid_table_t *rt; 1679 1680 rt = kcf_reqid_table[id & REQID_TABLE_MASK]; 1681 indx = REQID_HASH(id); 1682 1683 mutex_enter(&rt->rt_lock); 1684 1685 nextp = areq->an_idnext; 1686 prevp = areq->an_idprev; 1687 if (nextp != NULL) 1688 nextp->an_idprev = prevp; 1689 if (prevp != NULL) 1690 prevp->an_idnext = nextp; 1691 else 1692 rt->rt_idhash[indx] = nextp; 1693 1694 SET_REQID(areq, 0); 1695 cv_broadcast(&areq->an_done); 1696 1697 mutex_exit(&rt->rt_lock); 1698 } 1699 1700 /* 1701 * Cancel a single asynchronous request. 1702 * 1703 * We guarantee that no problems will result from calling 1704 * crypto_cancel_req() for a request which is either running, or 1705 * has already completed. We remove the request from any queues 1706 * if it is possible. We wait for request completion if the 1707 * request is dispatched to a provider. 1708 * 1709 * Calling context: 1710 * Can be called from user context only. 1711 * 1712 * NOTE: We acquire the following locks in this routine (in order): 1713 * - rt_lock (kcf_reqid_table_t) 1714 * - gswq->gs_lock 1715 * - areq->an_lock 1716 * - ictx->kc_in_use_lock (from kcf_removereq_in_ctxchain()) 1717 * 1718 * This locking order MUST be maintained in code every where else. 1719 */ 1720 void 1721 crypto_cancel_req(crypto_req_id_t id) 1722 { 1723 int indx; 1724 kcf_areq_node_t *areq; 1725 kcf_provider_desc_t *pd; 1726 kcf_context_t *ictx; 1727 kcf_reqid_table_t *rt; 1728 1729 rt = kcf_reqid_table[id & REQID_TABLE_MASK]; 1730 indx = REQID_HASH(id); 1731 1732 mutex_enter(&rt->rt_lock); 1733 for (areq = rt->rt_idhash[indx]; areq; areq = areq->an_idnext) { 1734 if (GET_REQID(areq) == id) { 1735 /* 1736 * We found the request. It is either still waiting 1737 * in the framework queues or running at the provider. 1738 */ 1739 pd = areq->an_provider; 1740 ASSERT(pd != NULL); 1741 1742 switch (pd->pd_prov_type) { 1743 case CRYPTO_SW_PROVIDER: 1744 mutex_enter(&gswq->gs_lock); 1745 mutex_enter(&areq->an_lock); 1746 1747 /* This request can be safely canceled. */ 1748 if (areq->an_state <= REQ_WAITING) { 1749 /* Remove from gswq, global software queue. */ 1750 kcf_remove_node(areq); 1751 if ((ictx = areq->an_context) != NULL) 1752 kcf_removereq_in_ctxchain(ictx, areq); 1753 1754 mutex_exit(&areq->an_lock); 1755 mutex_exit(&gswq->gs_lock); 1756 mutex_exit(&rt->rt_lock); 1757 1758 /* Remove areq from hash table and free it. */ 1759 kcf_reqid_delete(areq); 1760 KCF_AREQ_REFRELE(areq); 1761 return; 1762 } 1763 1764 mutex_exit(&areq->an_lock); 1765 mutex_exit(&gswq->gs_lock); 1766 break; 1767 1768 case CRYPTO_HW_PROVIDER: 1769 /* 1770 * There is no interface to remove an entry 1771 * once it is on the taskq. So, we do not do 1772 * any thing for a hardware provider. 1773 */ 1774 break; 1775 } 1776 1777 /* 1778 * The request is running. Wait for the request completion 1779 * to notify us. 1780 */ 1781 KCF_AREQ_REFHOLD(areq); 1782 while (GET_REQID(areq) == id) 1783 cv_wait(&areq->an_done, &rt->rt_lock); 1784 KCF_AREQ_REFRELE(areq); 1785 break; 1786 } 1787 } 1788 1789 mutex_exit(&rt->rt_lock); 1790 } 1791 1792 /* 1793 * Cancel all asynchronous requests associated with the 1794 * passed in crypto context and free it. 1795 * 1796 * A client SHOULD NOT call this routine after calling a crypto_*_final 1797 * routine. This routine is called only during intermediate operations. 1798 * The client should not use the crypto context after this function returns 1799 * since we destroy it. 1800 * 1801 * Calling context: 1802 * Can be called from user context only. 1803 */ 1804 void 1805 crypto_cancel_ctx(crypto_context_t ctx) 1806 { 1807 kcf_context_t *ictx; 1808 kcf_areq_node_t *areq; 1809 1810 if (ctx == NULL) 1811 return; 1812 1813 ictx = (kcf_context_t *)((crypto_ctx_t *)ctx)->cc_framework_private; 1814 1815 mutex_enter(&ictx->kc_in_use_lock); 1816 1817 /* Walk the chain and cancel each request */ 1818 while ((areq = ictx->kc_req_chain_first) != NULL) { 1819 /* 1820 * We have to drop the lock here as we may have 1821 * to wait for request completion. We hold the 1822 * request before dropping the lock though, so that it 1823 * won't be freed underneath us. 1824 */ 1825 KCF_AREQ_REFHOLD(areq); 1826 mutex_exit(&ictx->kc_in_use_lock); 1827 1828 crypto_cancel_req(GET_REQID(areq)); 1829 KCF_AREQ_REFRELE(areq); 1830 1831 mutex_enter(&ictx->kc_in_use_lock); 1832 } 1833 1834 mutex_exit(&ictx->kc_in_use_lock); 1835 KCF_CONTEXT_REFRELE(ictx); 1836 } 1837 1838 /* 1839 * Update kstats. 1840 */ 1841 static int 1842 kcf_misc_kstat_update(kstat_t *ksp, int rw) 1843 { 1844 uint_t tcnt; 1845 kcf_stats_t *ks_data; 1846 1847 if (rw == KSTAT_WRITE) 1848 return (EACCES); 1849 1850 ks_data = ksp->ks_data; 1851 1852 ks_data->ks_thrs_in_pool.value.ui32 = kcfpool->kp_threads; 1853 /* 1854 * The failover thread is counted in kp_idlethreads in 1855 * some corner cases. This is done to avoid doing more checks 1856 * when submitting a request. We account for those cases below. 1857 */ 1858 if ((tcnt = kcfpool->kp_idlethreads) == (kcfpool->kp_threads + 1)) 1859 tcnt--; 1860 ks_data->ks_idle_thrs.value.ui32 = tcnt; 1861 ks_data->ks_minthrs.value.ui32 = kcf_minthreads; 1862 ks_data->ks_maxthrs.value.ui32 = kcf_maxthreads; 1863 ks_data->ks_swq_njobs.value.ui32 = gswq->gs_njobs; 1864 ks_data->ks_swq_maxjobs.value.ui32 = gswq->gs_maxjobs; 1865 ks_data->ks_taskq_threads.value.ui32 = crypto_taskq_threads; 1866 ks_data->ks_taskq_minalloc.value.ui32 = crypto_taskq_minalloc; 1867 ks_data->ks_taskq_maxalloc.value.ui32 = crypto_taskq_maxalloc; 1868 1869 return (0); 1870 } 1871 1872 /* 1873 * Allocate and initiatize a kcf_dual_req, used for saving the arguments of 1874 * a dual operation or an atomic operation that has to be internally 1875 * simulated with multiple single steps. 1876 * crq determines the memory allocation flags. 1877 */ 1878 1879 kcf_dual_req_t * 1880 kcf_alloc_req(crypto_call_req_t *crq) 1881 { 1882 kcf_dual_req_t *kcr; 1883 1884 kcr = kmem_alloc(sizeof (kcf_dual_req_t), KCF_KMFLAG(crq)); 1885 1886 if (kcr == NULL) 1887 return (NULL); 1888 1889 /* Copy the whole crypto_call_req struct, as it isn't persistant */ 1890 if (crq != NULL) 1891 kcr->kr_callreq = *crq; 1892 else 1893 bzero(&(kcr->kr_callreq), sizeof (crypto_call_req_t)); 1894 kcr->kr_areq = NULL; 1895 kcr->kr_saveoffset = 0; 1896 kcr->kr_savelen = 0; 1897 1898 return (kcr); 1899 } 1900 1901 /* 1902 * Callback routine for the next part of a simulated dual part. 1903 * Schedules the next step. 1904 * 1905 * This routine can be called from interrupt context. 1906 */ 1907 void 1908 kcf_next_req(void *next_req_arg, int status) 1909 { 1910 kcf_dual_req_t *next_req = (kcf_dual_req_t *)next_req_arg; 1911 kcf_req_params_t *params = &(next_req->kr_params); 1912 kcf_areq_node_t *areq = next_req->kr_areq; 1913 int error = status; 1914 kcf_provider_desc_t *pd; 1915 crypto_dual_data_t *ct; 1916 1917 /* Stop the processing if an error occured at this step */ 1918 if (error != CRYPTO_SUCCESS) { 1919 out: 1920 areq->an_reqarg = next_req->kr_callreq; 1921 KCF_AREQ_REFRELE(areq); 1922 kmem_free(next_req, sizeof (kcf_dual_req_t)); 1923 areq->an_isdual = B_FALSE; 1924 kcf_aop_done(areq, error); 1925 return; 1926 } 1927 1928 switch (params->rp_opgrp) { 1929 case KCF_OG_MAC: { 1930 1931 /* 1932 * The next req is submitted with the same reqid as the 1933 * first part. The consumer only got back that reqid, and 1934 * should still be able to cancel the operation during its 1935 * second step. 1936 */ 1937 kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params); 1938 crypto_ctx_template_t mac_tmpl; 1939 kcf_mech_entry_t *me; 1940 1941 ct = (crypto_dual_data_t *)mops->mo_data; 1942 mac_tmpl = (crypto_ctx_template_t)mops->mo_templ; 1943 1944 /* No expected recoverable failures, so no retry list */ 1945 pd = kcf_get_mech_provider(mops->mo_framework_mechtype, 1946 &me, &error, NULL, CRYPTO_FG_MAC_ATOMIC, 1947 (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), ct->dd_len2); 1948 1949 if (pd == NULL) { 1950 error = CRYPTO_MECH_NOT_SUPPORTED; 1951 goto out; 1952 } 1953 /* Validate the MAC context template here */ 1954 if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && 1955 (mac_tmpl != NULL)) { 1956 kcf_ctx_template_t *ctx_mac_tmpl; 1957 1958 ctx_mac_tmpl = (kcf_ctx_template_t *)mac_tmpl; 1959 1960 if (ctx_mac_tmpl->ct_generation != me->me_gen_swprov) { 1961 KCF_PROV_REFRELE(pd); 1962 error = CRYPTO_OLD_CTX_TEMPLATE; 1963 goto out; 1964 } 1965 mops->mo_templ = ctx_mac_tmpl->ct_prov_tmpl; 1966 } 1967 1968 break; 1969 } 1970 case KCF_OG_DECRYPT: { 1971 kcf_decrypt_ops_params_t *dcrops = 1972 &(params->rp_u.decrypt_params); 1973 1974 ct = (crypto_dual_data_t *)dcrops->dop_ciphertext; 1975 /* No expected recoverable failures, so no retry list */ 1976 pd = kcf_get_mech_provider(dcrops->dop_framework_mechtype, 1977 NULL, &error, NULL, CRYPTO_FG_DECRYPT_ATOMIC, 1978 (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), ct->dd_len1); 1979 1980 if (pd == NULL) { 1981 error = CRYPTO_MECH_NOT_SUPPORTED; 1982 goto out; 1983 } 1984 break; 1985 } 1986 } 1987 1988 /* The second step uses len2 and offset2 of the dual_data */ 1989 next_req->kr_saveoffset = ct->dd_offset1; 1990 next_req->kr_savelen = ct->dd_len1; 1991 ct->dd_offset1 = ct->dd_offset2; 1992 ct->dd_len1 = ct->dd_len2; 1993 1994 /* preserve if the caller is restricted */ 1995 if (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED) { 1996 areq->an_reqarg.cr_flag = CRYPTO_RESTRICTED; 1997 } else { 1998 areq->an_reqarg.cr_flag = 0; 1999 } 2000 2001 areq->an_reqarg.cr_callback_func = kcf_last_req; 2002 areq->an_reqarg.cr_callback_arg = next_req; 2003 areq->an_isdual = B_TRUE; 2004 2005 /* 2006 * We would like to call kcf_submit_request() here. But, 2007 * that is not possible as that routine allocates a new 2008 * kcf_areq_node_t request structure, while we need to 2009 * reuse the existing request structure. 2010 */ 2011 switch (pd->pd_prov_type) { 2012 case CRYPTO_SW_PROVIDER: 2013 error = common_submit_request(pd, NULL, params, 2014 KCF_RHNDL(KM_NOSLEEP)); 2015 break; 2016 2017 case CRYPTO_HW_PROVIDER: { 2018 kcf_provider_desc_t *old_pd; 2019 taskq_t *taskq = pd->pd_taskq; 2020 2021 /* 2022 * Set the params for the second step in the 2023 * dual-ops. 2024 */ 2025 areq->an_params = *params; 2026 old_pd = areq->an_provider; 2027 KCF_PROV_REFRELE(old_pd); 2028 KCF_PROV_REFHOLD(pd); 2029 areq->an_provider = pd; 2030 2031 /* 2032 * Note that we have to do a taskq_dispatch() 2033 * here as we may be in interrupt context. 2034 */ 2035 if (taskq_dispatch(taskq, process_req_hwp, areq, 2036 TQ_NOSLEEP) == (taskqid_t)0) { 2037 error = CRYPTO_HOST_MEMORY; 2038 } else { 2039 error = CRYPTO_QUEUED; 2040 } 2041 break; 2042 } 2043 } 2044 2045 /* 2046 * We have to release the holds on the request and the provider 2047 * in all cases. 2048 */ 2049 KCF_AREQ_REFRELE(areq); 2050 KCF_PROV_REFRELE(pd); 2051 2052 if (error != CRYPTO_QUEUED) { 2053 /* restore, clean up, and invoke the client's callback */ 2054 2055 ct->dd_offset1 = next_req->kr_saveoffset; 2056 ct->dd_len1 = next_req->kr_savelen; 2057 areq->an_reqarg = next_req->kr_callreq; 2058 kmem_free(next_req, sizeof (kcf_dual_req_t)); 2059 areq->an_isdual = B_FALSE; 2060 kcf_aop_done(areq, error); 2061 } 2062 } 2063 2064 /* 2065 * Last part of an emulated dual operation. 2066 * Clean up and restore ... 2067 */ 2068 void 2069 kcf_last_req(void *last_req_arg, int status) 2070 { 2071 kcf_dual_req_t *last_req = (kcf_dual_req_t *)last_req_arg; 2072 2073 kcf_req_params_t *params = &(last_req->kr_params); 2074 kcf_areq_node_t *areq = last_req->kr_areq; 2075 crypto_dual_data_t *ct; 2076 2077 switch (params->rp_opgrp) { 2078 case KCF_OG_MAC: { 2079 kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params); 2080 2081 ct = (crypto_dual_data_t *)mops->mo_data; 2082 break; 2083 } 2084 case KCF_OG_DECRYPT: { 2085 kcf_decrypt_ops_params_t *dcrops = 2086 &(params->rp_u.decrypt_params); 2087 2088 ct = (crypto_dual_data_t *)dcrops->dop_ciphertext; 2089 break; 2090 } 2091 } 2092 ct->dd_offset1 = last_req->kr_saveoffset; 2093 ct->dd_len1 = last_req->kr_savelen; 2094 2095 /* The submitter used kcf_last_req as its callback */ 2096 2097 if (areq == NULL) { 2098 crypto_call_req_t *cr = &last_req->kr_callreq; 2099 2100 (*(cr->cr_callback_func))(cr->cr_callback_arg, status); 2101 kmem_free(last_req, sizeof (kcf_dual_req_t)); 2102 return; 2103 } 2104 areq->an_reqarg = last_req->kr_callreq; 2105 KCF_AREQ_REFRELE(areq); 2106 kmem_free(last_req, sizeof (kcf_dual_req_t)); 2107 areq->an_isdual = B_FALSE; 2108 kcf_aop_done(areq, status); 2109 } 2110