1 /* $OpenBSD: via.c,v 1.8 2006/11/17 07:47:56 tom Exp $ */ 2 /* $NetBSD: via_padlock.c,v 1.13 2010/04/22 21:47:32 jym Exp $ */ 3 4 /*- 5 * Copyright (c) 2003 Jason Wright 6 * Copyright (c) 2003, 2004 Theo de Raadt 7 * All rights reserved. 8 * 9 * Permission to use, copy, modify, and distribute this software for any 10 * purpose with or without fee is hereby granted, provided that the above 11 * copyright notice and this permission notice appear in all copies. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 14 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 15 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 16 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 17 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 18 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 19 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 20 */ 21 22 #include <sys/cdefs.h> 23 __KERNEL_RCSID(0, "$NetBSD: via_padlock.c,v 1.13 2010/04/22 21:47:32 jym Exp $"); 24 25 #include "rnd.h" 26 27 #if NRND == 0 28 #error padlock requires rnd pseudo-devices 29 #endif 30 31 #include "opt_viapadlock.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/signalvar.h> 36 #include <sys/kernel.h> 37 #include <sys/rnd.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/cpu.h> 41 #include <sys/rnd.h> 42 43 #include <x86/specialreg.h> 44 45 #include <machine/cpufunc.h> 46 #include <machine/cpuvar.h> 47 48 #include <opencrypto/cryptodev.h> 49 #include <opencrypto/cryptosoft.h> 50 #include <opencrypto/xform.h> 51 #include <crypto/rijndael/rijndael.h> 52 53 #include <opencrypto/cryptosoft_xform.c> 54 55 #ifdef VIA_PADLOCK 56 57 char xxx_via_buffer[1024]; 58 59 int via_padlock_crypto_newsession(void *, uint32_t *, struct cryptoini *); 60 int via_padlock_crypto_process(void *, struct cryptop *, int); 61 int via_padlock_crypto_swauth(struct cryptop *, struct cryptodesc *, 62 struct swcr_data *, void *); 63 int via_padlock_crypto_encdec(struct cryptop *, struct cryptodesc *, 64 struct via_padlock_session *, struct via_padlock_softc *, void *); 65 int via_padlock_crypto_freesession(void *, uint64_t); 66 static __inline void via_padlock_cbc(void *, void *, void *, void *, int, 67 void *); 68 69 static void 70 via_c3_rnd(void *arg) 71 { 72 struct via_padlock_softc *vp_sc = arg; 73 74 unsigned int rv, creg0, len = VIAC3_RNG_BUFSIZ; 75 static uint32_t buffer[VIAC3_RNG_BUFSIZ + 2]; /* XXX 2? */ 76 77 /* 78 * Sadly, we have to monkey with the coprocessor enable and fault 79 * registers, which are really for the FPU, in order to read 80 * from the RNG. 81 * 82 * Don't remove CR0_TS from the call below -- comments in the Linux 83 * driver indicate that the xstorerng instruction can generate 84 * spurious DNA faults though no FPU or SIMD state is changed 85 * even if such a fault is generated. 86 * 87 */ 88 kpreempt_disable(); 89 x86_disable_intr(); 90 creg0 = rcr0(); 91 lcr0(creg0 & ~(CR0_EM|CR0_TS)); /* Permit access to SIMD/FPU path */ 92 /* 93 * Collect the random data from the C3 RNG into our buffer. 94 * We turn on maximum whitening (is this actually desirable 95 * if we will feed the data to SHA1?) (%edx[0,1] = "11"). 96 */ 97 __asm __volatile("rep xstorerng" 98 : "=a" (rv) : "d" (3), "D" (buffer), 99 "c" (len * sizeof(int)) : "memory", "cc"); 100 /* Put CR0 back how it was */ 101 lcr0(creg0); 102 x86_enable_intr(); 103 kpreempt_enable(); 104 rnd_add_data(&vp_sc->sc_rnd_source, buffer, len * sizeof(int), 105 len * sizeof(int)); 106 callout_reset(&vp_sc->sc_rnd_co, vp_sc->sc_rnd_hz, via_c3_rnd, vp_sc); 107 } 108 109 static void 110 via_c3_rnd_init(struct via_padlock_softc *const vp_sc) 111 { 112 if (hz >= 100) { 113 vp_sc->sc_rnd_hz = 10 * hz / 100; 114 } else { 115 vp_sc->sc_rnd_hz = 10; 116 } 117 /* See hifn7751.c re use of RND_FLAG_NO_ESTIMATE */ 118 rnd_attach_source(&vp_sc->sc_rnd_source, "padlock", 119 RND_TYPE_RNG, RND_FLAG_NO_ESTIMATE); 120 callout_init(&vp_sc->sc_rnd_co, 0); 121 /* Call once to prime the pool early and set callout. */ 122 via_c3_rnd(vp_sc); 123 } 124 125 static void 126 via_c3_ace_init(struct via_padlock_softc *const vp_sc) 127 { 128 /* 129 * There is no reason to call into the kernel to use this 130 * driver from userspace, because the crypto instructions can 131 * be directly accessed there. Setting CRYPTOCAP_F_SOFTWARE 132 * has approximately the right semantics though the name is 133 * confusing (however, consider that crypto via unprivileged 134 * instructions _is_ "just software" in some sense). 135 */ 136 vp_sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_SOFTWARE); 137 if (vp_sc->sc_cid < 0) { 138 printf("PadLock: Could not get a crypto driver ID\n"); 139 free(vp_sc, M_DEVBUF); 140 return; 141 } 142 143 /* 144 * Ask the opencrypto subsystem to register ourselves. Although 145 * we don't support hardware offloading for various HMAC algorithms, 146 * we will handle them, because opencrypto prefers drivers that 147 * support all requested algorithms. 148 * 149 * 150 * XXX We should actually implement the HMAC modes this hardware 151 * XXX can accellerate (wrap its plain SHA1/SHA2 as HMAC) and 152 * XXX strongly consider removing those passed through to cryptosoft. 153 * XXX As it stands, we can "steal" sessions from drivers which could 154 * XXX better accellerate them. 155 * 156 * XXX Note the ordering dependency between when this (or any 157 * XXX crypto driver) attaches and when cryptosoft does. We are 158 * XXX basically counting on the swcrypto pseudo-device to just 159 * XXX happen to attach last, or _it_ will steal every session 160 * XXX from _us_! 161 */ 162 #define REGISTER(alg) \ 163 crypto_register(vp_sc->sc_cid, alg, 0, 0, \ 164 via_padlock_crypto_newsession, via_padlock_crypto_freesession, \ 165 via_padlock_crypto_process, vp_sc); 166 167 REGISTER(CRYPTO_AES_CBC); 168 REGISTER(CRYPTO_MD5_HMAC_96); 169 REGISTER(CRYPTO_MD5_HMAC); 170 REGISTER(CRYPTO_SHA1_HMAC_96); 171 REGISTER(CRYPTO_SHA1_HMAC); 172 REGISTER(CRYPTO_RIPEMD160_HMAC_96); 173 REGISTER(CRYPTO_RIPEMD160_HMAC); 174 REGISTER(CRYPTO_SHA2_HMAC); 175 } 176 177 void 178 via_padlock_attach(void) 179 { 180 struct via_padlock_softc *vp_sc; 181 182 printf("%s", xxx_via_buffer); 183 184 if (!((cpu_feature[4] & CPUID_VIA_HAS_ACE) || 185 (cpu_feature[4] & CPUID_VIA_HAS_RNG))) { 186 printf("PadLock: Nothing (%08x ! %08X ! %08X)\n", 187 cpu_feature[4], CPUID_VIA_HAS_ACE, 188 CPUID_VIA_HAS_RNG); 189 return; /* Nothing to see here, move along. */ 190 } 191 192 if ((vp_sc = malloc(sizeof(*vp_sc), M_DEVBUF, M_NOWAIT)) == NULL) 193 return; 194 memset(vp_sc, 0, sizeof(*vp_sc)); 195 196 if (cpu_feature[4] & CPUID_VIA_HAS_RNG) { 197 via_c3_rnd_init(vp_sc); 198 printf("PadLock: RNG attached\n"); 199 } 200 201 if (cpu_feature[4] & CPUID_VIA_HAS_ACE) { 202 via_c3_ace_init(vp_sc); 203 printf("PadLock: AES-CBC attached\n"); 204 } 205 } 206 207 int 208 via_padlock_crypto_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri) 209 { 210 struct cryptoini *c; 211 struct via_padlock_softc *sc = arg; 212 struct via_padlock_session *ses = NULL; 213 const struct swcr_auth_hash *axf; 214 struct swcr_data *swd; 215 int sesn, i, cw0; 216 217 KASSERT(sc != NULL /*, ("via_padlock_crypto_freesession: null softc")*/); 218 if (sc == NULL || sidp == NULL || cri == NULL) 219 return (EINVAL); 220 221 if (sc->sc_sessions == NULL) { 222 ses = sc->sc_sessions = malloc(sizeof(*ses), M_DEVBUF, 223 M_NOWAIT); 224 if (ses == NULL) 225 return (ENOMEM); 226 sesn = 0; 227 sc->sc_nsessions = 1; 228 } else { 229 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 230 if (sc->sc_sessions[sesn].ses_used == 0) { 231 ses = &sc->sc_sessions[sesn]; 232 break; 233 } 234 } 235 236 if (ses == NULL) { 237 sesn = sc->sc_nsessions; 238 ses = malloc((sesn + 1) * sizeof(*ses), M_DEVBUF, 239 M_NOWAIT); 240 if (ses == NULL) 241 return (ENOMEM); 242 memcpy(ses, sc->sc_sessions, sesn * sizeof(*ses)); 243 memset(sc->sc_sessions, 0, sesn * sizeof(*ses)); 244 free(sc->sc_sessions, M_DEVBUF); 245 sc->sc_sessions = ses; 246 ses = &sc->sc_sessions[sesn]; 247 sc->sc_nsessions++; 248 } 249 } 250 251 memset(ses, 0, sizeof(*ses)); 252 ses->ses_used = 1; 253 254 for (c = cri; c != NULL; c = c->cri_next) { 255 switch (c->cri_alg) { 256 case CRYPTO_AES_CBC: 257 switch (c->cri_klen) { 258 case 128: 259 cw0 = C3_CRYPT_CWLO_KEY128; 260 break; 261 case 192: 262 cw0 = C3_CRYPT_CWLO_KEY192; 263 break; 264 case 256: 265 cw0 = C3_CRYPT_CWLO_KEY256; 266 break; 267 default: 268 return (EINVAL); 269 } 270 cw0 |= C3_CRYPT_CWLO_ALG_AES | 271 C3_CRYPT_CWLO_KEYGEN_SW | 272 C3_CRYPT_CWLO_NORMAL; 273 274 #ifdef __NetBSD__ 275 rnd_extract_data(ses->ses_iv, sizeof(ses->ses_iv), 276 RND_EXTRACT_ANY); 277 #else 278 get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv)); 279 #endif 280 ses->ses_klen = c->cri_klen; 281 ses->ses_cw0 = cw0; 282 283 /* Build expanded keys for both directions */ 284 rijndaelKeySetupEnc(ses->ses_ekey, c->cri_key, 285 c->cri_klen); 286 rijndaelKeySetupDec(ses->ses_dkey, c->cri_key, 287 c->cri_klen); 288 for (i = 0; i < 4 * (RIJNDAEL_MAXNR + 1); i++) { 289 ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]); 290 ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]); 291 } 292 293 break; 294 295 /* Use hashing implementations from the cryptosoft code. */ 296 case CRYPTO_MD5_HMAC: 297 axf = &swcr_auth_hash_hmac_md5; 298 goto authcommon; 299 case CRYPTO_MD5_HMAC_96: 300 axf = &swcr_auth_hash_hmac_md5_96; 301 goto authcommon; 302 case CRYPTO_SHA1_HMAC: 303 axf = &swcr_auth_hash_hmac_sha1; 304 goto authcommon; 305 case CRYPTO_SHA1_HMAC_96: 306 axf = &swcr_auth_hash_hmac_sha1_96; 307 goto authcommon; 308 case CRYPTO_RIPEMD160_HMAC: 309 axf = &swcr_auth_hash_hmac_ripemd_160; 310 goto authcommon; 311 case CRYPTO_RIPEMD160_HMAC_96: 312 axf = &swcr_auth_hash_hmac_ripemd_160_96; 313 goto authcommon; 314 case CRYPTO_SHA2_HMAC: 315 if (cri->cri_klen == 256) 316 axf = &swcr_auth_hash_hmac_sha2_256; 317 else if (cri->cri_klen == 384) 318 axf = &swcr_auth_hash_hmac_sha2_384; 319 else if (cri->cri_klen == 512) 320 axf = &swcr_auth_hash_hmac_sha2_512; 321 else { 322 return EINVAL; 323 } 324 authcommon: 325 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA, 326 M_NOWAIT|M_ZERO); 327 if (swd == NULL) { 328 via_padlock_crypto_freesession(sc, sesn); 329 return (ENOMEM); 330 } 331 ses->swd = swd; 332 333 swd->sw_ictx = malloc(axf->auth_hash->ctxsize, 334 M_CRYPTO_DATA, M_NOWAIT); 335 if (swd->sw_ictx == NULL) { 336 via_padlock_crypto_freesession(sc, sesn); 337 return (ENOMEM); 338 } 339 340 swd->sw_octx = malloc(axf->auth_hash->ctxsize, 341 M_CRYPTO_DATA, M_NOWAIT); 342 if (swd->sw_octx == NULL) { 343 via_padlock_crypto_freesession(sc, sesn); 344 return (ENOMEM); 345 } 346 347 for (i = 0; i < c->cri_klen / 8; i++) 348 c->cri_key[i] ^= HMAC_IPAD_VAL; 349 350 axf->Init(swd->sw_ictx); 351 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8); 352 axf->Update(swd->sw_ictx, hmac_ipad_buffer, 353 HMAC_BLOCK_LEN - (c->cri_klen / 8)); 354 355 for (i = 0; i < c->cri_klen / 8; i++) 356 c->cri_key[i] ^= (HMAC_IPAD_VAL ^ 357 HMAC_OPAD_VAL); 358 359 axf->Init(swd->sw_octx); 360 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8); 361 axf->Update(swd->sw_octx, hmac_opad_buffer, 362 HMAC_BLOCK_LEN - (c->cri_klen / 8)); 363 364 for (i = 0; i < c->cri_klen / 8; i++) 365 c->cri_key[i] ^= HMAC_OPAD_VAL; 366 367 swd->sw_axf = axf; 368 swd->sw_alg = c->cri_alg; 369 370 break; 371 default: 372 return (EINVAL); 373 } 374 } 375 376 *sidp = VIAC3_SID(0, sesn); 377 return (0); 378 } 379 380 int 381 via_padlock_crypto_freesession(void *arg, uint64_t tid) 382 { 383 struct via_padlock_softc *sc = arg; 384 struct swcr_data *swd; 385 struct auth_hash *axf; 386 int sesn; 387 uint32_t sid = ((uint32_t)tid) & 0xffffffff; 388 389 KASSERT(sc != NULL /*, ("via_padlock_crypto_freesession: null softc")*/); 390 if (sc == NULL) 391 return (EINVAL); 392 393 sesn = VIAC3_SESSION(sid); 394 if (sesn >= sc->sc_nsessions) 395 return (EINVAL); 396 397 if (sc->sc_sessions[sesn].swd) { 398 swd = sc->sc_sessions[sesn].swd; 399 axf = swd->sw_axf->auth_hash; 400 401 if (swd->sw_ictx) { 402 memset(swd->sw_ictx, 0, axf->ctxsize); 403 free(swd->sw_ictx, M_CRYPTO_DATA); 404 } 405 if (swd->sw_octx) { 406 memset(swd->sw_octx, 0, axf->ctxsize); 407 free(swd->sw_octx, M_CRYPTO_DATA); 408 } 409 free(swd, M_CRYPTO_DATA); 410 } 411 412 memset(&sc->sc_sessions[sesn], 0, sizeof(sc->sc_sessions[sesn])); 413 return (0); 414 } 415 416 static __inline void 417 via_padlock_cbc(void *cw, void *src, void *dst, void *key, int rep, 418 void *iv) 419 { 420 unsigned int creg0; 421 422 creg0 = rcr0(); /* Permit access to SIMD/FPU path */ 423 lcr0(creg0 & ~(CR0_EM|CR0_TS)); 424 425 /* Do the deed */ 426 __asm __volatile("pushfl; popfl"); /* force key reload */ 427 __asm __volatile(".byte 0xf3, 0x0f, 0xa7, 0xd0" : /* rep xcrypt-cbc */ 428 : "a" (iv), "b" (key), "c" (rep), "d" (cw), "S" (src), "D" (dst) 429 : "memory", "cc"); 430 431 lcr0(creg0); 432 } 433 434 int 435 via_padlock_crypto_swauth(struct cryptop *crp, struct cryptodesc *crd, 436 struct swcr_data *sw, void *buf) 437 { 438 int type; 439 440 if (crp->crp_flags & CRYPTO_F_IMBUF) 441 type = CRYPTO_BUF_MBUF; 442 else 443 type= CRYPTO_BUF_IOV; 444 445 return (swcr_authcompute(crp, crd, sw, buf, type)); 446 } 447 448 int 449 via_padlock_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd, 450 struct via_padlock_session *ses, struct via_padlock_softc *sc, void *buf) 451 { 452 uint32_t *key; 453 int err = 0; 454 455 if ((crd->crd_len % 16) != 0) { 456 err = EINVAL; 457 return (err); 458 } 459 460 sc->op_buf = malloc(crd->crd_len, M_DEVBUF, M_NOWAIT); 461 if (sc->op_buf == NULL) { 462 err = ENOMEM; 463 return (err); 464 } 465 466 if (crd->crd_flags & CRD_F_ENCRYPT) { 467 sc->op_cw[0] = ses->ses_cw0 | C3_CRYPT_CWLO_ENCRYPT; 468 key = ses->ses_ekey; 469 if (crd->crd_flags & CRD_F_IV_EXPLICIT) 470 memcpy(sc->op_iv, crd->crd_iv, 16); 471 else 472 memcpy(sc->op_iv, ses->ses_iv, 16); 473 474 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 475 if (crp->crp_flags & CRYPTO_F_IMBUF) 476 m_copyback((struct mbuf *)crp->crp_buf, 477 crd->crd_inject, 16, sc->op_iv); 478 else if (crp->crp_flags & CRYPTO_F_IOV) 479 cuio_copyback((struct uio *)crp->crp_buf, 480 crd->crd_inject, 16, sc->op_iv); 481 else 482 memcpy((char *)crp->crp_buf + crd->crd_inject, 483 sc->op_iv, 16); 484 } 485 } else { 486 sc->op_cw[0] = ses->ses_cw0 | C3_CRYPT_CWLO_DECRYPT; 487 key = ses->ses_dkey; 488 if (crd->crd_flags & CRD_F_IV_EXPLICIT) 489 memcpy(sc->op_iv, crd->crd_iv, 16); 490 else { 491 if (crp->crp_flags & CRYPTO_F_IMBUF) 492 m_copydata((struct mbuf *)crp->crp_buf, 493 crd->crd_inject, 16, sc->op_iv); 494 else if (crp->crp_flags & CRYPTO_F_IOV) 495 cuio_copydata((struct uio *)crp->crp_buf, 496 crd->crd_inject, 16, sc->op_iv); 497 else 498 memcpy(sc->op_iv, (char *)crp->crp_buf + 499 crd->crd_inject, 16); 500 } 501 } 502 503 if (crp->crp_flags & CRYPTO_F_IMBUF) 504 m_copydata((struct mbuf *)crp->crp_buf, 505 crd->crd_skip, crd->crd_len, sc->op_buf); 506 else if (crp->crp_flags & CRYPTO_F_IOV) 507 cuio_copydata((struct uio *)crp->crp_buf, 508 crd->crd_skip, crd->crd_len, sc->op_buf); 509 else 510 memcpy(sc->op_buf, (char *)crp->crp_buf + crd->crd_skip, 511 crd->crd_len); 512 513 sc->op_cw[1] = sc->op_cw[2] = sc->op_cw[3] = 0; 514 via_padlock_cbc(&sc->op_cw, sc->op_buf, sc->op_buf, key, 515 crd->crd_len / 16, sc->op_iv); 516 517 if (crp->crp_flags & CRYPTO_F_IMBUF) 518 m_copyback((struct mbuf *)crp->crp_buf, 519 crd->crd_skip, crd->crd_len, sc->op_buf); 520 else if (crp->crp_flags & CRYPTO_F_IOV) 521 cuio_copyback((struct uio *)crp->crp_buf, 522 crd->crd_skip, crd->crd_len, sc->op_buf); 523 else 524 memcpy((char *)crp->crp_buf + crd->crd_skip, sc->op_buf, 525 crd->crd_len); 526 527 /* copy out last block for use as next session IV */ 528 if (crd->crd_flags & CRD_F_ENCRYPT) { 529 if (crp->crp_flags & CRYPTO_F_IMBUF) 530 m_copydata((struct mbuf *)crp->crp_buf, 531 crd->crd_skip + crd->crd_len - 16, 16, 532 ses->ses_iv); 533 else if (crp->crp_flags & CRYPTO_F_IOV) 534 cuio_copydata((struct uio *)crp->crp_buf, 535 crd->crd_skip + crd->crd_len - 16, 16, 536 ses->ses_iv); 537 else 538 memcpy(ses->ses_iv, (char *)crp->crp_buf + 539 crd->crd_skip + crd->crd_len - 16, 16); 540 } 541 542 if (sc->op_buf != NULL) { 543 memset(sc->op_buf, 0, crd->crd_len); 544 free(sc->op_buf, M_DEVBUF); 545 sc->op_buf = NULL; 546 } 547 548 return (err); 549 } 550 551 int 552 via_padlock_crypto_process(void *arg, struct cryptop *crp, int hint) 553 { 554 struct via_padlock_softc *sc = arg; 555 struct via_padlock_session *ses; 556 struct cryptodesc *crd; 557 int sesn, err = 0; 558 559 KASSERT(sc != NULL /*, ("via_padlock_crypto_process: null softc")*/); 560 if (crp == NULL || crp->crp_callback == NULL) { 561 err = EINVAL; 562 goto out; 563 } 564 565 sesn = VIAC3_SESSION(crp->crp_sid); 566 if (sesn >= sc->sc_nsessions) { 567 err = EINVAL; 568 goto out; 569 } 570 ses = &sc->sc_sessions[sesn]; 571 572 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 573 switch (crd->crd_alg) { 574 case CRYPTO_AES_CBC: 575 if ((err = via_padlock_crypto_encdec(crp, crd, ses, 576 sc, crp->crp_buf)) != 0) 577 goto out; 578 break; 579 580 case CRYPTO_MD5_HMAC: 581 case CRYPTO_SHA1_HMAC: 582 case CRYPTO_RIPEMD160_HMAC: 583 case CRYPTO_SHA2_HMAC: 584 if ((err = via_padlock_crypto_swauth(crp, crd, 585 ses->swd, crp->crp_buf)) != 0) 586 goto out; 587 break; 588 589 default: 590 err = EINVAL; 591 goto out; 592 } 593 } 594 out: 595 crp->crp_etype = err; 596 crypto_done(crp); 597 return (err); 598 } 599 600 #endif /* VIA_PADLOCK */ 601