1 /* 2 * (MPSAFE) 3 * 4 * Copyright (c) 2006 The DragonFly Project. All rights reserved. 5 * 6 * This code is derived from software contributed to The DragonFly Project 7 * by Matthew Dillon <dillon@backplane.com> 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in 17 * the documentation and/or other materials provided with the 18 * distribution. 19 * 3. Neither the name of The DragonFly Project nor the names of its 20 * contributors may be used to endorse or promote products derived 21 * from this software without specific, prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/systm.h> 40 #include <sys/sysproto.h> 41 #include <sys/kern_syscall.h> 42 #include <sys/mman.h> 43 #include <sys/thread.h> 44 #include <sys/proc.h> 45 #include <sys/malloc.h> 46 #include <sys/sysctl.h> 47 #include <sys/vkernel.h> 48 #include <sys/vmspace.h> 49 50 #include <vm/vm_extern.h> 51 #include <vm/pmap.h> 52 53 #include <machine/vmparam.h> 54 55 #include <sys/sysref2.h> 56 #include <sys/mplock2.h> 57 58 static struct vmspace_entry *vkernel_find_vmspace(struct vkernel_proc *vkp, 59 void *id); 60 static void vmspace_entry_delete(struct vmspace_entry *ve, 61 struct vkernel_proc *vkp); 62 63 static MALLOC_DEFINE(M_VKERNEL, "vkernel", "VKernel structures"); 64 65 /* 66 * vmspace_create (void *id, int type, void *data) 67 * 68 * Create a VMSPACE under the control of the caller with the specified id. 69 * An id of NULL cannot be used. The type and data fields must currently 70 * be 0. 71 * 72 * The vmspace starts out completely empty. Memory may be mapped into the 73 * VMSPACE with vmspace_mmap() and MAP_VPAGETABLE section(s) controlled 74 * with vmspace_mcontrol(). 75 * 76 * No requirements. 77 */ 78 int 79 sys_vmspace_create(struct vmspace_create_args *uap) 80 { 81 struct vmspace_entry *ve; 82 struct vkernel_proc *vkp; 83 struct proc *p = curproc; 84 int error; 85 86 if (vkernel_enable == 0) 87 return (EOPNOTSUPP); 88 89 /* 90 * Create a virtual kernel side-structure for the process if one 91 * does not exist. 92 * 93 * Implement a simple resolution for SMP races. 94 */ 95 if ((vkp = p->p_vkernel) == NULL) { 96 vkp = kmalloc(sizeof(*vkp), M_VKERNEL, M_WAITOK|M_ZERO); 97 lwkt_gettoken(&proc_token); 98 if (p->p_vkernel == NULL) { 99 vkp->refs = 1; 100 lwkt_token_init(&vkp->token, 1, "vkernel"); 101 RB_INIT(&vkp->root); 102 p->p_vkernel = vkp; 103 } else { 104 kfree(vkp, M_VKERNEL); 105 vkp = p->p_vkernel; 106 } 107 lwkt_reltoken(&proc_token); 108 } 109 110 get_mplock(); 111 112 /* 113 * Create a new VMSPACE, disallow conflicting ids 114 */ 115 ve = kmalloc(sizeof(struct vmspace_entry), M_VKERNEL, M_WAITOK|M_ZERO); 116 ve->vmspace = vmspace_alloc(VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS); 117 ve->id = uap->id; 118 pmap_pinit2(vmspace_pmap(ve->vmspace)); 119 120 lwkt_gettoken(&vkp->token); 121 if (RB_INSERT(vmspace_rb_tree, &vkp->root, ve)) { 122 sysref_put(&ve->vmspace->vm_sysref); 123 kfree(ve, M_VKERNEL); 124 error = EEXIST; 125 } else { 126 error = 0; 127 } 128 lwkt_reltoken(&vkp->token); 129 rel_mplock(); 130 return (error); 131 } 132 133 /* 134 * Destroy a VMSPACE given its identifier. 135 * 136 * No requirements. 137 */ 138 int 139 sys_vmspace_destroy(struct vmspace_destroy_args *uap) 140 { 141 struct vkernel_proc *vkp; 142 struct vmspace_entry *ve; 143 int error; 144 145 get_mplock(); 146 if ((vkp = curproc->p_vkernel) == NULL) { 147 error = EINVAL; 148 goto done3; 149 } 150 lwkt_gettoken(&vkp->token); 151 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) { 152 error = ENOENT; 153 goto done2; 154 } 155 if (ve->refs) { 156 error = EBUSY; 157 goto done2; 158 } 159 vmspace_entry_delete(ve, vkp); 160 error = 0; 161 done2: 162 lwkt_reltoken(&vkp->token); 163 done3: 164 rel_mplock(); 165 return(error); 166 } 167 168 /* 169 * vmspace_ctl (void *id, int cmd, struct trapframe *tframe, 170 * struct vextframe *vframe); 171 * 172 * Transfer control to a VMSPACE. Control is returned after the specified 173 * number of microseconds or if a page fault, signal, trap, or system call 174 * occurs. The context is updated as appropriate. 175 * 176 * No requirements. 177 */ 178 int 179 sys_vmspace_ctl(struct vmspace_ctl_args *uap) 180 { 181 struct vkernel_proc *vkp; 182 struct vkernel_lwp *vklp; 183 struct vmspace_entry *ve; 184 struct lwp *lp; 185 struct proc *p; 186 int framesz; 187 int error; 188 189 lp = curthread->td_lwp; 190 p = lp->lwp_proc; 191 192 if ((vkp = p->p_vkernel) == NULL) 193 return (EINVAL); 194 195 get_mplock(); 196 lwkt_gettoken(&vkp->token); 197 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) { 198 error = ENOENT; 199 goto done; 200 } 201 202 /* 203 * Signal mailbox interlock 204 */ 205 if (p->p_flag & P_MAILBOX) { 206 p->p_flag &= ~P_MAILBOX; 207 error = EINTR; 208 goto done; 209 } 210 211 switch(uap->cmd) { 212 case VMSPACE_CTL_RUN: 213 /* 214 * Save the caller's register context, swap VM spaces, and 215 * install the passed register context. Return with 216 * EJUSTRETURN so the syscall code doesn't adjust the context. 217 */ 218 atomic_add_int(&ve->refs, 1); 219 framesz = sizeof(struct trapframe); 220 if ((vklp = lp->lwp_vkernel) == NULL) { 221 vklp = kmalloc(sizeof(*vklp), M_VKERNEL, 222 M_WAITOK|M_ZERO); 223 lp->lwp_vkernel = vklp; 224 } 225 vklp->user_trapframe = uap->tframe; 226 vklp->user_vextframe = uap->vframe; 227 bcopy(uap->sysmsg_frame, &vklp->save_trapframe, framesz); 228 bcopy(&curthread->td_tls, &vklp->save_vextframe.vx_tls, 229 sizeof(vklp->save_vextframe.vx_tls)); 230 error = copyin(uap->tframe, uap->sysmsg_frame, framesz); 231 if (error == 0) { 232 error = copyin(&uap->vframe->vx_tls, 233 &curthread->td_tls, 234 sizeof(struct savetls)); 235 } 236 if (error == 0) 237 error = cpu_sanitize_frame(uap->sysmsg_frame); 238 if (error == 0) 239 error = cpu_sanitize_tls(&curthread->td_tls); 240 if (error) { 241 bcopy(&vklp->save_trapframe, uap->sysmsg_frame, 242 framesz); 243 bcopy(&vklp->save_vextframe.vx_tls, &curthread->td_tls, 244 sizeof(vklp->save_vextframe.vx_tls)); 245 set_user_TLS(); 246 atomic_subtract_int(&ve->refs, 1); 247 } else { 248 vklp->ve = ve; 249 pmap_setlwpvm(lp, ve->vmspace); 250 set_user_TLS(); 251 set_vkernel_fp(uap->sysmsg_frame); 252 error = EJUSTRETURN; 253 } 254 break; 255 default: 256 error = EOPNOTSUPP; 257 break; 258 } 259 done: 260 lwkt_reltoken(&vkp->token); 261 rel_mplock(); 262 return(error); 263 } 264 265 /* 266 * vmspace_mmap(id, addr, len, prot, flags, fd, offset) 267 * 268 * map memory within a VMSPACE. This function is just like a normal mmap() 269 * but operates on the vmspace's memory map. Most callers use this to create 270 * a MAP_VPAGETABLE mapping. 271 * 272 * No requirements. 273 */ 274 int 275 sys_vmspace_mmap(struct vmspace_mmap_args *uap) 276 { 277 struct vkernel_proc *vkp; 278 struct vmspace_entry *ve; 279 int error; 280 281 /* 282 * We hold the vmspace token to serialize calls to vkernel_find_vmspace 283 * and the vm token to serialize calls to kern_mmap. 284 */ 285 lwkt_gettoken(&vm_token); 286 lwkt_gettoken(&vmspace_token); 287 if ((vkp = curproc->p_vkernel) == NULL) { 288 error = EINVAL; 289 goto done3; 290 } 291 292 /* 293 * NOTE: kern_mmap() can block so we need to temporarily ref ve->refs. 294 */ 295 lwkt_gettoken(&vkp->token); 296 if ((ve = vkernel_find_vmspace(vkp, uap->id)) != NULL) { 297 atomic_add_int(&ve->refs, 1); 298 error = kern_mmap(ve->vmspace, uap->addr, uap->len, 299 uap->prot, uap->flags, 300 uap->fd, uap->offset, &uap->sysmsg_resultp); 301 atomic_subtract_int(&ve->refs, 1); 302 } else { 303 error = ENOENT; 304 } 305 lwkt_reltoken(&vkp->token); 306 done3: 307 lwkt_reltoken(&vmspace_token); 308 lwkt_reltoken(&vm_token); 309 return (error); 310 } 311 312 /* 313 * vmspace_munmap(id, addr, len) 314 * 315 * unmap memory within a VMSPACE. 316 * 317 * No requirements. 318 */ 319 int 320 sys_vmspace_munmap(struct vmspace_munmap_args *uap) 321 { 322 struct vkernel_proc *vkp; 323 struct vmspace_entry *ve; 324 vm_offset_t addr; 325 vm_offset_t tmpaddr; 326 vm_size_t size, pageoff; 327 vm_map_t map; 328 int error; 329 330 get_mplock(); 331 if ((vkp = curproc->p_vkernel) == NULL) { 332 error = EINVAL; 333 goto done3; 334 } 335 lwkt_gettoken(&vkp->token); 336 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) { 337 error = ENOENT; 338 goto done2; 339 } 340 341 /* 342 * NOTE: kern_munmap() can block so we need to temporarily 343 * ref ve->refs. 344 */ 345 atomic_add_int(&ve->refs, 1); 346 347 /* 348 * Copied from sys_munmap() 349 */ 350 addr = (vm_offset_t)uap->addr; 351 size = uap->len; 352 353 pageoff = (addr & PAGE_MASK); 354 addr -= pageoff; 355 size += pageoff; 356 size = (vm_size_t)round_page(size); 357 if (size < uap->len) { /* wrap */ 358 error = EINVAL; 359 goto done1; 360 } 361 tmpaddr = addr + size; /* workaround gcc4 opt */ 362 if (tmpaddr < addr) { /* wrap */ 363 error = EINVAL; 364 goto done1; 365 } 366 if (size == 0) { 367 error = 0; 368 goto done1; 369 } 370 371 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) { 372 error = EINVAL; 373 goto done1; 374 } 375 if (VM_MIN_USER_ADDRESS > 0 && addr < VM_MIN_USER_ADDRESS) { 376 error = EINVAL; 377 goto done1; 378 } 379 map = &ve->vmspace->vm_map; 380 if (!vm_map_check_protection(map, addr, tmpaddr, VM_PROT_NONE, FALSE)) { 381 error = EINVAL; 382 goto done1; 383 } 384 vm_map_remove(map, addr, addr + size); 385 error = 0; 386 done1: 387 atomic_subtract_int(&ve->refs, 1); 388 done2: 389 lwkt_reltoken(&vkp->token); 390 done3: 391 rel_mplock(); 392 return (error); 393 } 394 395 /* 396 * vmspace_pread(id, buf, nbyte, flags, offset) 397 * 398 * Read data from a vmspace. The number of bytes read is returned or 399 * -1 if an unrecoverable error occured. If the number of bytes read is 400 * less then the request size, a page fault occured in the VMSPACE which 401 * the caller must resolve in order to proceed. 402 * 403 * (not implemented yet) 404 * No requirements. 405 */ 406 int 407 sys_vmspace_pread(struct vmspace_pread_args *uap) 408 { 409 struct vkernel_proc *vkp; 410 struct vmspace_entry *ve; 411 int error; 412 413 get_mplock(); 414 if ((vkp = curproc->p_vkernel) == NULL) { 415 error = EINVAL; 416 goto done3; 417 } 418 lwkt_gettoken(&vkp->token); 419 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) { 420 error = ENOENT; 421 goto done2; 422 } 423 error = EINVAL; 424 done2: 425 lwkt_reltoken(&vkp->token); 426 done3: 427 rel_mplock(); 428 return (error); 429 } 430 431 /* 432 * vmspace_pwrite(id, buf, nbyte, flags, offset) 433 * 434 * Write data to a vmspace. The number of bytes written is returned or 435 * -1 if an unrecoverable error occured. If the number of bytes written is 436 * less then the request size, a page fault occured in the VMSPACE which 437 * the caller must resolve in order to proceed. 438 * 439 * (not implemented yet) 440 * No requirements. 441 */ 442 int 443 sys_vmspace_pwrite(struct vmspace_pwrite_args *uap) 444 { 445 struct vkernel_proc *vkp; 446 struct vmspace_entry *ve; 447 int error; 448 449 get_mplock(); 450 if ((vkp = curproc->p_vkernel) == NULL) { 451 error = EINVAL; 452 goto done3; 453 } 454 lwkt_gettoken(&vkp->token); 455 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) { 456 error = ENOENT; 457 goto done2; 458 } 459 error = EINVAL; 460 done2: 461 lwkt_reltoken(&vkp->token); 462 done3: 463 rel_mplock(); 464 return (error); 465 } 466 467 /* 468 * vmspace_mcontrol(id, addr, len, behav, value) 469 * 470 * madvise/mcontrol support for a vmspace. 471 * 472 * No requirements. 473 */ 474 int 475 sys_vmspace_mcontrol(struct vmspace_mcontrol_args *uap) 476 { 477 struct vkernel_proc *vkp; 478 struct vmspace_entry *ve; 479 vm_offset_t start, end; 480 vm_offset_t tmpaddr = (vm_offset_t)uap->addr + uap->len; 481 int error; 482 483 get_mplock(); 484 if ((vkp = curproc->p_vkernel) == NULL) { 485 error = EINVAL; 486 goto done3; 487 } 488 lwkt_gettoken(&vkp->token); 489 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) { 490 error = ENOENT; 491 goto done2; 492 } 493 494 /* 495 * NOTE: kern_madvise() can block so we need to temporarily 496 * ref ve->refs. 497 */ 498 atomic_add_int(&ve->refs, 1); 499 500 /* 501 * This code is basically copied from sys_mcontrol() 502 */ 503 if (uap->behav < 0 || uap->behav > MADV_CONTROL_END) { 504 error = EINVAL; 505 goto done1; 506 } 507 508 if (tmpaddr < (vm_offset_t)uap->addr) { 509 error = EINVAL; 510 goto done1; 511 } 512 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) { 513 error = EINVAL; 514 goto done1; 515 } 516 if (VM_MIN_USER_ADDRESS > 0 && uap->addr < VM_MIN_USER_ADDRESS) { 517 error = EINVAL; 518 goto done1; 519 } 520 521 start = trunc_page((vm_offset_t) uap->addr); 522 end = round_page(tmpaddr); 523 524 error = vm_map_madvise(&ve->vmspace->vm_map, start, end, 525 uap->behav, uap->value); 526 done1: 527 atomic_subtract_int(&ve->refs, 1); 528 done2: 529 lwkt_reltoken(&vkp->token); 530 done3: 531 rel_mplock(); 532 return (error); 533 } 534 535 /* 536 * Red black tree functions 537 */ 538 static int rb_vmspace_compare(struct vmspace_entry *, struct vmspace_entry *); 539 RB_GENERATE(vmspace_rb_tree, vmspace_entry, rb_entry, rb_vmspace_compare); 540 541 /* 542 * a->start is address, and the only field has to be initialized. 543 * The caller must hold vkp->token. 544 * 545 * The caller must hold vkp->token. 546 */ 547 static int 548 rb_vmspace_compare(struct vmspace_entry *a, struct vmspace_entry *b) 549 { 550 if ((char *)a->id < (char *)b->id) 551 return(-1); 552 else if ((char *)a->id > (char *)b->id) 553 return(1); 554 return(0); 555 } 556 557 /* 558 * The caller must hold vkp->token. 559 */ 560 static 561 int 562 rb_vmspace_delete(struct vmspace_entry *ve, void *data) 563 { 564 struct vkernel_proc *vkp = data; 565 566 KKASSERT(ve->refs == 0); 567 vmspace_entry_delete(ve, vkp); 568 return(0); 569 } 570 571 /* 572 * Remove a vmspace_entry from the RB tree and destroy it. We have to clean 573 * up the pmap, the vm_map, then destroy the vmspace. 574 * 575 * This function must remove the ve immediately before it might potentially 576 * block. 577 * 578 * The caller must hold vkp->token. 579 */ 580 static 581 void 582 vmspace_entry_delete(struct vmspace_entry *ve, struct vkernel_proc *vkp) 583 { 584 RB_REMOVE(vmspace_rb_tree, &vkp->root, ve); 585 586 pmap_remove_pages(vmspace_pmap(ve->vmspace), 587 VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS); 588 vm_map_remove(&ve->vmspace->vm_map, 589 VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS); 590 sysref_put(&ve->vmspace->vm_sysref); 591 kfree(ve, M_VKERNEL); 592 } 593 594 /* 595 * Locate the ve for (id), return the ve or NULL. If found this function 596 * will bump ve->refs which prevents the ve from being immediately destroyed 597 * (but it can still be removed). 598 * 599 * The caller must hold vkp->token. 600 */ 601 static 602 struct vmspace_entry * 603 vkernel_find_vmspace(struct vkernel_proc *vkp, void *id) 604 { 605 struct vmspace_entry *ve; 606 struct vmspace_entry key; 607 608 key.id = id; 609 ve = RB_FIND(vmspace_rb_tree, &vkp->root, &key); 610 return (ve); 611 } 612 613 /* 614 * Manage vkernel refs, used by the kernel when fork()ing or exit()ing 615 * a vkernel process. 616 * 617 * No requirements. 618 */ 619 void 620 vkernel_inherit(struct proc *p1, struct proc *p2) 621 { 622 struct vkernel_proc *vkp; 623 624 vkp = p1->p_vkernel; 625 KKASSERT(vkp->refs > 0); 626 atomic_add_int(&vkp->refs, 1); 627 p2->p_vkernel = vkp; 628 } 629 630 /* 631 * No requirements. 632 */ 633 void 634 vkernel_exit(struct proc *p) 635 { 636 struct vkernel_proc *vkp; 637 struct lwp *lp; 638 639 vkp = p->p_vkernel; 640 641 /* 642 * Restore the original VM context if we are killed while running 643 * a different one. 644 * 645 * This isn't supposed to happen. What is supposed to happen is 646 * that the process should enter vkernel_trap() before the handling 647 * the signal. 648 */ 649 RB_FOREACH(lp, lwp_rb_tree, &p->p_lwp_tree) { 650 vkernel_lwp_exit(lp); 651 } 652 653 /* 654 * Dereference the common area 655 */ 656 p->p_vkernel = NULL; 657 KKASSERT(vkp->refs > 0); 658 659 if (atomic_fetchadd_int(&vkp->refs, -1) == 1) { 660 lwkt_gettoken(&vkp->token); 661 RB_SCAN(vmspace_rb_tree, &vkp->root, NULL, 662 rb_vmspace_delete, vkp); 663 lwkt_reltoken(&vkp->token); 664 kfree(vkp, M_VKERNEL); 665 } 666 } 667 668 /* 669 * No requirements. 670 */ 671 void 672 vkernel_lwp_exit(struct lwp *lp) 673 { 674 struct vkernel_lwp *vklp; 675 struct vmspace_entry *ve; 676 677 if ((vklp = lp->lwp_vkernel) != NULL) { 678 if ((ve = vklp->ve) != NULL) { 679 kprintf("Warning, pid %d killed with " 680 "active VC!\n", lp->lwp_proc->p_pid); 681 print_backtrace(-1); 682 pmap_setlwpvm(lp, lp->lwp_proc->p_vmspace); 683 vklp->ve = NULL; 684 KKASSERT(ve->refs > 0); 685 atomic_subtract_int(&ve->refs, 1); 686 } 687 lp->lwp_vkernel = NULL; 688 kfree(vklp, M_VKERNEL); 689 } 690 } 691 692 /* 693 * A VM space under virtual kernel control trapped out or made a system call 694 * or otherwise needs to return control to the virtual kernel context. 695 * 696 * No requirements. 697 */ 698 void 699 vkernel_trap(struct lwp *lp, struct trapframe *frame) 700 { 701 struct proc *p = lp->lwp_proc; 702 struct vmspace_entry *ve; 703 struct vkernel_lwp *vklp; 704 int error; 705 706 /* 707 * Which vmspace entry was running? 708 */ 709 vklp = lp->lwp_vkernel; 710 KKASSERT(vklp); 711 ve = vklp->ve; 712 KKASSERT(ve != NULL); 713 714 /* 715 * Switch the LWP vmspace back to the virtual kernel's VM space. 716 */ 717 vklp->ve = NULL; 718 pmap_setlwpvm(lp, p->p_vmspace); 719 KKASSERT(ve->refs > 0); 720 atomic_subtract_int(&ve->refs, 1); 721 /* ve is invalid once we kill our ref */ 722 723 /* 724 * Copy the emulated process frame to the virtual kernel process. 725 * The emulated process cannot change TLS descriptors so don't 726 * bother saving them, we already have a copy. 727 * 728 * Restore the virtual kernel's saved context so the virtual kernel 729 * process can resume. 730 */ 731 error = copyout(frame, vklp->user_trapframe, sizeof(*frame)); 732 bcopy(&vklp->save_trapframe, frame, sizeof(*frame)); 733 bcopy(&vklp->save_vextframe.vx_tls, &curthread->td_tls, 734 sizeof(vklp->save_vextframe.vx_tls)); 735 set_user_TLS(); 736 cpu_vkernel_trap(frame, error); 737 } 738