1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * Copyright (c) 2003 Peter Wemm. 4 * Copyright (c) 2008 The DragonFly Project. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD: src/sys/amd64/amd64/support.S,v 1.127 2007/05/23 08:33:04 kib Exp $ 32 */ 33 34#include <machine/asmacros.h> 35#include <machine/pmap.h> 36 37#include "assym.s" 38 39 ALIGN_DATA 40 41 .text 42 43/* 44 * bzero(ptr:%rdi, bytes:%rsi) 45 * 46 * Using rep stosq is 70% faster than a %rax loop and almost as fast as 47 * a %xmm0 loop on a modern intel cpu. 48 * 49 * Do not use non-termportal instructions here as we do not know the caller's 50 * intent. 51 */ 52ENTRY(bzero) 53 movq %rsi,%rcx 54 xorl %eax,%eax 55 shrq $3,%rcx 56 rep 57 stosq 58 movq %rsi,%rcx 59 andq $7,%rcx 60 jnz 1f 61 ret 621: rep 63 stosb 64 ret 65END(bzero) 66 67 .weak _bzero 68 .equ _bzero, bzero 69 70/* 71 * void *memset(ptr:%rdi, char:%rsi, bytes:%rdx) 72 * 73 * Same as bzero except we load the char into all byte 74 * positions of %rax. Returns original (ptr). 75 */ 76ENTRY(memset) 77 movzbq %sil,%r8 78 movabs $0x0101010101010101,%rax 79 imulq %r8,%rax 80 81 movq %rdi,%r9 82 movq %rdx,%rcx 83 shrq $3,%rcx 84 rep 85 stosq 86 movq %rdx,%rcx 87 andq $7,%rcx 88 jnz 1f 89 movq %r9,%rax 90 ret 911: rep 92 stosb 93 movq %r9,%rax 94 ret 95END(memset) 96 97 .weak _memset 98 .equ _memset, memset 99 100/* 101 * pagezero(ptr:%rdi) 102 * 103 * Using rep stosq is nearly as fast as using %xmm0 on a modern intel cpu, 104 * and about 70% faster than a %rax loop. 105 * 106 * Do not use non-termportal instructions here as we do not know the caller's 107 * intent. 108 */ 109#if 0 110 111ENTRY(pagezero) 112 movq $PAGE_SIZE>>3,%rcx 113 xorl %eax,%eax 114 rep 115 stosq 116 ret 117END(pagezero) 118 119#endif 120 121ENTRY(pagezero) 122 addq $4096,%rdi 123 movq $-4096,%rax 124 ALIGN_TEXT 1251: 126 movq $0,(%rdi,%rax,1) 127 addq $8,%rax 128 jne 1b 129 ret 130END(pagezero) 131 132/* 133 * bcopy(src:%rdi, dst:%rsi, cnt:%rdx) 134 * 135 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 136 */ 137ENTRY(bcopy) 138 xchgq %rsi,%rdi 139 movq %rdx,%rcx 140 141 movq %rdi,%rax 142 subq %rsi,%rax 143 cmpq %rcx,%rax /* overlapping && src < dst? */ 144 jb 2f 145 146 shrq $3,%rcx /* copy by 64-bit words */ 147 rep 148 movsq 149 movq %rdx,%rcx 150 andq $7,%rcx /* any bytes left? */ 151 jnz 1f 152 ret 1531: rep 154 movsb 155 ret 156 157 ALIGN_TEXT 1582: 159 addq %rcx,%rdi /* copy backwards */ 160 addq %rcx,%rsi 161 std 162 decq %rdi 163 decq %rsi 164 andq $7,%rcx /* any fractional bytes? */ 165 jz 3f 166 rep 167 movsb 1683: movq %rdx,%rcx /* copy by 32-bit words */ 169 shrq $3,%rcx 170 subq $7,%rsi 171 subq $7,%rdi 172 rep 173 movsq 174 cld 175 ret 176END(bcopy) 177 178 /* 179 * Use in situations where a bcopy function pointer is needed. 180 */ 181 .weak _bcopy 182 .equ _bcopy, bcopy 183 184 /* 185 * memmove(dst:%rdi, src:%rsi, cnt:%rdx) 186 * (same as bcopy but without the xchgq, and must return (dst)). 187 * 188 * NOTE: gcc builtin backs-off to memmove() call 189 * 190 * NOTE: We leave %rdi in %rax for the return value. 191 */ 192ENTRY(memmove) 193 movq %rdx,%rcx 194 movq %rdi,%rax /* return value */ 195 movq %rdi,%r8 196 subq %rsi,%r8 197 cmpq %rcx,%r8 /* overlapping && src < dst? */ 198 jb 2f 199 200 shrq $3,%rcx /* copy by 64-bit words */ 201 rep 202 movsq 203 movq %rdx,%rcx 204 andq $7,%rcx /* any bytes left? */ 205 jnz 1f 206 ret 2071: rep 208 movsb 209 ret 210 211 ALIGN_TEXT 2122: 213 addq %rcx,%rdi /* copy backwards */ 214 addq %rcx,%rsi 215 std 216 decq %rdi 217 decq %rsi 218 andq $7,%rcx /* any fractional bytes? */ 219 jz 3f 220 rep 221 movsb 2223: movq %rdx,%rcx /* copy by 32-bit words */ 223 shrq $3,%rcx 224 subq $7,%rsi 225 subq $7,%rdi 226 rep 227 movsq 228 cld 229 ret 230END(memmove) 231 232 .weak _memmove 233 .equ _memmove, memmove 234 235ENTRY(reset_dbregs) 236 movq $0x200,%rax /* the manual says that bit 10 must be set to 1 */ 237 movq %rax,%dr7 /* disable all breapoints first */ 238 movq $0,%rax 239 movq %rax,%dr0 240 movq %rax,%dr1 241 movq %rax,%dr2 242 movq %rax,%dr3 243 movq %rax,%dr6 244 ret 245END(reset_dbregs) 246 247/* 248 * memcpy(dst:%rdi, src:%rsi, bytes:%rdx) 249 * 250 * NOTE: memcpy does not support overlapping copies 251 * NOTE: returns dst 252 */ 253ENTRY(memcpy) 254 movq %rdi,%r8 255 movq %rdx,%rcx 256 shrq $3,%rcx /* copy by 64-bit words */ 257 rep 258 movsq 259 movq %rdx,%rcx 260 andq $7,%rcx /* any bytes left? */ 261 jnz 1f 262 movq %r8,%rax 263 ret 2641: rep 265 movsb 266 movq %r8,%rax 267 ret 268END(memcpy) 269 270 .weak _memcpy 271 .equ _memcpy, memcpy 272 273/* fillw(pat, base, cnt) */ 274/* %rdi,%rsi, %rdx */ 275ENTRY(fillw) 276 movq %rdi,%rax 277 movq %rsi,%rdi 278 movq %rdx,%rcx 279 rep 280 stosw 281 ret 282END(fillw) 283 284/*****************************************************************************/ 285/* copyout and fubyte family */ 286/*****************************************************************************/ 287/* 288 * Access user memory from inside the kernel. These routines should be 289 * the only places that do this. 290 * 291 * These routines set curpcb->onfault for the time they execute. When a 292 * protection violation occurs inside the functions, the trap handler 293 * returns to *curpcb->onfault instead of the function. 294 */ 295 296/* 297 * uint64_t:%rax kreadmem64(addr:%rdi) 298 * 299 * Read kernel or user memory with fault protection. 300 */ 301ENTRY(kreadmem64) 302 movq PCPU(curthread),%rcx 303 movq TD_PCB(%rcx), %rcx 304 movq $kreadmem64fault,PCB_ONFAULT(%rcx) 305 movq %rsp,PCB_ONFAULT_SP(%rcx) 306 307 movq (%rdi),%rax 308 movq $0,PCB_ONFAULT(%rcx) 309 ret 310 311kreadmem64fault: 312 movq PCPU(curthread),%rcx 313 xorl %eax,%eax 314 movq TD_PCB(%rcx),%rcx 315 movq %rax,PCB_ONFAULT(%rcx) 316 decq %rax 317 ret 318END(kreadmem64) 319 320/* 321 * std_copyout(from_kernel, to_user, len) - MP SAFE 322 * %rdi, %rsi, %rdx 323 */ 324ENTRY(std_copyout) 325 movq PCPU(curthread),%rax 326 movq TD_PCB(%rax), %rax 327 movq $copyout_fault,PCB_ONFAULT(%rax) 328 movq %rsp,PCB_ONFAULT_SP(%rax) 329 testq %rdx,%rdx /* anything to do? */ 330 jz done_copyout 331 332 /* 333 * Check explicitly for non-user addresses. If 486 write protection 334 * is being used, this check is essential because we are in kernel 335 * mode so the h/w does not provide any protection against writing 336 * kernel addresses. 337 */ 338 339 /* 340 * First, prevent address wrapping. 341 */ 342 movq %rsi,%rax 343 addq %rdx,%rax 344 jc copyout_fault 345/* 346 * XXX STOP USING VM_MAX_USER_ADDRESS. 347 * It is an end address, not a max, so every time it is used correctly it 348 * looks like there is an off by one error, and of course it caused an off 349 * by one error in several places. 350 */ 351 movq $VM_MAX_USER_ADDRESS,%rcx 352 cmpq %rcx,%rax 353 ja copyout_fault 354 355 xchgq %rdi,%rsi 356 /* bcopy(%rsi, %rdi, %rdx) */ 357 movq %rdx,%rcx 358 359 shrq $3,%rcx 360 jz 1f 361 rep 362 movsq 3631: movq %rdx,%rcx 364 andq $7,%rcx 365 jz done_copyout 366 rep 367 movsb 368 369done_copyout: 370 xorl %eax,%eax 371 movq PCPU(curthread),%rdx 372 movq TD_PCB(%rdx), %rdx 373 movq %rax,PCB_ONFAULT(%rdx) 374 ret 375 376 ALIGN_TEXT 377copyout_fault: 378 movq PCPU(curthread),%rdx 379 movq TD_PCB(%rdx), %rdx 380 movq $0,PCB_ONFAULT(%rdx) 381 movq $EFAULT,%rax 382 ret 383END(std_copyout) 384 385/* 386 * std_copyin(from_user, to_kernel, len) - MP SAFE 387 * %rdi, %rsi, %rdx 388 */ 389ENTRY(std_copyin) 390 movq PCPU(curthread),%rax 391 movq TD_PCB(%rax), %rax 392 movq $copyin_fault,PCB_ONFAULT(%rax) 393 movq %rsp,PCB_ONFAULT_SP(%rax) 394 testq %rdx,%rdx /* anything to do? */ 395 jz done_copyin 396 397 /* 398 * make sure address is valid 399 */ 400 movq %rdi,%rax 401 addq %rdx,%rax 402 jc copyin_fault 403 movq $VM_MAX_USER_ADDRESS,%rcx 404 cmpq %rcx,%rax 405 ja copyin_fault 406 407 xchgq %rdi,%rsi 408 movq %rdx,%rcx 409 shrq $3,%rcx /* copy longword-wise */ 410 jz 1f 411 rep 412 movsq 4131: movq %rdx,%rcx 414 andq $7,%rcx /* copy remaining bytes */ 415 jz done_copyin 416 rep 417 movsb 418 419done_copyin: 420 xorl %eax,%eax 421 movq PCPU(curthread),%rdx 422 movq TD_PCB(%rdx), %rdx 423 movq %rax,PCB_ONFAULT(%rdx) 424 ret 425 426 ALIGN_TEXT 427copyin_fault: 428 movq PCPU(curthread),%rdx 429 movq TD_PCB(%rdx), %rdx 430 movq $0,PCB_ONFAULT(%rdx) 431 movq $EFAULT,%rax 432 ret 433END(std_copyin) 434 435/* 436 * casu32 - Compare and set user integer. Returns -1 or the current value. 437 * dst = %rdi, old = %rsi, new = %rdx 438 */ 439ENTRY(casu32) 440 movq PCPU(curthread),%rcx 441 movq TD_PCB(%rcx), %rcx 442 movq $fusufault,PCB_ONFAULT(%rcx) 443 movq %rsp,PCB_ONFAULT_SP(%rcx) 444 445 movq $VM_MAX_USER_ADDRESS-4,%rax 446 cmpq %rax,%rdi /* verify address is valid */ 447 ja fusufault 448 449 movl %esi,%eax /* old */ 450 lock 451 cmpxchgl %edx,(%rdi) /* new = %edx */ 452 453 /* 454 * The old value is in %eax. If the store succeeded it will be the 455 * value we expected (old) from before the store, otherwise it will 456 * be the current value. 457 */ 458 459 movq PCPU(curthread),%rcx 460 movq TD_PCB(%rcx), %rcx 461 movq $0,PCB_ONFAULT(%rcx) 462 ret 463END(casu32) 464 465/* 466 * swapu32 - Swap int in user space. ptr = %rdi, val = %rsi 467 */ 468ENTRY(std_swapu32) 469 movq PCPU(curthread),%rcx 470 movq TD_PCB(%rcx), %rcx 471 movq $fusufault,PCB_ONFAULT(%rcx) 472 movq %rsp,PCB_ONFAULT_SP(%rcx) 473 474 movq $VM_MAX_USER_ADDRESS-4,%rax 475 cmpq %rax,%rdi /* verify address is valid */ 476 ja fusufault 477 478 movq %rsi,%rax /* old */ 479 xchgl %eax,(%rdi) 480 481 /* 482 * The old value is in %rax. If the store succeeded it will be the 483 * value we expected (old) from before the store, otherwise it will 484 * be the current value. 485 */ 486 487 movq PCPU(curthread),%rcx 488 movq TD_PCB(%rcx), %rcx 489 movq $0,PCB_ONFAULT(%rcx) 490 ret 491END(std_swapu32) 492 493ENTRY(std_fuwordadd32) 494 movq PCPU(curthread),%rcx 495 movq TD_PCB(%rcx), %rcx 496 movq $fusufault,PCB_ONFAULT(%rcx) 497 movq %rsp,PCB_ONFAULT_SP(%rcx) 498 499 movq $VM_MAX_USER_ADDRESS-4,%rax 500 cmpq %rax,%rdi /* verify address is valid */ 501 ja fusufault 502 503 movq %rsi,%rax /* qty to add */ 504 lock xaddl %eax,(%rdi) 505 506 /* 507 * The old value is in %rax. If the store succeeded it will be the 508 * value we expected (old) from before the store, otherwise it will 509 * be the current value. 510 */ 511 512 movq PCPU(curthread),%rcx 513 movq TD_PCB(%rcx), %rcx 514 movq $0,PCB_ONFAULT(%rcx) 515 ret 516END(std_fuwordadd32) 517 518/* 519 * casu64 - Compare and set user word. Returns -1 or the current value. 520 * dst = %rdi, old = %rsi, new = %rdx 521 */ 522ENTRY(casu64) 523 movq PCPU(curthread),%rcx 524 movq TD_PCB(%rcx), %rcx 525 movq $fusufault,PCB_ONFAULT(%rcx) 526 movq %rsp,PCB_ONFAULT_SP(%rcx) 527 528 movq $VM_MAX_USER_ADDRESS-8,%rax 529 cmpq %rax,%rdi /* verify address is valid */ 530 ja fusufault 531 532 movq %rsi,%rax /* old */ 533 lock 534 cmpxchgq %rdx,(%rdi) /* new = %rdx */ 535 536 /* 537 * The old value is in %rax. If the store succeeded it will be the 538 * value we expected (old) from before the store, otherwise it will 539 * be the current value. 540 */ 541 542 movq PCPU(curthread),%rcx 543 movq TD_PCB(%rcx), %rcx 544 movq $0,PCB_ONFAULT(%rcx) 545 ret 546END(casu64) 547 548/* 549 * swapu64 - Swap long in user space. ptr = %rdi, val = %rsi 550 */ 551ENTRY(std_swapu64) 552 movq PCPU(curthread),%rcx 553 movq TD_PCB(%rcx), %rcx 554 movq $fusufault,PCB_ONFAULT(%rcx) 555 movq %rsp,PCB_ONFAULT_SP(%rcx) 556 557 movq $VM_MAX_USER_ADDRESS-8,%rax 558 cmpq %rax,%rdi /* verify address is valid */ 559 ja fusufault 560 561 movq %rsi,%rax /* old */ 562 xchgq %rax,(%rdi) 563 564 /* 565 * The old value is in %rax. If the store succeeded it will be the 566 * value we expected (old) from before the store, otherwise it will 567 * be the current value. 568 */ 569 570 movq PCPU(curthread),%rcx 571 movq TD_PCB(%rcx), %rcx 572 movq $0,PCB_ONFAULT(%rcx) 573 ret 574END(std_swapu64) 575 576ENTRY(std_fuwordadd64) 577 movq PCPU(curthread),%rcx 578 movq TD_PCB(%rcx), %rcx 579 movq $fusufault,PCB_ONFAULT(%rcx) 580 movq %rsp,PCB_ONFAULT_SP(%rcx) 581 582 movq $VM_MAX_USER_ADDRESS-8,%rax 583 cmpq %rax,%rdi /* verify address is valid */ 584 ja fusufault 585 586 movq %rsi,%rax /* value to add */ 587 lock xaddq %rax,(%rdi) 588 589 /* 590 * The old value is in %rax. If the store succeeded it will be the 591 * value we expected (old) from before the store, otherwise it will 592 * be the current value. 593 */ 594 595 movq PCPU(curthread),%rcx 596 movq TD_PCB(%rcx), %rcx 597 movq $0,PCB_ONFAULT(%rcx) 598 ret 599END(std_fuwordadd64) 600 601/* 602 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit 603 * byte from user memory. All these functions are MPSAFE. 604 * addr = %rdi 605 */ 606 607ENTRY(std_fuword64) 608 movq PCPU(curthread),%rcx 609 movq TD_PCB(%rcx), %rcx 610 movq $fusufault,PCB_ONFAULT(%rcx) 611 movq %rsp,PCB_ONFAULT_SP(%rcx) 612 613 movq $VM_MAX_USER_ADDRESS-8,%rax 614 cmpq %rax,%rdi /* verify address is valid */ 615 ja fusufault 616 617 movq (%rdi),%rax 618 movq $0,PCB_ONFAULT(%rcx) 619 ret 620END(std_fuword64) 621 622ENTRY(std_fuword32) 623 movq PCPU(curthread),%rcx 624 movq TD_PCB(%rcx), %rcx 625 movq $fusufault,PCB_ONFAULT(%rcx) 626 movq %rsp,PCB_ONFAULT_SP(%rcx) 627 628 movq $VM_MAX_USER_ADDRESS-4,%rax 629 cmpq %rax,%rdi /* verify address is valid */ 630 ja fusufault 631 632 movl (%rdi),%eax 633 movq $0,PCB_ONFAULT(%rcx) 634 ret 635END(std_fuword32) 636 637ENTRY(std_fubyte) 638 movq PCPU(curthread),%rcx 639 movq TD_PCB(%rcx), %rcx 640 movq $fusufault,PCB_ONFAULT(%rcx) 641 movq %rsp,PCB_ONFAULT_SP(%rcx) 642 643 movq $VM_MAX_USER_ADDRESS-1,%rax 644 cmpq %rax,%rdi 645 ja fusufault 646 647 movzbl (%rdi),%eax 648 movq $0,PCB_ONFAULT(%rcx) 649 ret 650 651 ALIGN_TEXT 652fusufault: 653 movq PCPU(curthread),%rcx 654 xorl %eax,%eax 655 movq TD_PCB(%rcx), %rcx 656 movq %rax,PCB_ONFAULT(%rcx) 657 decq %rax 658 ret 659END(std_fubyte) 660 661/* 662 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to 663 * user memory. All these functions are MPSAFE. 664 * 665 * addr = %rdi, value = %rsi 666 * 667 * Write a long 668 */ 669ENTRY(std_suword64) 670 movq PCPU(curthread),%rcx 671 movq TD_PCB(%rcx), %rcx 672 movq $fusufault,PCB_ONFAULT(%rcx) 673 movq %rsp,PCB_ONFAULT_SP(%rcx) 674 675 movq $VM_MAX_USER_ADDRESS-8,%rax 676 cmpq %rax,%rdi /* verify address validity */ 677 ja fusufault 678 679 movq %rsi,(%rdi) 680 xorl %eax,%eax 681 movq PCPU(curthread),%rcx 682 movq TD_PCB(%rcx), %rcx 683 movq %rax,PCB_ONFAULT(%rcx) 684 ret 685END(std_suword64) 686 687/* 688 * Write an int 689 */ 690ENTRY(std_suword32) 691 movq PCPU(curthread),%rcx 692 movq TD_PCB(%rcx), %rcx 693 movq $fusufault,PCB_ONFAULT(%rcx) 694 movq %rsp,PCB_ONFAULT_SP(%rcx) 695 696 movq $VM_MAX_USER_ADDRESS-4,%rax 697 cmpq %rax,%rdi /* verify address validity */ 698 ja fusufault 699 700 movl %esi,(%rdi) 701 xorl %eax,%eax 702 movq PCPU(curthread),%rcx 703 movq TD_PCB(%rcx), %rcx 704 movq %rax,PCB_ONFAULT(%rcx) 705 ret 706END(std_suword32) 707 708ENTRY(std_subyte) 709 movq PCPU(curthread),%rcx 710 movq TD_PCB(%rcx), %rcx 711 movq $fusufault,PCB_ONFAULT(%rcx) 712 movq %rsp,PCB_ONFAULT_SP(%rcx) 713 714 movq $VM_MAX_USER_ADDRESS-1,%rax 715 cmpq %rax,%rdi /* verify address validity */ 716 ja fusufault 717 718 movl %esi,%eax 719 movb %al,(%rdi) 720 xorl %eax,%eax 721 movq PCPU(curthread),%rcx /* restore trashed register */ 722 movq TD_PCB(%rcx), %rcx 723 movq %rax,PCB_ONFAULT(%rcx) 724 ret 725END(std_subyte) 726 727/* 728 * std_copyinstr(from, to, maxlen, int *lencopied) - MP SAFE 729 * %rdi, %rsi, %rdx, %rcx 730 * 731 * copy a string from from to to, stop when a 0 character is reached. 732 * return ENAMETOOLONG if string is longer than maxlen, and 733 * EFAULT on protection violations. If lencopied is non-zero, 734 * return the actual length in *lencopied. 735 */ 736ENTRY(std_copyinstr) 737 movq %rdx,%r8 /* %r8 = maxlen */ 738 movq %rcx,%r9 /* %r9 = *len */ 739 movq PCPU(curthread),%rcx 740 movq TD_PCB(%rcx), %rcx 741 movq $cpystrflt,PCB_ONFAULT(%rcx) 742 movq %rsp,PCB_ONFAULT_SP(%rcx) 743 744 movq $VM_MAX_USER_ADDRESS,%rax 745 746 /* make sure 'from' is within bounds */ 747 subq %rdi,%rax 748 jbe cpystrflt 749 750 /* restrict maxlen to <= VM_MAX_USER_ADDRESS-from */ 751 cmpq %rdx,%rax 752 jae 1f 753 movq %rax,%rdx 754 movq %rax,%r8 7551: 756 incq %rdx 757 7582: 759 decq %rdx 760 jz 3f 761 762 movb (%rdi),%al /* faster than lodsb+stosb */ 763 movb %al,(%rsi) 764 leaq 1(%rdi),%rdi 765 leaq 1(%rsi),%rsi 766 testb %al,%al 767 jnz 2b 768 769 /* Success -- 0 byte reached */ 770 decq %rdx 771 xorl %eax,%eax 772 jmp cpystrflt_x 7733: 774 /* rdx is zero - return ENAMETOOLONG or EFAULT */ 775 movq $VM_MAX_USER_ADDRESS,%rax 776 cmpq %rax,%rsi 777 jae cpystrflt 7784: 779 movq $ENAMETOOLONG,%rax 780 jmp cpystrflt_x 781 782cpystrflt: 783 movq $EFAULT,%rax 784 785cpystrflt_x: 786 /* set *lencopied and return %eax */ 787 movq PCPU(curthread),%rcx 788 movq TD_PCB(%rcx), %rcx 789 movq $0,PCB_ONFAULT(%rcx) 790 791 testq %r9,%r9 792 jz 1f 793 subq %rdx,%r8 794 movq %r8,(%r9) 7951: 796 ret 797END(std_copyinstr) 798 799/* 800 * copystr(from, to, maxlen, int *lencopied) - MP SAFE 801 * %rdi, %rsi, %rdx, %rcx 802 */ 803ENTRY(copystr) 804 movq %rdx,%r8 /* %r8 = maxlen */ 805 806 incq %rdx 8071: 808 decq %rdx 809 jz 4f 810 811 movb (%rdi),%al /* faster than lodsb+stosb */ 812 movb %al,(%rsi) 813 leaq 1(%rdi),%rdi 814 leaq 1(%rsi),%rsi 815 testb %al,%al 816 jnz 1b 817 818 /* Success -- 0 byte reached */ 819 decq %rdx 820 xorl %eax,%eax 821 jmp 6f 8224: 823 /* rdx is zero -- return ENAMETOOLONG */ 824 movq $ENAMETOOLONG,%rax 825 8266: 827 828 testq %rcx,%rcx 829 jz 7f 830 /* set *lencopied and return %rax */ 831 subq %rdx,%r8 832 movq %r8,(%rcx) 8337: 834 ret 835END(copystr) 836 837/* 838 * Handling of special x86_64 registers and descriptor tables etc 839 * %rdi 840 */ 841/* void lgdt(struct region_descriptor *rdp); */ 842ENTRY(lgdt) 843 /* reload the descriptor table */ 844 lgdt (%rdi) 845 846 /* flush the prefetch q */ 847 jmp 1f 848 nop 8491: 850 movl $KDSEL,%eax 851 movl %eax,%ds 852 movl %eax,%es 853 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */ 854 movl %eax,%gs /* Beware, use wrmsr to set 64 bit base */ 855 movl %eax,%ss 856 857 /* reload code selector by turning return into intersegmental return */ 858 popq %rax 859 pushq $KCSEL 860 pushq %rax 861 MEXITCOUNT 862 lretq 863END(lgdt) 864 865/*****************************************************************************/ 866/* setjmp, longjmp */ 867/*****************************************************************************/ 868 869ENTRY(setjmp) 870 movq %rbx,0(%rdi) /* save rbx */ 871 movq %rsp,8(%rdi) /* save rsp */ 872 movq %rbp,16(%rdi) /* save rbp */ 873 movq %r12,24(%rdi) /* save r12 */ 874 movq %r13,32(%rdi) /* save r13 */ 875 movq %r14,40(%rdi) /* save r14 */ 876 movq %r15,48(%rdi) /* save r15 */ 877 movq 0(%rsp),%rdx /* get rta */ 878 movq %rdx,56(%rdi) /* save rip */ 879 xorl %eax,%eax /* return(0); */ 880 ret 881END(setjmp) 882 883ENTRY(longjmp) 884 movq 0(%rdi),%rbx /* restore rbx */ 885 movq 8(%rdi),%rsp /* restore rsp */ 886 movq 16(%rdi),%rbp /* restore rbp */ 887 movq 24(%rdi),%r12 /* restore r12 */ 888 movq 32(%rdi),%r13 /* restore r13 */ 889 movq 40(%rdi),%r14 /* restore r14 */ 890 movq 48(%rdi),%r15 /* restore r15 */ 891 movq 56(%rdi),%rdx /* get rta */ 892 movq %rdx,0(%rsp) /* put in return frame */ 893 xorl %eax,%eax /* return(1); */ 894 incl %eax 895 ret 896END(longjmp) 897 898/* 899 * Support for reading MSRs in the safe manner. 900 */ 901ENTRY(rdmsr_safe) 902/* int rdmsr_safe(u_int msr, uint64_t *data) */ 903 movq PCPU(curthread),%r8 904 movq TD_PCB(%r8), %r8 905 movq $msr_onfault,PCB_ONFAULT(%r8) 906 movq %rsp,PCB_ONFAULT_SP(%r8) 907 movl %edi,%ecx 908 rdmsr /* Read MSR pointed by %ecx. Returns 909 hi byte in edx, lo in %eax */ 910 salq $32,%rdx /* sign-shift %rdx left */ 911 movl %eax,%eax /* zero-extend %eax -> %rax */ 912 orq %rdx,%rax 913 movq %rax,(%rsi) 914 xorq %rax,%rax 915 movq %rax,PCB_ONFAULT(%r8) 916 ret 917END(rdmsr_safe) 918 919/* 920 * Support for writing MSRs in the safe manner. 921 */ 922ENTRY(wrmsr_safe) 923/* int wrmsr_safe(u_int msr, uint64_t data) */ 924 movq PCPU(curthread),%r8 925 movq TD_PCB(%r8), %r8 926 movq $msr_onfault,PCB_ONFAULT(%r8) 927 movq %rsp,PCB_ONFAULT_SP(%r8) 928 movl %edi,%ecx 929 movl %esi,%eax 930 sarq $32,%rsi 931 movl %esi,%edx 932 wrmsr /* Write MSR pointed by %ecx. Accepts 933 hi byte in edx, lo in %eax. */ 934 xorq %rax,%rax 935 movq %rax,PCB_ONFAULT(%r8) 936 ret 937END(wrmsr_safe) 938 939/* 940 * MSR operations fault handler 941 */ 942 ALIGN_TEXT 943msr_onfault: 944 movq PCPU(curthread),%r8 945 movq TD_PCB(%r8), %r8 946 movq $0,PCB_ONFAULT(%r8) 947 movl $EFAULT,%eax 948 ret 949