1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * Copyright (c) 2003 Peter Wemm. 4 * Copyright (c) 2008 The DragonFly Project. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD: src/sys/amd64/amd64/support.S,v 1.127 2007/05/23 08:33:04 kib Exp $ 32 */ 33 34#include <machine/asmacros.h> 35#include <machine/pmap.h> 36 37#include "assym.s" 38 39 ALIGN_DATA 40 41 .text 42 43/* 44 * bzero(ptr:%rdi, bytes:%rsi) 45 * 46 * Using rep stosq is 70% faster than a %rax loop and almost as fast as 47 * a %xmm0 loop on a modern intel cpu. 48 * 49 * Do not use non-termportal instructions here as we do not know the caller's 50 * intent. 51 */ 52ENTRY(bzero) 53 movq %rsi,%rcx 54 xorl %eax,%eax 55 shrq $3,%rcx 56 rep 57 stosq 58 movq %rsi,%rcx 59 andq $7,%rcx 60 jnz 1f 61 ret 621: rep 63 stosb 64 ret 65END(bzero) 66 67 .weak _bzero 68 .equ _bzero, bzero 69 70/* 71 * void *memset(ptr:%rdi, char:%rsi, bytes:%rdx) 72 * 73 * Same as bzero except we load the char into all byte 74 * positions of %rax. Returns original (ptr). 75 */ 76ENTRY(memset) 77 movzbq %sil,%r8 78 movabs $0x0101010101010101,%rax 79 imulq %r8,%rax 80 81 movq %rdi,%r9 82 movq %rdx,%rcx 83 shrq $3,%rcx 84 rep 85 stosq 86 movq %rdx,%rcx 87 andq $7,%rcx 88 jnz 1f 89 movq %r9,%rax 90 ret 911: rep 92 stosb 93 movq %r9,%rax 94 ret 95END(memset) 96 97 .weak _memset 98 .equ _memset, memset 99 100/* 101 * pagezero(ptr:%rdi) 102 * 103 * Using rep stosq is nearly as fast as using %xmm0 on a modern intel cpu, 104 * and about 70% faster than a %rax loop. 105 * 106 * Do not use non-termportal instructions here as we do not know the caller's 107 * intent. 108 */ 109#if 0 110 111ENTRY(pagezero) 112 movq $PAGE_SIZE>>3,%rcx 113 xorl %eax,%eax 114 rep 115 stosq 116 ret 117END(pagezero) 118 119#endif 120 121ENTRY(pagezero) 122 addq $4096,%rdi 123 movq $-4096,%rax 124 ALIGN_TEXT 1251: 126 movq $0,(%rdi,%rax,1) 127 addq $8,%rax 128 jne 1b 129 ret 130END(pagezero) 131 132/* 133 * bcmp(ptr:%rdi, ptr:%rsi, bytes:%rdx) 134 */ 135ENTRY(bcmp) 136 movq %rdx,%rcx 137 shrq $3,%rcx 138 repe 139 cmpsq 140 jne 1f 141 142 movq %rdx,%rcx 143 andq $7,%rcx 144 je 1f 145 repe 146 cmpsb 1471: 148 setne %al 149 movsbl %al,%eax 150 ret 151END(bcmp) 152 153/* 154 * bcopy(src:%rdi, dst:%rsi, cnt:%rdx) 155 * 156 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 157 */ 158ENTRY(bcopy) 159 xchgq %rsi,%rdi 160 movq %rdx,%rcx 161 162 movq %rdi,%rax 163 subq %rsi,%rax 164 cmpq %rcx,%rax /* overlapping && src < dst? */ 165 jb 2f 166 167 shrq $3,%rcx /* copy by 64-bit words */ 168 rep 169 movsq 170 movq %rdx,%rcx 171 andq $7,%rcx /* any bytes left? */ 172 jnz 1f 173 ret 1741: rep 175 movsb 176 ret 177 178 ALIGN_TEXT 1792: 180 addq %rcx,%rdi /* copy backwards */ 181 addq %rcx,%rsi 182 std 183 decq %rdi 184 decq %rsi 185 andq $7,%rcx /* any fractional bytes? */ 186 jz 3f 187 rep 188 movsb 1893: movq %rdx,%rcx /* copy by 32-bit words */ 190 shrq $3,%rcx 191 subq $7,%rsi 192 subq $7,%rdi 193 rep 194 movsq 195 cld 196 ret 197END(bcopy) 198 199 /* 200 * Use in situations where a bcopy function pointer is needed. 201 */ 202 .weak _bcopy 203 .equ _bcopy, bcopy 204 205 /* 206 * memmove(dst:%rdi, src:%rsi, cnt:%rdx) 207 * (same as bcopy but without the xchgq, and must return (dst)). 208 * 209 * NOTE: gcc builtin backs-off to memmove() call 210 * 211 * NOTE: We leave %rdi in %rax for the return value. 212 */ 213ENTRY(memmove) 214 movq %rdx,%rcx 215 movq %rdi,%rax /* return value */ 216 movq %rdi,%r8 217 subq %rsi,%r8 218 cmpq %rcx,%r8 /* overlapping && src < dst? */ 219 jb 2f 220 221 shrq $3,%rcx /* copy by 64-bit words */ 222 rep 223 movsq 224 movq %rdx,%rcx 225 andq $7,%rcx /* any bytes left? */ 226 jnz 1f 227 ret 2281: rep 229 movsb 230 ret 231 232 ALIGN_TEXT 2332: 234 addq %rcx,%rdi /* copy backwards */ 235 addq %rcx,%rsi 236 std 237 decq %rdi 238 decq %rsi 239 andq $7,%rcx /* any fractional bytes? */ 240 jz 3f 241 rep 242 movsb 2433: movq %rdx,%rcx /* copy by 32-bit words */ 244 shrq $3,%rcx 245 subq $7,%rsi 246 subq $7,%rdi 247 rep 248 movsq 249 cld 250 ret 251END(memmove) 252 253 .weak _memmove 254 .equ _memmove, memmove 255 256ENTRY(reset_dbregs) 257 movq $0x200,%rax /* the manual says that bit 10 must be set to 1 */ 258 movq %rax,%dr7 /* disable all breapoints first */ 259 movq $0,%rax 260 movq %rax,%dr0 261 movq %rax,%dr1 262 movq %rax,%dr2 263 movq %rax,%dr3 264 movq %rax,%dr6 265 ret 266END(reset_dbregs) 267 268/* 269 * memcpy(dst:%rdi, src:%rsi, bytes:%rdx) 270 * 271 * NOTE: memcpy does not support overlapping copies 272 * NOTE: returns dst 273 */ 274ENTRY(memcpy) 275 movq %rdi,%r8 276 movq %rdx,%rcx 277 shrq $3,%rcx /* copy by 64-bit words */ 278 rep 279 movsq 280 movq %rdx,%rcx 281 andq $7,%rcx /* any bytes left? */ 282 jnz 1f 283 movq %r8,%rax 284 ret 2851: rep 286 movsb 287 movq %r8,%rax 288 ret 289END(memcpy) 290 291 .weak _memcpy 292 .equ _memcpy, memcpy 293 294/* fillw(pat, base, cnt) */ 295/* %rdi,%rsi, %rdx */ 296ENTRY(fillw) 297 movq %rdi,%rax 298 movq %rsi,%rdi 299 movq %rdx,%rcx 300 rep 301 stosw 302 ret 303END(fillw) 304 305/*****************************************************************************/ 306/* copyout and fubyte family */ 307/*****************************************************************************/ 308/* 309 * Access user memory from inside the kernel. These routines should be 310 * the only places that do this. 311 * 312 * These routines set curpcb->onfault for the time they execute. When a 313 * protection violation occurs inside the functions, the trap handler 314 * returns to *curpcb->onfault instead of the function. 315 */ 316 317/* 318 * uint64_t:%rax kreadmem64(addr:%rdi) 319 * 320 * Read kernel or user memory with fault protection. 321 */ 322ENTRY(kreadmem64) 323 movq PCPU(curthread),%rcx 324 movq TD_PCB(%rcx), %rcx 325 movq $kreadmem64fault,PCB_ONFAULT(%rcx) 326 movq %rsp,PCB_ONFAULT_SP(%rcx) 327 328 movq (%rdi),%rax 329 movq $0,PCB_ONFAULT(%rcx) 330 ret 331 332kreadmem64fault: 333 movq PCPU(curthread),%rcx 334 xorl %eax,%eax 335 movq TD_PCB(%rcx),%rcx 336 movq %rax,PCB_ONFAULT(%rcx) 337 decq %rax 338 ret 339END(kreadmem64) 340 341/* 342 * std_copyout(from_kernel, to_user, len) - MP SAFE 343 * %rdi, %rsi, %rdx 344 */ 345ENTRY(std_copyout) 346 movq PCPU(curthread),%rax 347 movq TD_PCB(%rax), %rax 348 movq $copyout_fault,PCB_ONFAULT(%rax) 349 movq %rsp,PCB_ONFAULT_SP(%rax) 350 testq %rdx,%rdx /* anything to do? */ 351 jz done_copyout 352 353 /* 354 * Check explicitly for non-user addresses. If 486 write protection 355 * is being used, this check is essential because we are in kernel 356 * mode so the h/w does not provide any protection against writing 357 * kernel addresses. 358 */ 359 360 /* 361 * First, prevent address wrapping. 362 */ 363 movq %rsi,%rax 364 addq %rdx,%rax 365 jc copyout_fault 366/* 367 * XXX STOP USING VM_MAX_USER_ADDRESS. 368 * It is an end address, not a max, so every time it is used correctly it 369 * looks like there is an off by one error, and of course it caused an off 370 * by one error in several places. 371 */ 372 movq $VM_MAX_USER_ADDRESS,%rcx 373 cmpq %rcx,%rax 374 ja copyout_fault 375 376 xchgq %rdi,%rsi 377 /* bcopy(%rsi, %rdi, %rdx) */ 378 movq %rdx,%rcx 379 380 shrq $3,%rcx 381 jz 1f 382 rep 383 movsq 3841: movq %rdx,%rcx 385 andq $7,%rcx 386 jz done_copyout 387 rep 388 movsb 389 390done_copyout: 391 xorl %eax,%eax 392 movq PCPU(curthread),%rdx 393 movq TD_PCB(%rdx), %rdx 394 movq %rax,PCB_ONFAULT(%rdx) 395 ret 396 397 ALIGN_TEXT 398copyout_fault: 399 movq PCPU(curthread),%rdx 400 movq TD_PCB(%rdx), %rdx 401 movq $0,PCB_ONFAULT(%rdx) 402 movq $EFAULT,%rax 403 ret 404END(std_copyout) 405 406/* 407 * std_copyin(from_user, to_kernel, len) - MP SAFE 408 * %rdi, %rsi, %rdx 409 */ 410ENTRY(std_copyin) 411 movq PCPU(curthread),%rax 412 movq TD_PCB(%rax), %rax 413 movq $copyin_fault,PCB_ONFAULT(%rax) 414 movq %rsp,PCB_ONFAULT_SP(%rax) 415 testq %rdx,%rdx /* anything to do? */ 416 jz done_copyin 417 418 /* 419 * make sure address is valid 420 */ 421 movq %rdi,%rax 422 addq %rdx,%rax 423 jc copyin_fault 424 movq $VM_MAX_USER_ADDRESS,%rcx 425 cmpq %rcx,%rax 426 ja copyin_fault 427 428 xchgq %rdi,%rsi 429 movq %rdx,%rcx 430 shrq $3,%rcx /* copy longword-wise */ 431 jz 1f 432 rep 433 movsq 4341: movq %rdx,%rcx 435 andq $7,%rcx /* copy remaining bytes */ 436 jz done_copyin 437 rep 438 movsb 439 440done_copyin: 441 xorl %eax,%eax 442 movq PCPU(curthread),%rdx 443 movq TD_PCB(%rdx), %rdx 444 movq %rax,PCB_ONFAULT(%rdx) 445 ret 446 447 ALIGN_TEXT 448copyin_fault: 449 movq PCPU(curthread),%rdx 450 movq TD_PCB(%rdx), %rdx 451 movq $0,PCB_ONFAULT(%rdx) 452 movq $EFAULT,%rax 453 ret 454END(std_copyin) 455 456/* 457 * casu32 - Compare and set user integer. Returns -1 or the current value. 458 * dst = %rdi, old = %rsi, new = %rdx 459 */ 460ENTRY(casu32) 461 movq PCPU(curthread),%rcx 462 movq TD_PCB(%rcx), %rcx 463 movq $fusufault,PCB_ONFAULT(%rcx) 464 movq %rsp,PCB_ONFAULT_SP(%rcx) 465 466 movq $VM_MAX_USER_ADDRESS-4,%rax 467 cmpq %rax,%rdi /* verify address is valid */ 468 ja fusufault 469 470 movl %esi,%eax /* old */ 471 lock 472 cmpxchgl %edx,(%rdi) /* new = %edx */ 473 474 /* 475 * The old value is in %eax. If the store succeeded it will be the 476 * value we expected (old) from before the store, otherwise it will 477 * be the current value. 478 */ 479 480 movq PCPU(curthread),%rcx 481 movq TD_PCB(%rcx), %rcx 482 movq $0,PCB_ONFAULT(%rcx) 483 ret 484END(casu32) 485 486/* 487 * swapu32 - Swap int in user space. ptr = %rdi, val = %rsi 488 */ 489ENTRY(std_swapu32) 490 movq PCPU(curthread),%rcx 491 movq TD_PCB(%rcx), %rcx 492 movq $fusufault,PCB_ONFAULT(%rcx) 493 movq %rsp,PCB_ONFAULT_SP(%rcx) 494 495 movq $VM_MAX_USER_ADDRESS-4,%rax 496 cmpq %rax,%rdi /* verify address is valid */ 497 ja fusufault 498 499 movq %rsi,%rax /* old */ 500 xchgl %eax,(%rdi) 501 502 /* 503 * The old value is in %rax. If the store succeeded it will be the 504 * value we expected (old) from before the store, otherwise it will 505 * be the current value. 506 */ 507 508 movq PCPU(curthread),%rcx 509 movq TD_PCB(%rcx), %rcx 510 movq $0,PCB_ONFAULT(%rcx) 511 ret 512END(std_swapu32) 513 514ENTRY(std_fuwordadd32) 515 movq PCPU(curthread),%rcx 516 movq TD_PCB(%rcx), %rcx 517 movq $fusufault,PCB_ONFAULT(%rcx) 518 movq %rsp,PCB_ONFAULT_SP(%rcx) 519 520 movq $VM_MAX_USER_ADDRESS-4,%rax 521 cmpq %rax,%rdi /* verify address is valid */ 522 ja fusufault 523 524 movq %rsi,%rax /* qty to add */ 525 lock xaddl %eax,(%rdi) 526 527 /* 528 * The old value is in %rax. If the store succeeded it will be the 529 * value we expected (old) from before the store, otherwise it will 530 * be the current value. 531 */ 532 533 movq PCPU(curthread),%rcx 534 movq TD_PCB(%rcx), %rcx 535 movq $0,PCB_ONFAULT(%rcx) 536 ret 537END(std_fuwordadd32) 538 539/* 540 * casu64 - Compare and set user word. Returns -1 or the current value. 541 * dst = %rdi, old = %rsi, new = %rdx 542 */ 543ENTRY(casu64) 544 movq PCPU(curthread),%rcx 545 movq TD_PCB(%rcx), %rcx 546 movq $fusufault,PCB_ONFAULT(%rcx) 547 movq %rsp,PCB_ONFAULT_SP(%rcx) 548 549 movq $VM_MAX_USER_ADDRESS-8,%rax 550 cmpq %rax,%rdi /* verify address is valid */ 551 ja fusufault 552 553 movq %rsi,%rax /* old */ 554 lock 555 cmpxchgq %rdx,(%rdi) /* new = %rdx */ 556 557 /* 558 * The old value is in %rax. If the store succeeded it will be the 559 * value we expected (old) from before the store, otherwise it will 560 * be the current value. 561 */ 562 563 movq PCPU(curthread),%rcx 564 movq TD_PCB(%rcx), %rcx 565 movq $0,PCB_ONFAULT(%rcx) 566 ret 567END(casu64) 568 569/* 570 * swapu64 - Swap long in user space. ptr = %rdi, val = %rsi 571 */ 572ENTRY(std_swapu64) 573 movq PCPU(curthread),%rcx 574 movq TD_PCB(%rcx), %rcx 575 movq $fusufault,PCB_ONFAULT(%rcx) 576 movq %rsp,PCB_ONFAULT_SP(%rcx) 577 578 movq $VM_MAX_USER_ADDRESS-8,%rax 579 cmpq %rax,%rdi /* verify address is valid */ 580 ja fusufault 581 582 movq %rsi,%rax /* old */ 583 xchgq %rax,(%rdi) 584 585 /* 586 * The old value is in %rax. If the store succeeded it will be the 587 * value we expected (old) from before the store, otherwise it will 588 * be the current value. 589 */ 590 591 movq PCPU(curthread),%rcx 592 movq TD_PCB(%rcx), %rcx 593 movq $0,PCB_ONFAULT(%rcx) 594 ret 595END(std_swapu64) 596 597ENTRY(std_fuwordadd64) 598 movq PCPU(curthread),%rcx 599 movq TD_PCB(%rcx), %rcx 600 movq $fusufault,PCB_ONFAULT(%rcx) 601 movq %rsp,PCB_ONFAULT_SP(%rcx) 602 603 movq $VM_MAX_USER_ADDRESS-8,%rax 604 cmpq %rax,%rdi /* verify address is valid */ 605 ja fusufault 606 607 movq %rsi,%rax /* value to add */ 608 lock xaddq %rax,(%rdi) 609 610 /* 611 * The old value is in %rax. If the store succeeded it will be the 612 * value we expected (old) from before the store, otherwise it will 613 * be the current value. 614 */ 615 616 movq PCPU(curthread),%rcx 617 movq TD_PCB(%rcx), %rcx 618 movq $0,PCB_ONFAULT(%rcx) 619 ret 620END(std_fuwordadd64) 621 622/* 623 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit 624 * byte from user memory. All these functions are MPSAFE. 625 * addr = %rdi 626 */ 627 628ENTRY(std_fuword64) 629 movq PCPU(curthread),%rcx 630 movq TD_PCB(%rcx), %rcx 631 movq $fusufault,PCB_ONFAULT(%rcx) 632 movq %rsp,PCB_ONFAULT_SP(%rcx) 633 634 movq $VM_MAX_USER_ADDRESS-8,%rax 635 cmpq %rax,%rdi /* verify address is valid */ 636 ja fusufault 637 638 movq (%rdi),%rax 639 movq $0,PCB_ONFAULT(%rcx) 640 ret 641END(std_fuword64) 642 643ENTRY(std_fuword32) 644 movq PCPU(curthread),%rcx 645 movq TD_PCB(%rcx), %rcx 646 movq $fusufault,PCB_ONFAULT(%rcx) 647 movq %rsp,PCB_ONFAULT_SP(%rcx) 648 649 movq $VM_MAX_USER_ADDRESS-4,%rax 650 cmpq %rax,%rdi /* verify address is valid */ 651 ja fusufault 652 653 movl (%rdi),%eax 654 movq $0,PCB_ONFAULT(%rcx) 655 ret 656END(std_fuword32) 657 658ENTRY(std_fubyte) 659 movq PCPU(curthread),%rcx 660 movq TD_PCB(%rcx), %rcx 661 movq $fusufault,PCB_ONFAULT(%rcx) 662 movq %rsp,PCB_ONFAULT_SP(%rcx) 663 664 movq $VM_MAX_USER_ADDRESS-1,%rax 665 cmpq %rax,%rdi 666 ja fusufault 667 668 movzbl (%rdi),%eax 669 movq $0,PCB_ONFAULT(%rcx) 670 ret 671 672 ALIGN_TEXT 673fusufault: 674 movq PCPU(curthread),%rcx 675 xorl %eax,%eax 676 movq TD_PCB(%rcx), %rcx 677 movq %rax,PCB_ONFAULT(%rcx) 678 decq %rax 679 ret 680END(std_fubyte) 681 682/* 683 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to 684 * user memory. All these functions are MPSAFE. 685 * 686 * addr = %rdi, value = %rsi 687 * 688 * Write a long 689 */ 690ENTRY(std_suword64) 691 movq PCPU(curthread),%rcx 692 movq TD_PCB(%rcx), %rcx 693 movq $fusufault,PCB_ONFAULT(%rcx) 694 movq %rsp,PCB_ONFAULT_SP(%rcx) 695 696 movq $VM_MAX_USER_ADDRESS-8,%rax 697 cmpq %rax,%rdi /* verify address validity */ 698 ja fusufault 699 700 movq %rsi,(%rdi) 701 xorl %eax,%eax 702 movq PCPU(curthread),%rcx 703 movq TD_PCB(%rcx), %rcx 704 movq %rax,PCB_ONFAULT(%rcx) 705 ret 706END(std_suword64) 707 708/* 709 * Write an int 710 */ 711ENTRY(std_suword32) 712 movq PCPU(curthread),%rcx 713 movq TD_PCB(%rcx), %rcx 714 movq $fusufault,PCB_ONFAULT(%rcx) 715 movq %rsp,PCB_ONFAULT_SP(%rcx) 716 717 movq $VM_MAX_USER_ADDRESS-4,%rax 718 cmpq %rax,%rdi /* verify address validity */ 719 ja fusufault 720 721 movl %esi,(%rdi) 722 xorl %eax,%eax 723 movq PCPU(curthread),%rcx 724 movq TD_PCB(%rcx), %rcx 725 movq %rax,PCB_ONFAULT(%rcx) 726 ret 727END(std_suword32) 728 729ENTRY(std_subyte) 730 movq PCPU(curthread),%rcx 731 movq TD_PCB(%rcx), %rcx 732 movq $fusufault,PCB_ONFAULT(%rcx) 733 movq %rsp,PCB_ONFAULT_SP(%rcx) 734 735 movq $VM_MAX_USER_ADDRESS-1,%rax 736 cmpq %rax,%rdi /* verify address validity */ 737 ja fusufault 738 739 movl %esi,%eax 740 movb %al,(%rdi) 741 xorl %eax,%eax 742 movq PCPU(curthread),%rcx /* restore trashed register */ 743 movq TD_PCB(%rcx), %rcx 744 movq %rax,PCB_ONFAULT(%rcx) 745 ret 746END(std_subyte) 747 748/* 749 * std_copyinstr(from, to, maxlen, int *lencopied) - MP SAFE 750 * %rdi, %rsi, %rdx, %rcx 751 * 752 * copy a string from from to to, stop when a 0 character is reached. 753 * return ENAMETOOLONG if string is longer than maxlen, and 754 * EFAULT on protection violations. If lencopied is non-zero, 755 * return the actual length in *lencopied. 756 */ 757ENTRY(std_copyinstr) 758 movq %rdx,%r8 /* %r8 = maxlen */ 759 movq %rcx,%r9 /* %r9 = *len */ 760 movq PCPU(curthread),%rcx 761 movq TD_PCB(%rcx), %rcx 762 movq $cpystrflt,PCB_ONFAULT(%rcx) 763 movq %rsp,PCB_ONFAULT_SP(%rcx) 764 765 movq $VM_MAX_USER_ADDRESS,%rax 766 767 /* make sure 'from' is within bounds */ 768 subq %rdi,%rax 769 jbe cpystrflt 770 771 /* restrict maxlen to <= VM_MAX_USER_ADDRESS-from */ 772 cmpq %rdx,%rax 773 jae 1f 774 movq %rax,%rdx 775 movq %rax,%r8 7761: 777 incq %rdx 778 7792: 780 decq %rdx 781 jz 3f 782 783 movb (%rdi),%al /* faster than lodsb+stosb */ 784 movb %al,(%rsi) 785 leaq 1(%rdi),%rdi 786 leaq 1(%rsi),%rsi 787 testb %al,%al 788 jnz 2b 789 790 /* Success -- 0 byte reached */ 791 decq %rdx 792 xorl %eax,%eax 793 jmp cpystrflt_x 7943: 795 /* rdx is zero - return ENAMETOOLONG or EFAULT */ 796 movq $VM_MAX_USER_ADDRESS,%rax 797 cmpq %rax,%rsi 798 jae cpystrflt 7994: 800 movq $ENAMETOOLONG,%rax 801 jmp cpystrflt_x 802 803cpystrflt: 804 movq $EFAULT,%rax 805 806cpystrflt_x: 807 /* set *lencopied and return %eax */ 808 movq PCPU(curthread),%rcx 809 movq TD_PCB(%rcx), %rcx 810 movq $0,PCB_ONFAULT(%rcx) 811 812 testq %r9,%r9 813 jz 1f 814 subq %rdx,%r8 815 movq %r8,(%r9) 8161: 817 ret 818END(std_copyinstr) 819 820/* 821 * copystr(from, to, maxlen, int *lencopied) - MP SAFE 822 * %rdi, %rsi, %rdx, %rcx 823 */ 824ENTRY(copystr) 825 movq %rdx,%r8 /* %r8 = maxlen */ 826 827 incq %rdx 8281: 829 decq %rdx 830 jz 4f 831 832 movb (%rdi),%al /* faster than lodsb+stosb */ 833 movb %al,(%rsi) 834 leaq 1(%rdi),%rdi 835 leaq 1(%rsi),%rsi 836 testb %al,%al 837 jnz 1b 838 839 /* Success -- 0 byte reached */ 840 decq %rdx 841 xorl %eax,%eax 842 jmp 6f 8434: 844 /* rdx is zero -- return ENAMETOOLONG */ 845 movq $ENAMETOOLONG,%rax 846 8476: 848 849 testq %rcx,%rcx 850 jz 7f 851 /* set *lencopied and return %rax */ 852 subq %rdx,%r8 853 movq %r8,(%rcx) 8547: 855 ret 856END(copystr) 857 858/* 859 * Handling of special x86_64 registers and descriptor tables etc 860 * %rdi 861 */ 862/* void lgdt(struct region_descriptor *rdp); */ 863ENTRY(lgdt) 864 /* reload the descriptor table */ 865 lgdt (%rdi) 866 867 /* flush the prefetch q */ 868 jmp 1f 869 nop 8701: 871 movl $KDSEL,%eax 872 movl %eax,%ds 873 movl %eax,%es 874 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */ 875 movl %eax,%gs /* Beware, use wrmsr to set 64 bit base */ 876 movl %eax,%ss 877 878 /* reload code selector by turning return into intersegmental return */ 879 popq %rax 880 pushq $KCSEL 881 pushq %rax 882 MEXITCOUNT 883 lretq 884END(lgdt) 885 886/*****************************************************************************/ 887/* setjmp, longjmp */ 888/*****************************************************************************/ 889 890ENTRY(setjmp) 891 movq %rbx,0(%rdi) /* save rbx */ 892 movq %rsp,8(%rdi) /* save rsp */ 893 movq %rbp,16(%rdi) /* save rbp */ 894 movq %r12,24(%rdi) /* save r12 */ 895 movq %r13,32(%rdi) /* save r13 */ 896 movq %r14,40(%rdi) /* save r14 */ 897 movq %r15,48(%rdi) /* save r15 */ 898 movq 0(%rsp),%rdx /* get rta */ 899 movq %rdx,56(%rdi) /* save rip */ 900 xorl %eax,%eax /* return(0); */ 901 ret 902END(setjmp) 903 904ENTRY(longjmp) 905 movq 0(%rdi),%rbx /* restore rbx */ 906 movq 8(%rdi),%rsp /* restore rsp */ 907 movq 16(%rdi),%rbp /* restore rbp */ 908 movq 24(%rdi),%r12 /* restore r12 */ 909 movq 32(%rdi),%r13 /* restore r13 */ 910 movq 40(%rdi),%r14 /* restore r14 */ 911 movq 48(%rdi),%r15 /* restore r15 */ 912 movq 56(%rdi),%rdx /* get rta */ 913 movq %rdx,0(%rsp) /* put in return frame */ 914 xorl %eax,%eax /* return(1); */ 915 incl %eax 916 ret 917END(longjmp) 918 919/* 920 * Support for reading MSRs in the safe manner. 921 */ 922ENTRY(rdmsr_safe) 923/* int rdmsr_safe(u_int msr, uint64_t *data) */ 924 movq PCPU(curthread),%r8 925 movq TD_PCB(%r8), %r8 926 movq $msr_onfault,PCB_ONFAULT(%r8) 927 movq %rsp,PCB_ONFAULT_SP(%r8) 928 movl %edi,%ecx 929 rdmsr /* Read MSR pointed by %ecx. Returns 930 hi byte in edx, lo in %eax */ 931 salq $32,%rdx /* sign-shift %rdx left */ 932 movl %eax,%eax /* zero-extend %eax -> %rax */ 933 orq %rdx,%rax 934 movq %rax,(%rsi) 935 xorq %rax,%rax 936 movq %rax,PCB_ONFAULT(%r8) 937 ret 938END(rdmsr_safe) 939 940/* 941 * Support for writing MSRs in the safe manner. 942 */ 943ENTRY(wrmsr_safe) 944/* int wrmsr_safe(u_int msr, uint64_t data) */ 945 movq PCPU(curthread),%r8 946 movq TD_PCB(%r8), %r8 947 movq $msr_onfault,PCB_ONFAULT(%r8) 948 movq %rsp,PCB_ONFAULT_SP(%r8) 949 movl %edi,%ecx 950 movl %esi,%eax 951 sarq $32,%rsi 952 movl %esi,%edx 953 wrmsr /* Write MSR pointed by %ecx. Accepts 954 hi byte in edx, lo in %eax. */ 955 xorq %rax,%rax 956 movq %rax,PCB_ONFAULT(%r8) 957 ret 958END(wrmsr_safe) 959 960/* 961 * MSR operations fault handler 962 */ 963 ALIGN_TEXT 964msr_onfault: 965 movq PCPU(curthread),%r8 966 movq TD_PCB(%r8), %r8 967 movq $0,PCB_ONFAULT(%r8) 968 movl $EFAULT,%eax 969 ret 970