1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * Copyright (c) 2003 Peter Wemm. 4 * Copyright (c) 2008 The DragonFly Project. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD: src/sys/amd64/amd64/support.S,v 1.127 2007/05/23 08:33:04 kib Exp $ 32 */ 33 34#include <machine/asmacros.h> 35#include <machine/pmap.h> 36 37#include "assym.s" 38 39 ALIGN_DATA 40 41 .text 42 43/* 44 * bzero(ptr:%rdi, bytes:%rsi) 45 * 46 * Using rep stosq is 70% faster than a %rax loop and almost as fast as 47 * a %xmm0 loop on a modern intel cpu. 48 * 49 * Do not use non-termportal instructions here as we do not know the caller's 50 * intent. 51 */ 52ENTRY(bzero) 53 movq %rsi,%rcx 54 xorl %eax,%eax 55 shrq $3,%rcx 56 rep 57 stosq 58 movq %rsi,%rcx 59 andq $7,%rcx 60 jnz 1f 61 ret 621: rep 63 stosb 64 ret 65END(bzero) 66 67 .weak _bzero 68 .equ _bzero, bzero 69 70/* 71 * void *memset(ptr:%rdi, char:%rsi, bytes:%rdx) 72 * 73 * Same as bzero except we load the char into all byte 74 * positions of %rax. Returns original (ptr). 75 */ 76ENTRY(memset) 77 movzbq %sil,%r8 78 movabs $0x0101010101010101,%rax 79 imulq %r8,%rax 80 81 movq %rdi,%r9 82 movq %rdx,%rcx 83 shrq $3,%rcx 84 rep 85 stosq 86 movq %rdx,%rcx 87 andq $7,%rcx 88 jnz 1f 89 movq %r9,%rax 90 ret 911: rep 92 stosb 93 movq %r9,%rax 94 ret 95END(memset) 96 97 .weak _memset 98 .equ _memset, memset 99 100/* 101 * pagezero(ptr:%rdi) 102 * 103 * Using rep stosq is nearly as fast as using %xmm0 on a modern intel cpu, 104 * and about 70% faster than a %rax loop. 105 * 106 * Do not use non-termportal instructions here as we do not know the caller's 107 * intent. 108 */ 109#if 0 110 111ENTRY(pagezero) 112 movq $PAGE_SIZE>>3,%rcx 113 xorl %eax,%eax 114 rep 115 stosq 116 ret 117END(pagezero) 118 119#endif 120 121ENTRY(pagezero) 122 addq $4096,%rdi 123 movq $-4096,%rax 124 ALIGN_TEXT 1251: 126 movq $0,(%rdi,%rax,1) 127 addq $8,%rax 128 jne 1b 129 ret 130END(pagezero) 131 132/* 133 * bcopy(src:%rdi, dst:%rsi, cnt:%rdx) 134 * 135 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 136 */ 137ENTRY(bcopy) 138 xchgq %rsi,%rdi 139 movq %rdx,%rcx 140 141 movq %rdi,%rax 142 subq %rsi,%rax 143 cmpq %rcx,%rax /* overlapping && src < dst? */ 144 jb 2f 145 146 shrq $3,%rcx /* copy by 64-bit words */ 147 rep 148 movsq 149 movq %rdx,%rcx 150 andq $7,%rcx /* any bytes left? */ 151 jnz 1f 152 ret 1531: rep 154 movsb 155 ret 156 157 ALIGN_TEXT 1582: 159 addq %rcx,%rdi /* copy backwards */ 160 addq %rcx,%rsi 161 std 162 decq %rdi 163 decq %rsi 164 andq $7,%rcx /* any fractional bytes? */ 165 jz 3f 166 rep 167 movsb 1683: movq %rdx,%rcx /* copy by 32-bit words */ 169 shrq $3,%rcx 170 subq $7,%rsi 171 subq $7,%rdi 172 rep 173 movsq 174 cld 175 ret 176END(bcopy) 177 178 /* 179 * Use in situations where a bcopy function pointer is needed. 180 */ 181 .weak _bcopy 182 .equ _bcopy, bcopy 183 184 /* 185 * memmove(dst:%rdi, src:%rsi, cnt:%rdx) 186 * (same as bcopy but without the xchgq, and must return (dst)). 187 * 188 * NOTE: gcc builtin backs-off to memmove() call 189 * 190 * NOTE: We leave %rdi in %rax for the return value. 191 */ 192ENTRY(memmove) 193 movq %rdx,%rcx 194 movq %rdi,%rax /* return value */ 195 movq %rdi,%r8 196 subq %rsi,%r8 197 cmpq %rcx,%r8 /* overlapping && src < dst? */ 198 jb 2f 199 200 shrq $3,%rcx /* copy by 64-bit words */ 201 rep 202 movsq 203 movq %rdx,%rcx 204 andq $7,%rcx /* any bytes left? */ 205 jnz 1f 206 ret 2071: rep 208 movsb 209 ret 210 211 ALIGN_TEXT 2122: 213 addq %rcx,%rdi /* copy backwards */ 214 addq %rcx,%rsi 215 std 216 decq %rdi 217 decq %rsi 218 andq $7,%rcx /* any fractional bytes? */ 219 jz 3f 220 rep 221 movsb 2223: movq %rdx,%rcx /* copy by 32-bit words */ 223 shrq $3,%rcx 224 subq $7,%rsi 225 subq $7,%rdi 226 rep 227 movsq 228 cld 229 ret 230END(memmove) 231 232 .weak _memmove 233 .equ _memmove, memmove 234 235ENTRY(reset_dbregs) 236 movq $0x200,%rax /* the manual says that bit 10 must be set to 1 */ 237 movq %rax,%dr7 /* disable all breapoints first */ 238 movq $0,%rax 239 movq %rax,%dr0 240 movq %rax,%dr1 241 movq %rax,%dr2 242 movq %rax,%dr3 243 movq %rax,%dr6 244 ret 245END(reset_dbregs) 246 247/* 248 * memcpy(dst:%rdi, src:%rsi, bytes:%rdx) 249 * 250 * NOTE: memcpy does not support overlapping copies 251 * NOTE: returns dst 252 */ 253ENTRY(memcpy) 254 movq %rdi,%r8 255 movq %rdx,%rcx 256 shrq $3,%rcx /* copy by 64-bit words */ 257 rep 258 movsq 259 movq %rdx,%rcx 260 andq $7,%rcx /* any bytes left? */ 261 jnz 1f 262 movq %r8,%rax 263 ret 2641: rep 265 movsb 266 movq %r8,%rax 267 ret 268END(memcpy) 269 270 .weak _memcpy 271 .equ _memcpy, memcpy 272 273/* fillw(pat, base, cnt) */ 274/* %rdi,%rsi, %rdx */ 275ENTRY(fillw) 276 movq %rdi,%rax 277 movq %rsi,%rdi 278 movq %rdx,%rcx 279 rep 280 stosw 281 ret 282END(fillw) 283 284/*****************************************************************************/ 285/* copyout and fubyte family */ 286/*****************************************************************************/ 287/* 288 * Access user memory from inside the kernel. These routines should be 289 * the only places that do this. 290 * 291 * These routines set curpcb->onfault for the time they execute. When a 292 * protection violation occurs inside the functions, the trap handler 293 * returns to *curpcb->onfault instead of the function. 294 */ 295 296/* 297 * uint64_t:%rax kreadmem64(addr:%rdi) 298 * 299 * Read kernel or user memory with fault protection. 300 */ 301ENTRY(kreadmem64) 302 SMAP_OPEN 303 movq PCPU(curthread),%rcx 304 movq TD_PCB(%rcx), %rcx 305 movq $kreadmem64fault,PCB_ONFAULT(%rcx) 306 movq %rsp,PCB_ONFAULT_SP(%rcx) 307 movq (%rdi),%rax 308 movq $0,PCB_ONFAULT(%rcx) 309 SMAP_CLOSE 310 ret 311 312kreadmem64fault: 313 SMAP_CLOSE 314 movq PCPU(curthread),%rcx 315 xorl %eax,%eax 316 movq TD_PCB(%rcx),%rcx 317 movq %rax,PCB_ONFAULT(%rcx) 318 decq %rax 319 ret 320END(kreadmem64) 321 322/* 323 * std_copyout(from_kernel, to_user, len) - MP SAFE 324 * %rdi, %rsi, %rdx 325 */ 326ENTRY(std_copyout) 327 SMAP_OPEN 328 movq PCPU(curthread),%rax 329 movq TD_PCB(%rax), %rax 330 movq $copyout_fault,PCB_ONFAULT(%rax) 331 movq %rsp,PCB_ONFAULT_SP(%rax) 332 testq %rdx,%rdx /* anything to do? */ 333 jz done_copyout 334 335 /* 336 * Check explicitly for non-user addresses. If 486 write protection 337 * is being used, this check is essential because we are in kernel 338 * mode so the h/w does not provide any protection against writing 339 * kernel addresses. 340 */ 341 342 /* 343 * First, prevent address wrapping. 344 */ 345 movq %rsi,%rax 346 addq %rdx,%rax 347 jc copyout_fault 348/* 349 * XXX STOP USING VM_MAX_USER_ADDRESS. 350 * It is an end address, not a max, so every time it is used correctly it 351 * looks like there is an off by one error, and of course it caused an off 352 * by one error in several places. 353 */ 354 movq $VM_MAX_USER_ADDRESS,%rcx 355 cmpq %rcx,%rax 356 ja copyout_fault 357 358 xchgq %rdi,%rsi 359 /* bcopy(%rsi, %rdi, %rdx) */ 360 movq %rdx,%rcx 361 362 shrq $3,%rcx 363 jz 1f 364 rep 365 movsq 3661: movq %rdx,%rcx 367 andq $7,%rcx 368 jz done_copyout 369 rep 370 movsb 371 372done_copyout: 373 SMAP_CLOSE 374 xorl %eax,%eax 375 movq PCPU(curthread),%rdx 376 movq TD_PCB(%rdx), %rdx 377 movq %rax,PCB_ONFAULT(%rdx) 378 ret 379 380 ALIGN_TEXT 381copyout_fault: 382 SMAP_CLOSE 383 movq PCPU(curthread),%rdx 384 movq TD_PCB(%rdx), %rdx 385 movq $0,PCB_ONFAULT(%rdx) 386 movq $EFAULT,%rax 387 ret 388END(std_copyout) 389 390/* 391 * std_copyin(from_user, to_kernel, len) - MP SAFE 392 * %rdi, %rsi, %rdx 393 */ 394ENTRY(std_copyin) 395 SMAP_OPEN 396 movq PCPU(curthread),%rax 397 movq TD_PCB(%rax), %rax 398 movq $copyin_fault,PCB_ONFAULT(%rax) 399 movq %rsp,PCB_ONFAULT_SP(%rax) 400 testq %rdx,%rdx /* anything to do? */ 401 jz done_copyin 402 403 /* 404 * make sure address is valid 405 */ 406 movq %rdi,%rax 407 addq %rdx,%rax 408 jc copyin_fault 409 movq $VM_MAX_USER_ADDRESS,%rcx 410 cmpq %rcx,%rax 411 ja copyin_fault 412 413 xchgq %rdi,%rsi 414 movq %rdx,%rcx 415 shrq $3,%rcx /* copy longword-wise */ 416 jz 1f 417 rep 418 movsq 4191: movq %rdx,%rcx 420 andq $7,%rcx /* copy remaining bytes */ 421 jz done_copyin 422 rep 423 movsb 424 425done_copyin: 426 SMAP_CLOSE 427 xorl %eax,%eax 428 movq PCPU(curthread),%rdx 429 movq TD_PCB(%rdx), %rdx 430 movq %rax,PCB_ONFAULT(%rdx) 431 ret 432 433 ALIGN_TEXT 434copyin_fault: 435 SMAP_CLOSE 436 movq PCPU(curthread),%rdx 437 movq TD_PCB(%rdx), %rdx 438 movq $0,PCB_ONFAULT(%rdx) 439 movq $EFAULT,%rax 440 ret 441END(std_copyin) 442 443/* 444 * casu32 - Compare and set user integer. Returns -1 or the current value. 445 * dst = %rdi, old = %rsi, new = %rdx 446 */ 447ENTRY(casu32) 448 SMAP_OPEN 449 movq PCPU(curthread),%rcx 450 movq TD_PCB(%rcx), %rcx 451 movq $fusufault,PCB_ONFAULT(%rcx) 452 movq %rsp,PCB_ONFAULT_SP(%rcx) 453 454 movq $VM_MAX_USER_ADDRESS-4,%rax 455 cmpq %rax,%rdi /* verify address is valid */ 456 ja fusufault 457 458 movl %esi,%eax /* old */ 459 lock 460 cmpxchgl %edx,(%rdi) /* new = %edx */ 461 462 /* 463 * The old value is in %eax. If the store succeeded it will be the 464 * value we expected (old) from before the store, otherwise it will 465 * be the current value. 466 */ 467 468 movq PCPU(curthread),%rcx 469 movq TD_PCB(%rcx), %rcx 470 movq $0,PCB_ONFAULT(%rcx) 471 SMAP_CLOSE 472 ret 473END(casu32) 474 475/* 476 * swapu32 - Swap int in user space. ptr = %rdi, val = %rsi 477 */ 478ENTRY(std_swapu32) 479 SMAP_OPEN 480 movq PCPU(curthread),%rcx 481 movq TD_PCB(%rcx), %rcx 482 movq $fusufault,PCB_ONFAULT(%rcx) 483 movq %rsp,PCB_ONFAULT_SP(%rcx) 484 485 movq $VM_MAX_USER_ADDRESS-4,%rax 486 cmpq %rax,%rdi /* verify address is valid */ 487 ja fusufault 488 489 movq %rsi,%rax /* old */ 490 xchgl %eax,(%rdi) 491 492 /* 493 * The old value is in %rax. If the store succeeded it will be the 494 * value we expected (old) from before the store, otherwise it will 495 * be the current value. 496 */ 497 498 movq PCPU(curthread),%rcx 499 movq TD_PCB(%rcx), %rcx 500 movq $0,PCB_ONFAULT(%rcx) 501 SMAP_CLOSE 502 ret 503END(std_swapu32) 504 505ENTRY(std_fuwordadd32) 506 SMAP_OPEN 507 movq PCPU(curthread),%rcx 508 movq TD_PCB(%rcx), %rcx 509 movq $fusufault,PCB_ONFAULT(%rcx) 510 movq %rsp,PCB_ONFAULT_SP(%rcx) 511 512 movq $VM_MAX_USER_ADDRESS-4,%rax 513 cmpq %rax,%rdi /* verify address is valid */ 514 ja fusufault 515 516 movq %rsi,%rax /* qty to add */ 517 lock xaddl %eax,(%rdi) 518 519 /* 520 * The old value is in %rax. If the store succeeded it will be the 521 * value we expected (old) from before the store, otherwise it will 522 * be the current value. 523 */ 524 movq PCPU(curthread),%rcx 525 movq TD_PCB(%rcx), %rcx 526 movq $0,PCB_ONFAULT(%rcx) 527 SMAP_CLOSE 528 ret 529END(std_fuwordadd32) 530 531/* 532 * casu64 - Compare and set user word. Returns -1 or the current value. 533 * dst = %rdi, old = %rsi, new = %rdx 534 */ 535ENTRY(casu64) 536 SMAP_OPEN 537 movq PCPU(curthread),%rcx 538 movq TD_PCB(%rcx), %rcx 539 movq $fusufault,PCB_ONFAULT(%rcx) 540 movq %rsp,PCB_ONFAULT_SP(%rcx) 541 542 movq $VM_MAX_USER_ADDRESS-8,%rax 543 cmpq %rax,%rdi /* verify address is valid */ 544 ja fusufault 545 546 movq %rsi,%rax /* old */ 547 lock 548 cmpxchgq %rdx,(%rdi) /* new = %rdx */ 549 550 /* 551 * The old value is in %rax. If the store succeeded it will be the 552 * value we expected (old) from before the store, otherwise it will 553 * be the current value. 554 */ 555 556 movq PCPU(curthread),%rcx 557 movq TD_PCB(%rcx), %rcx 558 movq $0,PCB_ONFAULT(%rcx) 559 SMAP_CLOSE 560 ret 561END(casu64) 562 563/* 564 * swapu64 - Swap long in user space. ptr = %rdi, val = %rsi 565 */ 566ENTRY(std_swapu64) 567 SMAP_OPEN 568 movq PCPU(curthread),%rcx 569 movq TD_PCB(%rcx), %rcx 570 movq $fusufault,PCB_ONFAULT(%rcx) 571 movq %rsp,PCB_ONFAULT_SP(%rcx) 572 573 movq $VM_MAX_USER_ADDRESS-8,%rax 574 cmpq %rax,%rdi /* verify address is valid */ 575 ja fusufault 576 577 movq %rsi,%rax /* old */ 578 xchgq %rax,(%rdi) 579 580 /* 581 * The old value is in %rax. If the store succeeded it will be the 582 * value we expected (old) from before the store, otherwise it will 583 * be the current value. 584 */ 585 586 movq PCPU(curthread),%rcx 587 movq TD_PCB(%rcx), %rcx 588 movq $0,PCB_ONFAULT(%rcx) 589 SMAP_CLOSE 590 ret 591END(std_swapu64) 592 593ENTRY(std_fuwordadd64) 594 SMAP_OPEN 595 movq PCPU(curthread),%rcx 596 movq TD_PCB(%rcx), %rcx 597 movq $fusufault,PCB_ONFAULT(%rcx) 598 movq %rsp,PCB_ONFAULT_SP(%rcx) 599 600 movq $VM_MAX_USER_ADDRESS-8,%rax 601 cmpq %rax,%rdi /* verify address is valid */ 602 ja fusufault 603 604 movq %rsi,%rax /* value to add */ 605 lock xaddq %rax,(%rdi) 606 607 /* 608 * The old value is in %rax. If the store succeeded it will be the 609 * value we expected (old) from before the store, otherwise it will 610 * be the current value. 611 */ 612 613 movq PCPU(curthread),%rcx 614 movq TD_PCB(%rcx), %rcx 615 movq $0,PCB_ONFAULT(%rcx) 616 SMAP_CLOSE 617 ret 618END(std_fuwordadd64) 619 620/* 621 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit 622 * byte from user memory. All these functions are MPSAFE. 623 * addr = %rdi 624 */ 625 626ENTRY(std_fuword64) 627 SMAP_OPEN 628 movq PCPU(curthread),%rcx 629 movq TD_PCB(%rcx), %rcx 630 movq $fusufault,PCB_ONFAULT(%rcx) 631 movq %rsp,PCB_ONFAULT_SP(%rcx) 632 633 movq $VM_MAX_USER_ADDRESS-8,%rax 634 cmpq %rax,%rdi /* verify address is valid */ 635 ja fusufault 636 637 movq (%rdi),%rax 638 movq $0,PCB_ONFAULT(%rcx) 639 SMAP_CLOSE 640 ret 641END(std_fuword64) 642 643ENTRY(std_fuword32) 644 SMAP_OPEN 645 movq PCPU(curthread),%rcx 646 movq TD_PCB(%rcx), %rcx 647 movq $fusufault,PCB_ONFAULT(%rcx) 648 movq %rsp,PCB_ONFAULT_SP(%rcx) 649 650 movq $VM_MAX_USER_ADDRESS-4,%rax 651 cmpq %rax,%rdi /* verify address is valid */ 652 ja fusufault 653 654 movl (%rdi),%eax 655 movq $0,PCB_ONFAULT(%rcx) 656 SMAP_CLOSE 657 ret 658END(std_fuword32) 659 660ENTRY(std_fubyte) 661 SMAP_OPEN 662 movq PCPU(curthread),%rcx 663 movq TD_PCB(%rcx), %rcx 664 movq $fusufault,PCB_ONFAULT(%rcx) 665 movq %rsp,PCB_ONFAULT_SP(%rcx) 666 667 movq $VM_MAX_USER_ADDRESS-1,%rax 668 cmpq %rax,%rdi 669 ja fusufault 670 671 movzbl (%rdi),%eax 672 movq $0,PCB_ONFAULT(%rcx) 673 SMAP_CLOSE 674 ret 675 676 ALIGN_TEXT 677fusufault: 678 movq PCPU(curthread),%rcx 679 xorl %eax,%eax 680 movq TD_PCB(%rcx), %rcx 681 movq %rax,PCB_ONFAULT(%rcx) 682 decq %rax 683 SMAP_CLOSE 684 ret 685END(std_fubyte) 686 687/* 688 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to 689 * user memory. All these functions are MPSAFE. 690 * 691 * addr = %rdi, value = %rsi 692 * 693 * Write a long 694 */ 695ENTRY(std_suword64) 696 SMAP_OPEN 697 movq PCPU(curthread),%rcx 698 movq TD_PCB(%rcx), %rcx 699 movq $fusufault,PCB_ONFAULT(%rcx) 700 movq %rsp,PCB_ONFAULT_SP(%rcx) 701 702 movq $VM_MAX_USER_ADDRESS-8,%rax 703 cmpq %rax,%rdi /* verify address validity */ 704 ja fusufault 705 706 movq %rsi,(%rdi) 707 xorl %eax,%eax 708 movq PCPU(curthread),%rcx 709 movq TD_PCB(%rcx), %rcx 710 movq %rax,PCB_ONFAULT(%rcx) 711 SMAP_CLOSE 712 ret 713END(std_suword64) 714 715/* 716 * Write an int 717 */ 718ENTRY(std_suword32) 719 SMAP_OPEN 720 movq PCPU(curthread),%rcx 721 movq TD_PCB(%rcx), %rcx 722 movq $fusufault,PCB_ONFAULT(%rcx) 723 movq %rsp,PCB_ONFAULT_SP(%rcx) 724 725 movq $VM_MAX_USER_ADDRESS-4,%rax 726 cmpq %rax,%rdi /* verify address validity */ 727 ja fusufault 728 729 movl %esi,(%rdi) 730 xorl %eax,%eax 731 movq PCPU(curthread),%rcx 732 movq TD_PCB(%rcx), %rcx 733 movq %rax,PCB_ONFAULT(%rcx) 734 SMAP_CLOSE 735 ret 736END(std_suword32) 737 738ENTRY(std_subyte) 739 SMAP_OPEN 740 movq PCPU(curthread),%rcx 741 movq TD_PCB(%rcx), %rcx 742 movq $fusufault,PCB_ONFAULT(%rcx) 743 movq %rsp,PCB_ONFAULT_SP(%rcx) 744 745 movq $VM_MAX_USER_ADDRESS-1,%rax 746 cmpq %rax,%rdi /* verify address validity */ 747 ja fusufault 748 749 movl %esi,%eax 750 movb %al,(%rdi) 751 xorl %eax,%eax 752 movq PCPU(curthread),%rcx /* restore trashed register */ 753 movq TD_PCB(%rcx), %rcx 754 movq %rax,PCB_ONFAULT(%rcx) 755 SMAP_CLOSE 756 ret 757END(std_subyte) 758 759/* 760 * std_copyinstr(from, to, maxlen, int *lencopied) - MP SAFE 761 * %rdi, %rsi, %rdx, %rcx 762 * 763 * copy a string from from to to, stop when a 0 character is reached. 764 * return ENAMETOOLONG if string is longer than maxlen, and 765 * EFAULT on protection violations. If lencopied is non-zero, 766 * return the actual length in *lencopied. 767 */ 768ENTRY(std_copyinstr) 769 SMAP_OPEN 770 movq %rdx,%r8 /* %r8 = maxlen */ 771 movq %rcx,%r9 /* %r9 = *len */ 772 movq PCPU(curthread),%rcx 773 movq TD_PCB(%rcx), %rcx 774 movq $cpystrflt,PCB_ONFAULT(%rcx) 775 movq %rsp,PCB_ONFAULT_SP(%rcx) 776 777 movq $VM_MAX_USER_ADDRESS,%rax 778 779 /* make sure 'from' is within bounds */ 780 subq %rdi,%rax 781 jbe cpystrflt 782 783 /* restrict maxlen to <= VM_MAX_USER_ADDRESS-from */ 784 cmpq %rdx,%rax 785 jae 1f 786 movq %rax,%rdx 787 movq %rax,%r8 7881: 789 incq %rdx 790 7912: 792 decq %rdx 793 jz 3f 794 795 movb (%rdi),%al /* faster than lodsb+stosb */ 796 movb %al,(%rsi) 797 leaq 1(%rdi),%rdi 798 leaq 1(%rsi),%rsi 799 testb %al,%al 800 jnz 2b 801 802 /* Success -- 0 byte reached */ 803 decq %rdx 804 xorl %eax,%eax 805 jmp cpystrflt_x 8063: 807 /* rdx is zero - return ENAMETOOLONG or EFAULT */ 808 movq $VM_MAX_USER_ADDRESS,%rax 809 cmpq %rax,%rsi 810 jae cpystrflt 8114: 812 movq $ENAMETOOLONG,%rax 813 jmp cpystrflt_x 814 815cpystrflt: 816 movq $EFAULT,%rax 817 818cpystrflt_x: 819 SMAP_CLOSE 820 /* set *lencopied and return %eax */ 821 movq PCPU(curthread),%rcx 822 movq TD_PCB(%rcx), %rcx 823 movq $0,PCB_ONFAULT(%rcx) 824 825 testq %r9,%r9 826 jz 1f 827 subq %rdx,%r8 828 movq %r8,(%r9) 8291: 830 ret 831END(std_copyinstr) 832 833/* 834 * copystr(from, to, maxlen, int *lencopied) - MP SAFE 835 * %rdi, %rsi, %rdx, %rcx 836 */ 837ENTRY(copystr) 838 movq %rdx,%r8 /* %r8 = maxlen */ 839 incq %rdx 8401: 841 decq %rdx 842 jz 4f 843 844 movb (%rdi),%al /* faster than lodsb+stosb */ 845 movb %al,(%rsi) 846 leaq 1(%rdi),%rdi 847 leaq 1(%rsi),%rsi 848 testb %al,%al 849 jnz 1b 850 851 /* Success -- 0 byte reached */ 852 decq %rdx 853 xorl %eax,%eax 854 jmp 6f 8554: 856 /* rdx is zero -- return ENAMETOOLONG */ 857 movq $ENAMETOOLONG,%rax 858 8596: 860 testq %rcx,%rcx 861 jz 7f 862 /* set *lencopied and return %rax */ 863 subq %rdx,%r8 864 movq %r8,(%rcx) 8657: 866 ret 867END(copystr) 868 869/* 870 * Handling of special x86_64 registers and descriptor tables etc 871 * %rdi 872 */ 873/* void lgdt(struct region_descriptor *rdp); */ 874ENTRY(lgdt) 875 /* reload the descriptor table */ 876 lgdt (%rdi) 877 878 /* flush the prefetch q */ 879 jmp 1f 880 nop 8811: 882 movl $KDSEL,%eax 883 movl %eax,%ds 884 movl %eax,%es 885 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */ 886 movl %eax,%gs /* Beware, use wrmsr to set 64 bit base */ 887 movl %eax,%ss 888 889 /* reload code selector by turning return into intersegmental return */ 890 popq %rax 891 pushq $KCSEL 892 pushq %rax 893 MEXITCOUNT 894 lretq 895END(lgdt) 896 897/*****************************************************************************/ 898/* setjmp, longjmp */ 899/*****************************************************************************/ 900 901ENTRY(setjmp) 902 movq %rbx,0(%rdi) /* save rbx */ 903 movq %rsp,8(%rdi) /* save rsp */ 904 movq %rbp,16(%rdi) /* save rbp */ 905 movq %r12,24(%rdi) /* save r12 */ 906 movq %r13,32(%rdi) /* save r13 */ 907 movq %r14,40(%rdi) /* save r14 */ 908 movq %r15,48(%rdi) /* save r15 */ 909 movq 0(%rsp),%rdx /* get rta */ 910 movq %rdx,56(%rdi) /* save rip */ 911 xorl %eax,%eax /* return(0); */ 912 ret 913END(setjmp) 914 915ENTRY(longjmp) 916 movq 0(%rdi),%rbx /* restore rbx */ 917 movq 8(%rdi),%rsp /* restore rsp */ 918 movq 16(%rdi),%rbp /* restore rbp */ 919 movq 24(%rdi),%r12 /* restore r12 */ 920 movq 32(%rdi),%r13 /* restore r13 */ 921 movq 40(%rdi),%r14 /* restore r14 */ 922 movq 48(%rdi),%r15 /* restore r15 */ 923 movq 56(%rdi),%rdx /* get rta */ 924 movq %rdx,0(%rsp) /* put in return frame */ 925 xorl %eax,%eax /* return(1); */ 926 incl %eax 927 ret 928END(longjmp) 929 930/* 931 * Support for reading MSRs in the safe manner. 932 */ 933ENTRY(rdmsr_safe) 934/* int rdmsr_safe(u_int msr, uint64_t *data) */ 935 movq PCPU(curthread),%r8 936 movq TD_PCB(%r8), %r8 937 movq $msr_onfault,PCB_ONFAULT(%r8) 938 movq %rsp,PCB_ONFAULT_SP(%r8) 939 movl %edi,%ecx 940 rdmsr /* Read MSR pointed by %ecx. Returns 941 hi byte in edx, lo in %eax */ 942 salq $32,%rdx /* sign-shift %rdx left */ 943 movl %eax,%eax /* zero-extend %eax -> %rax */ 944 orq %rdx,%rax 945 movq %rax,(%rsi) 946 xorq %rax,%rax 947 movq %rax,PCB_ONFAULT(%r8) 948 ret 949END(rdmsr_safe) 950 951/* 952 * Support for writing MSRs in the safe manner. 953 */ 954ENTRY(wrmsr_safe) 955/* int wrmsr_safe(u_int msr, uint64_t data) */ 956 movq PCPU(curthread),%r8 957 movq TD_PCB(%r8), %r8 958 movq $msr_onfault,PCB_ONFAULT(%r8) 959 movq %rsp,PCB_ONFAULT_SP(%r8) 960 movl %edi,%ecx 961 movl %esi,%eax 962 sarq $32,%rsi 963 movl %esi,%edx 964 wrmsr /* Write MSR pointed by %ecx. Accepts 965 hi byte in edx, lo in %eax. */ 966 xorq %rax,%rax 967 movq %rax,PCB_ONFAULT(%r8) 968 ret 969END(wrmsr_safe) 970 971/* 972 * MSR operations fault handler 973 */ 974 ALIGN_TEXT 975msr_onfault: 976 movq PCPU(curthread),%r8 977 movq TD_PCB(%r8), %r8 978 movq $0,PCB_ONFAULT(%r8) 979 movl $EFAULT,%eax 980 ret 981