1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * Copyright (c) 2003 Peter Wemm. 4 * Copyright (c) 2008 The DragonFly Project. 5 * Copyright (c) 2008-2020 The DragonFly Project. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * $FreeBSD: src/sys/amd64/amd64/support.S,v 1.127 2007/05/23 08:33:04 kib Exp $ 33 */ 34 35#include <machine/asmacros.h> 36#include <machine/asm_mjgmacros.h> 37#include <machine/pmap.h> 38 39#include "assym.s" 40 41 ALIGN_DATA 42 43 .text 44 45/* 46 * bzero(ptr:%rdi, bytes:%rsi) 47 * 48 * Using rep stosq is 70% faster than a %rax loop and almost as fast as 49 * a %xmm0 loop on a modern intel cpu. 50 * 51 * Do not use non-termportal instructions here as we do not know the caller's 52 * intent. 53 */ 54ENTRY(bzero) 55 subq %r10,%r10 56 movq %rsi,%rdx 57 MEMSET erms=0 end=ret 58END(bzero) 59 60 .weak _bzero 61 .equ _bzero, bzero 62 63/* 64 * void *memset(ptr:%rdi, char:%rsi, bytes:%rdx) 65 * 66 * Same as bzero except we load the char into all byte 67 * positions of %rax. Returns original (ptr). 68 */ 69ENTRY(memset) 70 movzbq %sil,%r8 71 movabs $0x0101010101010101,%r10 72 imulq %r8,%r10 73 MEMSET erms=0 end=ret 74END(memset) 75 76 .weak _memset 77 .equ _memset, memset 78 79/* 80 * pagezero(ptr:%rdi) 81 * 82 * Modern intel and AMD cpus do a good job with rep stosq on page-sized 83 * blocks. The cross-point on intel is at the 256 byte mark and on AMD 84 * it is around the 1024 byte mark. With large counts, rep stosq will 85 * internally use non-termporal instructions and a cache sync at the end. 86 */ 87#if 1 88 89ENTRY(pagezero) 90 movq $PAGE_SIZE>>3,%rcx 91 xorl %eax,%eax 92 rep 93 stosq 94 ret 95END(pagezero) 96 97#else 98 99ENTRY(pagezero) 100 addq $4096,%rdi 101 movq $-4096,%rax 102 ALIGN_TEXT 1031: 104 movq $0,(%rdi,%rax,1) 105 movq $0,8(%rdi,%rax,1) 106 addq $16,%rax 107 jne 1b 108 ret 109END(pagezero) 110 111#endif 112 113/* 114 * bcopy(src:%rdi, dst:%rsi, cnt:%rdx) 115 * 116 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 117 */ 118ENTRY(bcopy) 119 xchgq %rsi,%rdi 120 MEMMOVE erms=0 overlap=1 end=ret 121END(bcopy) 122 123 /* 124 * Use in situations where a bcopy function pointer is needed. 125 */ 126 .weak _bcopy 127 .equ _bcopy, bcopy 128 129 /* 130 * memmove(dst:%rdi, src:%rsi, cnt:%rdx) 131 * (same as bcopy but without the xchgq, and must return (dst)). 132 * 133 * NOTE: gcc builtin backs-off to memmove() call 134 * NOTE: returns dst 135 */ 136ENTRY(memmove) 137 movq %rdi,%rax 138 MEMMOVE erms=0 overlap=1 end=ret 139END(memmove) 140 141 .weak _memmove 142 .equ _memmove, memmove 143 144ENTRY(reset_dbregs) 145 movq $0x200,%rax /* the manual says that bit 10 must be set to 1 */ 146 movq %rax,%dr7 /* disable all breapoints first */ 147 movq $0,%rax 148 movq %rax,%dr0 149 movq %rax,%dr1 150 movq %rax,%dr2 151 movq %rax,%dr3 152 movq %rax,%dr6 153 ret 154END(reset_dbregs) 155 156/* 157 * memcpy(dst:%rdi, src:%rsi, bytes:%rdx) 158 * 159 * NOTE: memcpy does not support overlapping copies 160 * NOTE: returns dst 161 */ 162ENTRY(memcpy) 163 movq %rdi,%rax 164 MEMMOVE erms=0 overlap=0 end=ret 165END(memcpy) 166 167 .weak _memcpy 168 .equ _memcpy, memcpy 169 170/* fillw(pat, base, cnt) */ 171/* %rdi,%rsi, %rdx */ 172ENTRY(fillw) 173 movq %rdi,%rax 174 movq %rsi,%rdi 175 movq %rdx,%rcx 176 rep 177 stosw 178 ret 179END(fillw) 180 181/*****************************************************************************/ 182/* copyout and fubyte family */ 183/*****************************************************************************/ 184/* 185 * Access user memory from inside the kernel. These routines should be 186 * the only places that do this. 187 * 188 * These routines set curpcb->onfault for the time they execute. When a 189 * protection violation occurs inside the functions, the trap handler 190 * returns to *curpcb->onfault instead of the function. 191 */ 192 193/* 194 * uint64_t:%rax kreadmem64(addr:%rdi) 195 * 196 * Read kernel or user memory with fault protection. 197 */ 198ENTRY(kreadmem64) 199 SMAP_OPEN 200 movq PCPU(curthread),%rcx 201 movq TD_PCB(%rcx), %rcx 202 movq $kreadmem64fault,PCB_ONFAULT(%rcx) 203 movq %rsp,PCB_ONFAULT_SP(%rcx) 204 movq (%rdi),%rax 205 movq $0,PCB_ONFAULT(%rcx) 206 SMAP_CLOSE 207 ret 208 209kreadmem64fault: 210 SMAP_CLOSE 211 movq PCPU(curthread),%rcx 212 xorl %eax,%eax 213 movq TD_PCB(%rcx),%rcx 214 movq %rax,PCB_ONFAULT(%rcx) 215 decq %rax 216 ret 217END(kreadmem64) 218 219.macro COPYOUT_END 220 jmp done_copyout 221 nop 222.endm 223 224/* 225 * std_copyout(from_kernel, to_user, len) - MP SAFE 226 * %rdi, %rsi, %rdx 227 */ 228ENTRY(std_copyout) 229 SMAP_OPEN 230 movq PCPU(curthread),%rax 231 movq TD_PCB(%rax), %rax 232 movq $copyout_fault,PCB_ONFAULT(%rax) 233 movq %rsp,PCB_ONFAULT_SP(%rax) 234 testq %rdx,%rdx /* anything to do? */ 235 jz done_copyout 236 237 /* 238 * Check explicitly for non-user addresses. If 486 write protection 239 * is being used, this check is essential because we are in kernel 240 * mode so the h/w does not provide any protection against writing 241 * kernel addresses. 242 */ 243 244 /* 245 * First, prevent address wrapping. 246 */ 247 movq %rsi,%rax 248 addq %rdx,%rax 249 jc copyout_fault 250/* 251 * XXX STOP USING VM_MAX_USER_ADDRESS. 252 * It is an end address, not a max, so every time it is used correctly it 253 * looks like there is an off by one error, and of course it caused an off 254 * by one error in several places. 255 */ 256 movq $VM_MAX_USER_ADDRESS,%rcx 257 cmpq %rcx,%rax 258 ja copyout_fault 259 260 xchgq %rdi,%rsi 261 MEMMOVE erms=0 overlap=0 end=COPYOUT_END 262 263done_copyout: 264 SMAP_CLOSE 265 xorl %eax,%eax 266 movq PCPU(curthread),%rdx 267 movq TD_PCB(%rdx), %rdx 268 movq %rax,PCB_ONFAULT(%rdx) 269 ret 270 271 ALIGN_TEXT 272copyout_fault: 273 SMAP_CLOSE 274 movq PCPU(curthread),%rdx 275 movq TD_PCB(%rdx), %rdx 276 movq $0,PCB_ONFAULT(%rdx) 277 movq $EFAULT,%rax 278 ret 279END(std_copyout) 280 281.macro COPYIN_END 282 jmp done_copyin 283 nop 284.endm 285 286/* 287 * std_copyin(from_user, to_kernel, len) - MP SAFE 288 * %rdi, %rsi, %rdx 289 */ 290ENTRY(std_copyin) 291 SMAP_OPEN 292 movq PCPU(curthread),%rax 293 movq TD_PCB(%rax), %rax 294 movq $copyin_fault,PCB_ONFAULT(%rax) 295 movq %rsp,PCB_ONFAULT_SP(%rax) 296 testq %rdx,%rdx /* anything to do? */ 297 jz done_copyin 298 299 /* 300 * make sure address is valid 301 */ 302 movq %rdi,%rax 303 addq %rdx,%rax 304 jc copyin_fault 305 movq $VM_MAX_USER_ADDRESS,%rcx 306 cmpq %rcx,%rax 307 ja copyin_fault 308 309 xchgq %rdi,%rsi 310 MEMMOVE erms=0 overlap=0 end=COPYIN_END 311 312done_copyin: 313 SMAP_CLOSE 314 xorl %eax,%eax 315 movq PCPU(curthread),%rdx 316 movq TD_PCB(%rdx), %rdx 317 movq %rax,PCB_ONFAULT(%rdx) 318 ret 319 320 ALIGN_TEXT 321copyin_fault: 322 SMAP_CLOSE 323 movq PCPU(curthread),%rdx 324 movq TD_PCB(%rdx), %rdx 325 movq $0,PCB_ONFAULT(%rdx) 326 movq $EFAULT,%rax 327 ret 328END(std_copyin) 329 330/* 331 * casu32 - Compare and set user integer. Returns -1 or the current value. 332 * dst = %rdi, old = %rsi, new = %rdx 333 */ 334ENTRY(casu32) 335 SMAP_OPEN 336 movq PCPU(curthread),%rcx 337 movq TD_PCB(%rcx), %rcx 338 movq $fusufault,PCB_ONFAULT(%rcx) 339 movq %rsp,PCB_ONFAULT_SP(%rcx) 340 341 movq $VM_MAX_USER_ADDRESS-4,%rax 342 cmpq %rax,%rdi /* verify address is valid */ 343 ja fusufault 344 345 movl %esi,%eax /* old */ 346 lock 347 cmpxchgl %edx,(%rdi) /* new = %edx */ 348 349 /* 350 * The old value is in %eax. If the store succeeded it will be the 351 * value we expected (old) from before the store, otherwise it will 352 * be the current value. 353 */ 354 355 movq PCPU(curthread),%rcx 356 movq TD_PCB(%rcx), %rcx 357 movq $0,PCB_ONFAULT(%rcx) 358 SMAP_CLOSE 359 ret 360END(casu32) 361 362/* 363 * swapu32 - Swap int in user space. ptr = %rdi, val = %rsi 364 */ 365ENTRY(std_swapu32) 366 SMAP_OPEN 367 movq PCPU(curthread),%rcx 368 movq TD_PCB(%rcx), %rcx 369 movq $fusufault,PCB_ONFAULT(%rcx) 370 movq %rsp,PCB_ONFAULT_SP(%rcx) 371 372 movq $VM_MAX_USER_ADDRESS-4,%rax 373 cmpq %rax,%rdi /* verify address is valid */ 374 ja fusufault 375 376 movq %rsi,%rax /* old */ 377 xchgl %eax,(%rdi) 378 379 /* 380 * The old value is in %rax. If the store succeeded it will be the 381 * value we expected (old) from before the store, otherwise it will 382 * be the current value. 383 */ 384 385 movq PCPU(curthread),%rcx 386 movq TD_PCB(%rcx), %rcx 387 movq $0,PCB_ONFAULT(%rcx) 388 SMAP_CLOSE 389 ret 390END(std_swapu32) 391 392ENTRY(std_fuwordadd32) 393 SMAP_OPEN 394 movq PCPU(curthread),%rcx 395 movq TD_PCB(%rcx), %rcx 396 movq $fusufault,PCB_ONFAULT(%rcx) 397 movq %rsp,PCB_ONFAULT_SP(%rcx) 398 399 movq $VM_MAX_USER_ADDRESS-4,%rax 400 cmpq %rax,%rdi /* verify address is valid */ 401 ja fusufault 402 403 movq %rsi,%rax /* qty to add */ 404 lock xaddl %eax,(%rdi) 405 406 /* 407 * The old value is in %rax. If the store succeeded it will be the 408 * value we expected (old) from before the store, otherwise it will 409 * be the current value. 410 */ 411 movq PCPU(curthread),%rcx 412 movq TD_PCB(%rcx), %rcx 413 movq $0,PCB_ONFAULT(%rcx) 414 SMAP_CLOSE 415 ret 416END(std_fuwordadd32) 417 418/* 419 * casu64 - Compare and set user word. Returns -1 or the current value. 420 * dst = %rdi, old = %rsi, new = %rdx 421 */ 422ENTRY(casu64) 423 SMAP_OPEN 424 movq PCPU(curthread),%rcx 425 movq TD_PCB(%rcx), %rcx 426 movq $fusufault,PCB_ONFAULT(%rcx) 427 movq %rsp,PCB_ONFAULT_SP(%rcx) 428 429 movq $VM_MAX_USER_ADDRESS-8,%rax 430 cmpq %rax,%rdi /* verify address is valid */ 431 ja fusufault 432 433 movq %rsi,%rax /* old */ 434 lock 435 cmpxchgq %rdx,(%rdi) /* new = %rdx */ 436 437 /* 438 * The old value is in %rax. If the store succeeded it will be the 439 * value we expected (old) from before the store, otherwise it will 440 * be the current value. 441 */ 442 443 movq PCPU(curthread),%rcx 444 movq TD_PCB(%rcx), %rcx 445 movq $0,PCB_ONFAULT(%rcx) 446 SMAP_CLOSE 447 ret 448END(casu64) 449 450/* 451 * swapu64 - Swap long in user space. ptr = %rdi, val = %rsi 452 */ 453ENTRY(std_swapu64) 454 SMAP_OPEN 455 movq PCPU(curthread),%rcx 456 movq TD_PCB(%rcx), %rcx 457 movq $fusufault,PCB_ONFAULT(%rcx) 458 movq %rsp,PCB_ONFAULT_SP(%rcx) 459 460 movq $VM_MAX_USER_ADDRESS-8,%rax 461 cmpq %rax,%rdi /* verify address is valid */ 462 ja fusufault 463 464 movq %rsi,%rax /* old */ 465 xchgq %rax,(%rdi) 466 467 /* 468 * The old value is in %rax. If the store succeeded it will be the 469 * value we expected (old) from before the store, otherwise it will 470 * be the current value. 471 */ 472 473 movq PCPU(curthread),%rcx 474 movq TD_PCB(%rcx), %rcx 475 movq $0,PCB_ONFAULT(%rcx) 476 SMAP_CLOSE 477 ret 478END(std_swapu64) 479 480ENTRY(std_fuwordadd64) 481 SMAP_OPEN 482 movq PCPU(curthread),%rcx 483 movq TD_PCB(%rcx), %rcx 484 movq $fusufault,PCB_ONFAULT(%rcx) 485 movq %rsp,PCB_ONFAULT_SP(%rcx) 486 487 movq $VM_MAX_USER_ADDRESS-8,%rax 488 cmpq %rax,%rdi /* verify address is valid */ 489 ja fusufault 490 491 movq %rsi,%rax /* value to add */ 492 lock xaddq %rax,(%rdi) 493 494 /* 495 * The old value is in %rax. If the store succeeded it will be the 496 * value we expected (old) from before the store, otherwise it will 497 * be the current value. 498 */ 499 500 movq PCPU(curthread),%rcx 501 movq TD_PCB(%rcx), %rcx 502 movq $0,PCB_ONFAULT(%rcx) 503 SMAP_CLOSE 504 ret 505END(std_fuwordadd64) 506 507/* 508 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit 509 * byte from user memory. All these functions are MPSAFE. 510 * addr = %rdi 511 */ 512 513ENTRY(std_fuword64) 514 SMAP_OPEN 515 movq PCPU(curthread),%rcx 516 movq TD_PCB(%rcx), %rcx 517 movq $fusufault,PCB_ONFAULT(%rcx) 518 movq %rsp,PCB_ONFAULT_SP(%rcx) 519 520 movq $VM_MAX_USER_ADDRESS-8,%rax 521 cmpq %rax,%rdi /* verify address is valid */ 522 ja fusufault 523 524 movq (%rdi),%rax 525 movq $0,PCB_ONFAULT(%rcx) 526 SMAP_CLOSE 527 ret 528END(std_fuword64) 529 530ENTRY(std_fuword32) 531 SMAP_OPEN 532 movq PCPU(curthread),%rcx 533 movq TD_PCB(%rcx), %rcx 534 movq $fusufault,PCB_ONFAULT(%rcx) 535 movq %rsp,PCB_ONFAULT_SP(%rcx) 536 537 movq $VM_MAX_USER_ADDRESS-4,%rax 538 cmpq %rax,%rdi /* verify address is valid */ 539 ja fusufault 540 541 movl (%rdi),%eax 542 movq $0,PCB_ONFAULT(%rcx) 543 SMAP_CLOSE 544 ret 545END(std_fuword32) 546 547ENTRY(std_fubyte) 548 SMAP_OPEN 549 movq PCPU(curthread),%rcx 550 movq TD_PCB(%rcx), %rcx 551 movq $fusufault,PCB_ONFAULT(%rcx) 552 movq %rsp,PCB_ONFAULT_SP(%rcx) 553 554 movq $VM_MAX_USER_ADDRESS-1,%rax 555 cmpq %rax,%rdi 556 ja fusufault 557 558 movzbl (%rdi),%eax 559 movq $0,PCB_ONFAULT(%rcx) 560 SMAP_CLOSE 561 ret 562 563 ALIGN_TEXT 564fusufault: 565 movq PCPU(curthread),%rcx 566 xorl %eax,%eax 567 movq TD_PCB(%rcx), %rcx 568 movq %rax,PCB_ONFAULT(%rcx) 569 decq %rax 570 SMAP_CLOSE 571 ret 572END(std_fubyte) 573 574/* 575 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to 576 * user memory. All these functions are MPSAFE. 577 * 578 * addr = %rdi, value = %rsi 579 * 580 * Write a long 581 */ 582ENTRY(std_suword64) 583 SMAP_OPEN 584 movq PCPU(curthread),%rcx 585 movq TD_PCB(%rcx), %rcx 586 movq $fusufault,PCB_ONFAULT(%rcx) 587 movq %rsp,PCB_ONFAULT_SP(%rcx) 588 589 movq $VM_MAX_USER_ADDRESS-8,%rax 590 cmpq %rax,%rdi /* verify address validity */ 591 ja fusufault 592 593 movq %rsi,(%rdi) 594 xorl %eax,%eax 595 movq PCPU(curthread),%rcx 596 movq TD_PCB(%rcx), %rcx 597 movq %rax,PCB_ONFAULT(%rcx) 598 SMAP_CLOSE 599 ret 600END(std_suword64) 601 602/* 603 * Write an int 604 */ 605ENTRY(std_suword32) 606 SMAP_OPEN 607 movq PCPU(curthread),%rcx 608 movq TD_PCB(%rcx), %rcx 609 movq $fusufault,PCB_ONFAULT(%rcx) 610 movq %rsp,PCB_ONFAULT_SP(%rcx) 611 612 movq $VM_MAX_USER_ADDRESS-4,%rax 613 cmpq %rax,%rdi /* verify address validity */ 614 ja fusufault 615 616 movl %esi,(%rdi) 617 xorl %eax,%eax 618 movq PCPU(curthread),%rcx 619 movq TD_PCB(%rcx), %rcx 620 movq %rax,PCB_ONFAULT(%rcx) 621 SMAP_CLOSE 622 ret 623END(std_suword32) 624 625ENTRY(std_subyte) 626 SMAP_OPEN 627 movq PCPU(curthread),%rcx 628 movq TD_PCB(%rcx), %rcx 629 movq $fusufault,PCB_ONFAULT(%rcx) 630 movq %rsp,PCB_ONFAULT_SP(%rcx) 631 632 movq $VM_MAX_USER_ADDRESS-1,%rax 633 cmpq %rax,%rdi /* verify address validity */ 634 ja fusufault 635 636 movl %esi,%eax 637 movb %al,(%rdi) 638 xorl %eax,%eax 639 movq PCPU(curthread),%rcx /* restore trashed register */ 640 movq TD_PCB(%rcx), %rcx 641 movq %rax,PCB_ONFAULT(%rcx) 642 SMAP_CLOSE 643 ret 644END(std_subyte) 645 646/* 647 * std_copyinstr(from, to, maxlen, int *lencopied) - MP SAFE 648 * %rdi, %rsi, %rdx, %rcx 649 * 650 * copy a string from from to to, stop when a 0 character is reached. 651 * return ENAMETOOLONG if string is longer than maxlen, and 652 * EFAULT on protection violations. If lencopied is non-zero, 653 * return the actual length in *lencopied. 654 */ 655ENTRY(std_copyinstr) 656 SMAP_OPEN 657 movq %rdx,%r8 /* %r8 = maxlen */ 658 movq %rcx,%r9 /* %r9 = *len */ 659 movq PCPU(curthread),%rcx 660 movq TD_PCB(%rcx), %rcx 661 movq $cpystrflt,PCB_ONFAULT(%rcx) 662 movq %rsp,PCB_ONFAULT_SP(%rcx) 663 664 movq $VM_MAX_USER_ADDRESS,%rax 665 666 /* make sure 'from' is within bounds */ 667 subq %rdi,%rax 668 jbe cpystrflt 669 670 /* restrict maxlen to <= VM_MAX_USER_ADDRESS-from */ 671 cmpq %rdx,%rax 672 jae 1f 673 movq %rax,%rdx 674 movq %rax,%r8 6751: 676 incq %rdx 677 6782: 679 decq %rdx 680 jz 3f 681 682 movb (%rdi),%al /* faster than lodsb+stosb */ 683 movb %al,(%rsi) 684 leaq 1(%rdi),%rdi 685 leaq 1(%rsi),%rsi 686 testb %al,%al 687 jnz 2b 688 689 /* Success -- 0 byte reached */ 690 decq %rdx 691 xorl %eax,%eax 692 jmp cpystrflt_x 6933: 694 /* rdx is zero - return ENAMETOOLONG or EFAULT */ 695 movq $VM_MAX_USER_ADDRESS,%rax 696 cmpq %rax,%rsi 697 jae cpystrflt 6984: 699 movq $ENAMETOOLONG,%rax 700 jmp cpystrflt_x 701 702cpystrflt: 703 movq $EFAULT,%rax 704 705cpystrflt_x: 706 SMAP_CLOSE 707 /* set *lencopied and return %eax */ 708 movq PCPU(curthread),%rcx 709 movq TD_PCB(%rcx), %rcx 710 movq $0,PCB_ONFAULT(%rcx) 711 712 testq %r9,%r9 713 jz 1f 714 subq %rdx,%r8 715 movq %r8,(%r9) 7161: 717 ret 718END(std_copyinstr) 719 720/* 721 * copystr(from, to, maxlen, int *lencopied) - MP SAFE 722 * %rdi, %rsi, %rdx, %rcx 723 */ 724ENTRY(copystr) 725 movq %rdx,%r8 /* %r8 = maxlen */ 726 incq %rdx 7271: 728 decq %rdx 729 jz 4f 730 731 movb (%rdi),%al /* faster than lodsb+stosb */ 732 movb %al,(%rsi) 733 leaq 1(%rdi),%rdi 734 leaq 1(%rsi),%rsi 735 testb %al,%al 736 jnz 1b 737 738 /* Success -- 0 byte reached */ 739 decq %rdx 740 xorl %eax,%eax 741 jmp 6f 7424: 743 /* rdx is zero -- return ENAMETOOLONG */ 744 movq $ENAMETOOLONG,%rax 745 7466: 747 testq %rcx,%rcx 748 jz 7f 749 /* set *lencopied and return %rax */ 750 subq %rdx,%r8 751 movq %r8,(%rcx) 7527: 753 ret 754END(copystr) 755 756/* 757 * Handling of special x86_64 registers and descriptor tables etc 758 * %rdi 759 */ 760/* void lgdt(struct region_descriptor *rdp); */ 761ENTRY(lgdt) 762 /* reload the descriptor table */ 763 lgdt (%rdi) 764 765 /* flush the prefetch q */ 766 jmp 1f 767 nop 7681: 769 movl $KDSEL,%eax 770 movl %eax,%ds 771 movl %eax,%es 772 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */ 773 movl %eax,%gs /* Beware, use wrmsr to set 64 bit base */ 774 movl %eax,%ss 775 776 /* reload code selector by turning return into intersegmental return */ 777 popq %rax 778 pushq $KCSEL 779 pushq %rax 780 MEXITCOUNT 781 lretq 782END(lgdt) 783 784/*****************************************************************************/ 785/* setjmp, longjmp */ 786/*****************************************************************************/ 787 788ENTRY(setjmp) 789 movq %rbx,0(%rdi) /* save rbx */ 790 movq %rsp,8(%rdi) /* save rsp */ 791 movq %rbp,16(%rdi) /* save rbp */ 792 movq %r12,24(%rdi) /* save r12 */ 793 movq %r13,32(%rdi) /* save r13 */ 794 movq %r14,40(%rdi) /* save r14 */ 795 movq %r15,48(%rdi) /* save r15 */ 796 movq 0(%rsp),%rdx /* get rta */ 797 movq %rdx,56(%rdi) /* save rip */ 798 xorl %eax,%eax /* return(0); */ 799 ret 800END(setjmp) 801 802ENTRY(longjmp) 803 movq 0(%rdi),%rbx /* restore rbx */ 804 movq 8(%rdi),%rsp /* restore rsp */ 805 movq 16(%rdi),%rbp /* restore rbp */ 806 movq 24(%rdi),%r12 /* restore r12 */ 807 movq 32(%rdi),%r13 /* restore r13 */ 808 movq 40(%rdi),%r14 /* restore r14 */ 809 movq 48(%rdi),%r15 /* restore r15 */ 810 movq 56(%rdi),%rdx /* get rta */ 811 movq %rdx,0(%rsp) /* put in return frame */ 812 xorl %eax,%eax /* return(1); */ 813 incl %eax 814 ret 815END(longjmp) 816 817/* 818 * Support for reading MSRs in the safe manner. 819 */ 820ENTRY(rdmsr_safe) 821/* int rdmsr_safe(u_int msr, uint64_t *data) */ 822 movq PCPU(curthread),%r8 823 movq TD_PCB(%r8), %r8 824 movq $msr_onfault,PCB_ONFAULT(%r8) 825 movq %rsp,PCB_ONFAULT_SP(%r8) 826 movl %edi,%ecx 827 rdmsr /* Read MSR pointed by %ecx. Returns 828 hi byte in edx, lo in %eax */ 829 salq $32,%rdx /* sign-shift %rdx left */ 830 movl %eax,%eax /* zero-extend %eax -> %rax */ 831 orq %rdx,%rax 832 movq %rax,(%rsi) 833 xorq %rax,%rax 834 movq %rax,PCB_ONFAULT(%r8) 835 ret 836END(rdmsr_safe) 837 838/* 839 * Support for writing MSRs in the safe manner. 840 */ 841ENTRY(wrmsr_safe) 842/* int wrmsr_safe(u_int msr, uint64_t data) */ 843 movq PCPU(curthread),%r8 844 movq TD_PCB(%r8), %r8 845 movq $msr_onfault,PCB_ONFAULT(%r8) 846 movq %rsp,PCB_ONFAULT_SP(%r8) 847 movl %edi,%ecx 848 movl %esi,%eax 849 sarq $32,%rsi 850 movl %esi,%edx 851 wrmsr /* Write MSR pointed by %ecx. Accepts 852 hi byte in edx, lo in %eax. */ 853 xorq %rax,%rax 854 movq %rax,PCB_ONFAULT(%r8) 855 ret 856END(wrmsr_safe) 857 858/* 859 * MSR operations fault handler 860 */ 861 ALIGN_TEXT 862msr_onfault: 863 movq PCPU(curthread),%r8 864 movq TD_PCB(%r8), %r8 865 movq $0,PCB_ONFAULT(%r8) 866 movl $EFAULT,%eax 867 ret 868 869ENTRY(smap_open) 870 SMAP_OPEN 871 ret 872END(smap_open) 873 874ENTRY(smap_close) 875 SMAP_CLOSE 876 ret 877END(smap_close) 878