1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 #ifdef XEN 46 extern void xen_cli(void); 47 extern void xen_sti(void); 48 extern u_int xen_rcr2(void); 49 extern void xen_load_cr3(u_int data); 50 extern void xen_tlb_flush(void); 51 extern void xen_invlpg(u_int addr); 52 extern void write_eflags(u_int eflags); 53 extern u_int read_eflags(void); 54 #endif 55 56 struct region_descriptor; 57 58 #define readb(va) (*(volatile uint8_t *) (va)) 59 #define readw(va) (*(volatile uint16_t *) (va)) 60 #define readl(va) (*(volatile uint32_t *) (va)) 61 62 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68 static __inline void 69 breakpoint(void) 70 { 71 __asm __volatile("int $3"); 72 } 73 74 static __inline u_int 75 bsfl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline u_int 84 bsrl(u_int mask) 85 { 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90 } 91 92 static __inline void 93 clflush(u_long addr) 94 { 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97 } 98 99 static __inline void 100 clts(void) 101 { 102 103 __asm __volatile("clts"); 104 } 105 106 static __inline void 107 disable_intr(void) 108 { 109 #ifdef XEN 110 xen_cli(); 111 #else 112 __asm __volatile("cli" : : : "memory"); 113 #endif 114 } 115 116 static __inline void 117 do_cpuid(u_int ax, u_int *p) 118 { 119 __asm __volatile("cpuid" 120 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 121 : "0" (ax)); 122 } 123 124 static __inline void 125 cpuid_count(u_int ax, u_int cx, u_int *p) 126 { 127 __asm __volatile("cpuid" 128 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 129 : "0" (ax), "c" (cx)); 130 } 131 132 static __inline void 133 enable_intr(void) 134 { 135 #ifdef XEN 136 xen_sti(); 137 #else 138 __asm __volatile("sti"); 139 #endif 140 } 141 142 static __inline void 143 cpu_monitor(const void *addr, u_long extensions, u_int hints) 144 { 145 146 __asm __volatile("monitor" 147 : : "a" (addr), "c" (extensions), "d" (hints)); 148 } 149 150 static __inline void 151 cpu_mwait(u_long extensions, u_int hints) 152 { 153 154 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 155 } 156 157 static __inline void 158 lfence(void) 159 { 160 161 __asm __volatile("lfence" : : : "memory"); 162 } 163 164 static __inline void 165 mfence(void) 166 { 167 168 __asm __volatile("mfence" : : : "memory"); 169 } 170 171 #ifdef _KERNEL 172 173 #define HAVE_INLINE_FFS 174 175 static __inline int 176 ffs(int mask) 177 { 178 /* 179 * Note that gcc-2's builtin ffs would be used if we didn't declare 180 * this inline or turn off the builtin. The builtin is faster but 181 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 182 * versions. 183 */ 184 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 185 } 186 187 #define HAVE_INLINE_FFSL 188 189 static __inline int 190 ffsl(long mask) 191 { 192 return (ffs((int)mask)); 193 } 194 195 #define HAVE_INLINE_FLS 196 197 static __inline int 198 fls(int mask) 199 { 200 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 201 } 202 203 #define HAVE_INLINE_FLSL 204 205 static __inline int 206 flsl(long mask) 207 { 208 return (fls((int)mask)); 209 } 210 211 #endif /* _KERNEL */ 212 213 static __inline void 214 halt(void) 215 { 216 __asm __volatile("hlt"); 217 } 218 219 static __inline u_char 220 inb(u_int port) 221 { 222 u_char data; 223 224 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 225 return (data); 226 } 227 228 static __inline u_int 229 inl(u_int port) 230 { 231 u_int data; 232 233 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 234 return (data); 235 } 236 237 static __inline void 238 insb(u_int port, void *addr, size_t count) 239 { 240 __asm __volatile("cld; rep; insb" 241 : "+D" (addr), "+c" (count) 242 : "d" (port) 243 : "memory"); 244 } 245 246 static __inline void 247 insw(u_int port, void *addr, size_t count) 248 { 249 __asm __volatile("cld; rep; insw" 250 : "+D" (addr), "+c" (count) 251 : "d" (port) 252 : "memory"); 253 } 254 255 static __inline void 256 insl(u_int port, void *addr, size_t count) 257 { 258 __asm __volatile("cld; rep; insl" 259 : "+D" (addr), "+c" (count) 260 : "d" (port) 261 : "memory"); 262 } 263 264 static __inline void 265 invd(void) 266 { 267 __asm __volatile("invd"); 268 } 269 270 static __inline u_short 271 inw(u_int port) 272 { 273 u_short data; 274 275 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 276 return (data); 277 } 278 279 static __inline void 280 outb(u_int port, u_char data) 281 { 282 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 283 } 284 285 static __inline void 286 outl(u_int port, u_int data) 287 { 288 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 289 } 290 291 static __inline void 292 outsb(u_int port, const void *addr, size_t count) 293 { 294 __asm __volatile("cld; rep; outsb" 295 : "+S" (addr), "+c" (count) 296 : "d" (port)); 297 } 298 299 static __inline void 300 outsw(u_int port, const void *addr, size_t count) 301 { 302 __asm __volatile("cld; rep; outsw" 303 : "+S" (addr), "+c" (count) 304 : "d" (port)); 305 } 306 307 static __inline void 308 outsl(u_int port, const void *addr, size_t count) 309 { 310 __asm __volatile("cld; rep; outsl" 311 : "+S" (addr), "+c" (count) 312 : "d" (port)); 313 } 314 315 static __inline void 316 outw(u_int port, u_short data) 317 { 318 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 319 } 320 321 static __inline void 322 ia32_pause(void) 323 { 324 __asm __volatile("pause"); 325 } 326 327 static __inline u_int 328 #ifdef XEN 329 _read_eflags(void) 330 #else 331 read_eflags(void) 332 #endif 333 { 334 u_int ef; 335 336 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 337 return (ef); 338 } 339 340 static __inline uint64_t 341 rdmsr(u_int msr) 342 { 343 uint64_t rv; 344 345 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 346 return (rv); 347 } 348 349 static __inline uint32_t 350 rdmsr32(u_int msr) 351 { 352 uint32_t low; 353 354 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 355 return (low); 356 } 357 358 static __inline uint64_t 359 rdpmc(u_int pmc) 360 { 361 uint64_t rv; 362 363 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 364 return (rv); 365 } 366 367 static __inline uint64_t 368 rdtsc(void) 369 { 370 uint64_t rv; 371 372 __asm __volatile("rdtsc" : "=A" (rv)); 373 return (rv); 374 } 375 376 static __inline uint32_t 377 rdtsc32(void) 378 { 379 uint32_t rv; 380 381 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 382 return (rv); 383 } 384 385 static __inline void 386 wbinvd(void) 387 { 388 __asm __volatile("wbinvd"); 389 } 390 391 static __inline void 392 #ifdef XEN 393 _write_eflags(u_int ef) 394 #else 395 write_eflags(u_int ef) 396 #endif 397 { 398 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 399 } 400 401 static __inline void 402 wrmsr(u_int msr, uint64_t newval) 403 { 404 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 405 } 406 407 static __inline void 408 load_cr0(u_int data) 409 { 410 411 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 412 } 413 414 static __inline u_int 415 rcr0(void) 416 { 417 u_int data; 418 419 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 420 return (data); 421 } 422 423 static __inline u_int 424 rcr2(void) 425 { 426 u_int data; 427 428 #ifdef XEN 429 return (xen_rcr2()); 430 #endif 431 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 432 return (data); 433 } 434 435 static __inline void 436 load_cr3(u_int data) 437 { 438 #ifdef XEN 439 xen_load_cr3(data); 440 #else 441 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 442 #endif 443 } 444 445 static __inline u_int 446 rcr3(void) 447 { 448 u_int data; 449 450 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 451 return (data); 452 } 453 454 static __inline void 455 load_cr4(u_int data) 456 { 457 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 458 } 459 460 static __inline u_int 461 rcr4(void) 462 { 463 u_int data; 464 465 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 466 return (data); 467 } 468 469 static __inline uint64_t 470 rxcr(u_int reg) 471 { 472 u_int low, high; 473 474 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 475 return (low | ((uint64_t)high << 32)); 476 } 477 478 static __inline void 479 load_xcr(u_int reg, uint64_t val) 480 { 481 u_int low, high; 482 483 low = val; 484 high = val >> 32; 485 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 486 } 487 488 /* 489 * Global TLB flush (except for thise for pages marked PG_G) 490 */ 491 static __inline void 492 invltlb(void) 493 { 494 #ifdef XEN 495 xen_tlb_flush(); 496 #else 497 load_cr3(rcr3()); 498 #endif 499 } 500 501 /* 502 * TLB flush for an individual page (even if it has PG_G). 503 * Only works on 486+ CPUs (i386 does not have PG_G). 504 */ 505 static __inline void 506 invlpg(u_int addr) 507 { 508 509 #ifdef XEN 510 xen_invlpg(addr); 511 #else 512 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 513 #endif 514 } 515 516 static __inline u_short 517 rfs(void) 518 { 519 u_short sel; 520 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 521 return (sel); 522 } 523 524 static __inline uint64_t 525 rgdt(void) 526 { 527 uint64_t gdtr; 528 __asm __volatile("sgdt %0" : "=m" (gdtr)); 529 return (gdtr); 530 } 531 532 static __inline u_short 533 rgs(void) 534 { 535 u_short sel; 536 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 537 return (sel); 538 } 539 540 static __inline uint64_t 541 ridt(void) 542 { 543 uint64_t idtr; 544 __asm __volatile("sidt %0" : "=m" (idtr)); 545 return (idtr); 546 } 547 548 static __inline u_short 549 rldt(void) 550 { 551 u_short ldtr; 552 __asm __volatile("sldt %0" : "=g" (ldtr)); 553 return (ldtr); 554 } 555 556 static __inline u_short 557 rss(void) 558 { 559 u_short sel; 560 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 561 return (sel); 562 } 563 564 static __inline u_short 565 rtr(void) 566 { 567 u_short tr; 568 __asm __volatile("str %0" : "=g" (tr)); 569 return (tr); 570 } 571 572 static __inline void 573 load_fs(u_short sel) 574 { 575 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 576 } 577 578 static __inline void 579 load_gs(u_short sel) 580 { 581 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 582 } 583 584 static __inline void 585 lidt(struct region_descriptor *addr) 586 { 587 __asm __volatile("lidt (%0)" : : "r" (addr)); 588 } 589 590 static __inline void 591 lldt(u_short sel) 592 { 593 __asm __volatile("lldt %0" : : "r" (sel)); 594 } 595 596 static __inline void 597 ltr(u_short sel) 598 { 599 __asm __volatile("ltr %0" : : "r" (sel)); 600 } 601 602 static __inline u_int 603 rdr0(void) 604 { 605 u_int data; 606 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 607 return (data); 608 } 609 610 static __inline void 611 load_dr0(u_int dr0) 612 { 613 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 614 } 615 616 static __inline u_int 617 rdr1(void) 618 { 619 u_int data; 620 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 621 return (data); 622 } 623 624 static __inline void 625 load_dr1(u_int dr1) 626 { 627 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 628 } 629 630 static __inline u_int 631 rdr2(void) 632 { 633 u_int data; 634 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 635 return (data); 636 } 637 638 static __inline void 639 load_dr2(u_int dr2) 640 { 641 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 642 } 643 644 static __inline u_int 645 rdr3(void) 646 { 647 u_int data; 648 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 649 return (data); 650 } 651 652 static __inline void 653 load_dr3(u_int dr3) 654 { 655 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 656 } 657 658 static __inline u_int 659 rdr4(void) 660 { 661 u_int data; 662 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 663 return (data); 664 } 665 666 static __inline void 667 load_dr4(u_int dr4) 668 { 669 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 670 } 671 672 static __inline u_int 673 rdr5(void) 674 { 675 u_int data; 676 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 677 return (data); 678 } 679 680 static __inline void 681 load_dr5(u_int dr5) 682 { 683 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 684 } 685 686 static __inline u_int 687 rdr6(void) 688 { 689 u_int data; 690 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 691 return (data); 692 } 693 694 static __inline void 695 load_dr6(u_int dr6) 696 { 697 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 698 } 699 700 static __inline u_int 701 rdr7(void) 702 { 703 u_int data; 704 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 705 return (data); 706 } 707 708 static __inline void 709 load_dr7(u_int dr7) 710 { 711 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 712 } 713 714 static __inline u_char 715 read_cyrix_reg(u_char reg) 716 { 717 outb(0x22, reg); 718 return inb(0x23); 719 } 720 721 static __inline void 722 write_cyrix_reg(u_char reg, u_char data) 723 { 724 outb(0x22, reg); 725 outb(0x23, data); 726 } 727 728 static __inline register_t 729 intr_disable(void) 730 { 731 register_t eflags; 732 733 eflags = read_eflags(); 734 disable_intr(); 735 return (eflags); 736 } 737 738 static __inline void 739 intr_restore(register_t eflags) 740 { 741 write_eflags(eflags); 742 } 743 744 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 745 746 int breakpoint(void); 747 u_int bsfl(u_int mask); 748 u_int bsrl(u_int mask); 749 void clflush(u_long addr); 750 void clts(void); 751 void cpuid_count(u_int ax, u_int cx, u_int *p); 752 void disable_intr(void); 753 void do_cpuid(u_int ax, u_int *p); 754 void enable_intr(void); 755 void halt(void); 756 void ia32_pause(void); 757 u_char inb(u_int port); 758 u_int inl(u_int port); 759 void insb(u_int port, void *addr, size_t count); 760 void insl(u_int port, void *addr, size_t count); 761 void insw(u_int port, void *addr, size_t count); 762 register_t intr_disable(void); 763 void intr_restore(register_t ef); 764 void invd(void); 765 void invlpg(u_int addr); 766 void invltlb(void); 767 u_short inw(u_int port); 768 void lidt(struct region_descriptor *addr); 769 void lldt(u_short sel); 770 void load_cr0(u_int cr0); 771 void load_cr3(u_int cr3); 772 void load_cr4(u_int cr4); 773 void load_dr0(u_int dr0); 774 void load_dr1(u_int dr1); 775 void load_dr2(u_int dr2); 776 void load_dr3(u_int dr3); 777 void load_dr4(u_int dr4); 778 void load_dr5(u_int dr5); 779 void load_dr6(u_int dr6); 780 void load_dr7(u_int dr7); 781 void load_fs(u_short sel); 782 void load_gs(u_short sel); 783 void ltr(u_short sel); 784 void outb(u_int port, u_char data); 785 void outl(u_int port, u_int data); 786 void outsb(u_int port, const void *addr, size_t count); 787 void outsl(u_int port, const void *addr, size_t count); 788 void outsw(u_int port, const void *addr, size_t count); 789 void outw(u_int port, u_short data); 790 u_int rcr0(void); 791 u_int rcr2(void); 792 u_int rcr3(void); 793 u_int rcr4(void); 794 uint64_t rdmsr(u_int msr); 795 uint64_t rdpmc(u_int pmc); 796 u_int rdr0(void); 797 u_int rdr1(void); 798 u_int rdr2(void); 799 u_int rdr3(void); 800 u_int rdr4(void); 801 u_int rdr5(void); 802 u_int rdr6(void); 803 u_int rdr7(void); 804 uint64_t rdtsc(void); 805 u_char read_cyrix_reg(u_char reg); 806 u_int read_eflags(void); 807 u_int rfs(void); 808 uint64_t rgdt(void); 809 u_int rgs(void); 810 uint64_t ridt(void); 811 u_short rldt(void); 812 u_short rtr(void); 813 void wbinvd(void); 814 void write_cyrix_reg(u_char reg, u_char data); 815 void write_eflags(u_int ef); 816 void wrmsr(u_int msr, uint64_t newval); 817 818 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 819 820 void reset_dbregs(void); 821 822 #ifdef _KERNEL 823 int rdmsr_safe(u_int msr, uint64_t *val); 824 int wrmsr_safe(u_int msr, uint64_t newval); 825 #endif 826 827 #endif /* !_MACHINE_CPUFUNC_H_ */ 828