1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 struct region_descriptor; 46 47 #define readb(va) (*(volatile uint8_t *) (va)) 48 #define readw(va) (*(volatile uint16_t *) (va)) 49 #define readl(va) (*(volatile uint32_t *) (va)) 50 51 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 52 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 53 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 54 55 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 56 57 static __inline void 58 breakpoint(void) 59 { 60 __asm __volatile("int $3"); 61 } 62 63 static __inline u_int 64 bsfl(u_int mask) 65 { 66 u_int result; 67 68 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 69 return (result); 70 } 71 72 static __inline u_int 73 bsrl(u_int mask) 74 { 75 u_int result; 76 77 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 78 return (result); 79 } 80 81 static __inline void 82 clflush(u_long addr) 83 { 84 85 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 86 } 87 88 static __inline void 89 clflushopt(u_long addr) 90 { 91 92 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 93 } 94 95 static __inline void 96 clts(void) 97 { 98 99 __asm __volatile("clts"); 100 } 101 102 static __inline void 103 disable_intr(void) 104 { 105 106 __asm __volatile("cli" : : : "memory"); 107 } 108 109 static __inline void 110 do_cpuid(u_int ax, u_int *p) 111 { 112 __asm __volatile("cpuid" 113 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 114 : "0" (ax)); 115 } 116 117 static __inline void 118 cpuid_count(u_int ax, u_int cx, u_int *p) 119 { 120 __asm __volatile("cpuid" 121 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 122 : "0" (ax), "c" (cx)); 123 } 124 125 static __inline void 126 enable_intr(void) 127 { 128 129 __asm __volatile("sti"); 130 } 131 132 static __inline void 133 cpu_monitor(const void *addr, u_long extensions, u_int hints) 134 { 135 136 __asm __volatile("monitor" 137 : : "a" (addr), "c" (extensions), "d" (hints)); 138 } 139 140 static __inline void 141 cpu_mwait(u_long extensions, u_int hints) 142 { 143 144 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 145 } 146 147 static __inline void 148 lfence(void) 149 { 150 151 __asm __volatile("lfence" : : : "memory"); 152 } 153 154 static __inline void 155 mfence(void) 156 { 157 158 __asm __volatile("mfence" : : : "memory"); 159 } 160 161 static __inline void 162 sfence(void) 163 { 164 165 __asm __volatile("sfence" : : : "memory"); 166 } 167 168 #ifdef _KERNEL 169 170 #define HAVE_INLINE_FFS 171 172 static __inline int 173 ffs(int mask) 174 { 175 /* 176 * Note that gcc-2's builtin ffs would be used if we didn't declare 177 * this inline or turn off the builtin. The builtin is faster but 178 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 179 * versions. 180 */ 181 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 182 } 183 184 #define HAVE_INLINE_FFSL 185 186 static __inline int 187 ffsl(long mask) 188 { 189 return (ffs((int)mask)); 190 } 191 192 #define HAVE_INLINE_FLS 193 194 static __inline int 195 fls(int mask) 196 { 197 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 198 } 199 200 #define HAVE_INLINE_FLSL 201 202 static __inline int 203 flsl(long mask) 204 { 205 return (fls((int)mask)); 206 } 207 208 #endif /* _KERNEL */ 209 210 static __inline void 211 halt(void) 212 { 213 __asm __volatile("hlt"); 214 } 215 216 static __inline u_char 217 inb(u_int port) 218 { 219 u_char data; 220 221 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 222 return (data); 223 } 224 225 static __inline u_int 226 inl(u_int port) 227 { 228 u_int data; 229 230 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 231 return (data); 232 } 233 234 static __inline void 235 insb(u_int port, void *addr, size_t count) 236 { 237 __asm __volatile("cld; rep; insb" 238 : "+D" (addr), "+c" (count) 239 : "d" (port) 240 : "memory"); 241 } 242 243 static __inline void 244 insw(u_int port, void *addr, size_t count) 245 { 246 __asm __volatile("cld; rep; insw" 247 : "+D" (addr), "+c" (count) 248 : "d" (port) 249 : "memory"); 250 } 251 252 static __inline void 253 insl(u_int port, void *addr, size_t count) 254 { 255 __asm __volatile("cld; rep; insl" 256 : "+D" (addr), "+c" (count) 257 : "d" (port) 258 : "memory"); 259 } 260 261 static __inline void 262 invd(void) 263 { 264 __asm __volatile("invd"); 265 } 266 267 static __inline u_short 268 inw(u_int port) 269 { 270 u_short data; 271 272 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 273 return (data); 274 } 275 276 static __inline void 277 outb(u_int port, u_char data) 278 { 279 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 280 } 281 282 static __inline void 283 outl(u_int port, u_int data) 284 { 285 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 286 } 287 288 static __inline void 289 outsb(u_int port, const void *addr, size_t count) 290 { 291 __asm __volatile("cld; rep; outsb" 292 : "+S" (addr), "+c" (count) 293 : "d" (port)); 294 } 295 296 static __inline void 297 outsw(u_int port, const void *addr, size_t count) 298 { 299 __asm __volatile("cld; rep; outsw" 300 : "+S" (addr), "+c" (count) 301 : "d" (port)); 302 } 303 304 static __inline void 305 outsl(u_int port, const void *addr, size_t count) 306 { 307 __asm __volatile("cld; rep; outsl" 308 : "+S" (addr), "+c" (count) 309 : "d" (port)); 310 } 311 312 static __inline void 313 outw(u_int port, u_short data) 314 { 315 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 316 } 317 318 static __inline void 319 ia32_pause(void) 320 { 321 __asm __volatile("pause"); 322 } 323 324 static __inline u_int 325 read_eflags(void) 326 { 327 u_int ef; 328 329 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 330 return (ef); 331 } 332 333 static __inline uint64_t 334 rdmsr(u_int msr) 335 { 336 uint64_t rv; 337 338 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 339 return (rv); 340 } 341 342 static __inline uint32_t 343 rdmsr32(u_int msr) 344 { 345 uint32_t low; 346 347 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 348 return (low); 349 } 350 351 static __inline uint64_t 352 rdpmc(u_int pmc) 353 { 354 uint64_t rv; 355 356 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 357 return (rv); 358 } 359 360 static __inline uint64_t 361 rdtsc(void) 362 { 363 uint64_t rv; 364 365 __asm __volatile("rdtsc" : "=A" (rv)); 366 return (rv); 367 } 368 369 static __inline uint32_t 370 rdtsc32(void) 371 { 372 uint32_t rv; 373 374 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 375 return (rv); 376 } 377 378 static __inline void 379 wbinvd(void) 380 { 381 __asm __volatile("wbinvd"); 382 } 383 384 static __inline void 385 write_eflags(u_int ef) 386 { 387 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 388 } 389 390 static __inline void 391 wrmsr(u_int msr, uint64_t newval) 392 { 393 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 394 } 395 396 static __inline void 397 load_cr0(u_int data) 398 { 399 400 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 401 } 402 403 static __inline u_int 404 rcr0(void) 405 { 406 u_int data; 407 408 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 409 return (data); 410 } 411 412 static __inline u_int 413 rcr2(void) 414 { 415 u_int data; 416 417 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 418 return (data); 419 } 420 421 static __inline void 422 load_cr3(u_int data) 423 { 424 425 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 426 } 427 428 static __inline u_int 429 rcr3(void) 430 { 431 u_int data; 432 433 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 434 return (data); 435 } 436 437 static __inline void 438 load_cr4(u_int data) 439 { 440 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 441 } 442 443 static __inline u_int 444 rcr4(void) 445 { 446 u_int data; 447 448 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 449 return (data); 450 } 451 452 static __inline uint64_t 453 rxcr(u_int reg) 454 { 455 u_int low, high; 456 457 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 458 return (low | ((uint64_t)high << 32)); 459 } 460 461 static __inline void 462 load_xcr(u_int reg, uint64_t val) 463 { 464 u_int low, high; 465 466 low = val; 467 high = val >> 32; 468 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 469 } 470 471 /* 472 * Global TLB flush (except for thise for pages marked PG_G) 473 */ 474 static __inline void 475 invltlb(void) 476 { 477 478 load_cr3(rcr3()); 479 } 480 481 /* 482 * TLB flush for an individual page (even if it has PG_G). 483 * Only works on 486+ CPUs (i386 does not have PG_G). 484 */ 485 static __inline void 486 invlpg(u_int addr) 487 { 488 489 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 490 } 491 492 static __inline u_short 493 rfs(void) 494 { 495 u_short sel; 496 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 497 return (sel); 498 } 499 500 static __inline uint64_t 501 rgdt(void) 502 { 503 uint64_t gdtr; 504 __asm __volatile("sgdt %0" : "=m" (gdtr)); 505 return (gdtr); 506 } 507 508 static __inline u_short 509 rgs(void) 510 { 511 u_short sel; 512 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 513 return (sel); 514 } 515 516 static __inline uint64_t 517 ridt(void) 518 { 519 uint64_t idtr; 520 __asm __volatile("sidt %0" : "=m" (idtr)); 521 return (idtr); 522 } 523 524 static __inline u_short 525 rldt(void) 526 { 527 u_short ldtr; 528 __asm __volatile("sldt %0" : "=g" (ldtr)); 529 return (ldtr); 530 } 531 532 static __inline u_short 533 rss(void) 534 { 535 u_short sel; 536 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 537 return (sel); 538 } 539 540 static __inline u_short 541 rtr(void) 542 { 543 u_short tr; 544 __asm __volatile("str %0" : "=g" (tr)); 545 return (tr); 546 } 547 548 static __inline void 549 load_fs(u_short sel) 550 { 551 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 552 } 553 554 static __inline void 555 load_gs(u_short sel) 556 { 557 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 558 } 559 560 static __inline void 561 lidt(struct region_descriptor *addr) 562 { 563 __asm __volatile("lidt (%0)" : : "r" (addr)); 564 } 565 566 static __inline void 567 lldt(u_short sel) 568 { 569 __asm __volatile("lldt %0" : : "r" (sel)); 570 } 571 572 static __inline void 573 ltr(u_short sel) 574 { 575 __asm __volatile("ltr %0" : : "r" (sel)); 576 } 577 578 static __inline u_int 579 rdr0(void) 580 { 581 u_int data; 582 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 583 return (data); 584 } 585 586 static __inline void 587 load_dr0(u_int dr0) 588 { 589 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 590 } 591 592 static __inline u_int 593 rdr1(void) 594 { 595 u_int data; 596 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 597 return (data); 598 } 599 600 static __inline void 601 load_dr1(u_int dr1) 602 { 603 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 604 } 605 606 static __inline u_int 607 rdr2(void) 608 { 609 u_int data; 610 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 611 return (data); 612 } 613 614 static __inline void 615 load_dr2(u_int dr2) 616 { 617 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 618 } 619 620 static __inline u_int 621 rdr3(void) 622 { 623 u_int data; 624 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 625 return (data); 626 } 627 628 static __inline void 629 load_dr3(u_int dr3) 630 { 631 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 632 } 633 634 static __inline u_int 635 rdr6(void) 636 { 637 u_int data; 638 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 639 return (data); 640 } 641 642 static __inline void 643 load_dr6(u_int dr6) 644 { 645 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 646 } 647 648 static __inline u_int 649 rdr7(void) 650 { 651 u_int data; 652 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 653 return (data); 654 } 655 656 static __inline void 657 load_dr7(u_int dr7) 658 { 659 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 660 } 661 662 static __inline u_char 663 read_cyrix_reg(u_char reg) 664 { 665 outb(0x22, reg); 666 return inb(0x23); 667 } 668 669 static __inline void 670 write_cyrix_reg(u_char reg, u_char data) 671 { 672 outb(0x22, reg); 673 outb(0x23, data); 674 } 675 676 static __inline register_t 677 intr_disable(void) 678 { 679 register_t eflags; 680 681 eflags = read_eflags(); 682 disable_intr(); 683 return (eflags); 684 } 685 686 static __inline void 687 intr_restore(register_t eflags) 688 { 689 write_eflags(eflags); 690 } 691 692 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 693 694 int breakpoint(void); 695 u_int bsfl(u_int mask); 696 u_int bsrl(u_int mask); 697 void clflush(u_long addr); 698 void clts(void); 699 void cpuid_count(u_int ax, u_int cx, u_int *p); 700 void disable_intr(void); 701 void do_cpuid(u_int ax, u_int *p); 702 void enable_intr(void); 703 void halt(void); 704 void ia32_pause(void); 705 u_char inb(u_int port); 706 u_int inl(u_int port); 707 void insb(u_int port, void *addr, size_t count); 708 void insl(u_int port, void *addr, size_t count); 709 void insw(u_int port, void *addr, size_t count); 710 register_t intr_disable(void); 711 void intr_restore(register_t ef); 712 void invd(void); 713 void invlpg(u_int addr); 714 void invltlb(void); 715 u_short inw(u_int port); 716 void lidt(struct region_descriptor *addr); 717 void lldt(u_short sel); 718 void load_cr0(u_int cr0); 719 void load_cr3(u_int cr3); 720 void load_cr4(u_int cr4); 721 void load_dr0(u_int dr0); 722 void load_dr1(u_int dr1); 723 void load_dr2(u_int dr2); 724 void load_dr3(u_int dr3); 725 void load_dr6(u_int dr6); 726 void load_dr7(u_int dr7); 727 void load_fs(u_short sel); 728 void load_gs(u_short sel); 729 void ltr(u_short sel); 730 void outb(u_int port, u_char data); 731 void outl(u_int port, u_int data); 732 void outsb(u_int port, const void *addr, size_t count); 733 void outsl(u_int port, const void *addr, size_t count); 734 void outsw(u_int port, const void *addr, size_t count); 735 void outw(u_int port, u_short data); 736 u_int rcr0(void); 737 u_int rcr2(void); 738 u_int rcr3(void); 739 u_int rcr4(void); 740 uint64_t rdmsr(u_int msr); 741 uint64_t rdpmc(u_int pmc); 742 u_int rdr0(void); 743 u_int rdr1(void); 744 u_int rdr2(void); 745 u_int rdr3(void); 746 u_int rdr6(void); 747 u_int rdr7(void); 748 uint64_t rdtsc(void); 749 u_char read_cyrix_reg(u_char reg); 750 u_int read_eflags(void); 751 u_int rfs(void); 752 uint64_t rgdt(void); 753 u_int rgs(void); 754 uint64_t ridt(void); 755 u_short rldt(void); 756 u_short rtr(void); 757 void wbinvd(void); 758 void write_cyrix_reg(u_char reg, u_char data); 759 void write_eflags(u_int ef); 760 void wrmsr(u_int msr, uint64_t newval); 761 762 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 763 764 void reset_dbregs(void); 765 766 #ifdef _KERNEL 767 int rdmsr_safe(u_int msr, uint64_t *val); 768 int wrmsr_safe(u_int msr, uint64_t newval); 769 #endif 770 771 #endif /* !_MACHINE_CPUFUNC_H_ */ 772