1 /*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 /* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38 #ifndef _MACHINE_CPUFUNC_H_ 39 #define _MACHINE_CPUFUNC_H_ 40 41 #ifndef _SYS_CDEFS_H_ 42 #error this file needs sys/cdefs.h as a prerequisite 43 #endif 44 45 #ifdef XEN 46 extern void xen_cli(void); 47 extern void xen_sti(void); 48 extern u_int xen_rcr2(void); 49 extern void xen_load_cr3(u_int data); 50 extern void xen_tlb_flush(void); 51 extern void xen_invlpg(u_int addr); 52 extern void write_eflags(u_int eflags); 53 extern u_int read_eflags(void); 54 #endif 55 56 struct region_descriptor; 57 58 #define readb(va) (*(volatile u_int8_t *) (va)) 59 #define readw(va) (*(volatile u_int16_t *) (va)) 60 #define readl(va) (*(volatile u_int32_t *) (va)) 61 62 #define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 63 #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 64 #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 65 66 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68 static __inline void 69 breakpoint(void) 70 { 71 __asm __volatile("int $3"); 72 } 73 74 static __inline u_int 75 bsfl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline u_int 84 bsrl(u_int mask) 85 { 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90 } 91 92 static __inline void 93 clflush(u_long addr) 94 { 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97 } 98 99 static __inline void 100 disable_intr(void) 101 { 102 #ifdef XEN 103 xen_cli(); 104 #else 105 __asm __volatile("cli" : : : "memory"); 106 #endif 107 } 108 109 static __inline void 110 do_cpuid(u_int ax, u_int *p) 111 { 112 __asm __volatile("cpuid" 113 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 114 : "0" (ax)); 115 } 116 117 static __inline void 118 cpuid_count(u_int ax, u_int cx, u_int *p) 119 { 120 __asm __volatile("cpuid" 121 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 122 : "0" (ax), "c" (cx)); 123 } 124 125 static __inline void 126 enable_intr(void) 127 { 128 #ifdef XEN 129 xen_sti(); 130 #else 131 __asm __volatile("sti"); 132 #endif 133 } 134 135 static __inline void 136 cpu_monitor(const void *addr, int extensions, int hints) 137 { 138 __asm __volatile("monitor;" 139 : :"a" (addr), "c" (extensions), "d"(hints)); 140 } 141 142 static __inline void 143 cpu_mwait(int extensions, int hints) 144 { 145 __asm __volatile("mwait;" : :"a" (hints), "c" (extensions)); 146 } 147 148 static __inline void 149 mfence(void) 150 { 151 152 __asm __volatile("mfence" : : : "memory"); 153 } 154 155 #ifdef _KERNEL 156 157 #define HAVE_INLINE_FFS 158 159 static __inline int 160 ffs(int mask) 161 { 162 /* 163 * Note that gcc-2's builtin ffs would be used if we didn't declare 164 * this inline or turn off the builtin. The builtin is faster but 165 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 166 * versions. 167 */ 168 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 169 } 170 171 #define HAVE_INLINE_FLS 172 173 static __inline int 174 fls(int mask) 175 { 176 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 177 } 178 179 #endif /* _KERNEL */ 180 181 static __inline void 182 halt(void) 183 { 184 __asm __volatile("hlt"); 185 } 186 187 static __inline u_char 188 inb(u_int port) 189 { 190 u_char data; 191 192 __asm volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 193 return (data); 194 } 195 196 static __inline u_int 197 inl(u_int port) 198 { 199 u_int data; 200 201 __asm volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 202 return (data); 203 } 204 205 static __inline void 206 insb(u_int port, void *addr, size_t count) 207 { 208 __asm __volatile("cld; rep; insb" 209 : "+D" (addr), "+c" (count) 210 : "d" (port) 211 : "memory"); 212 } 213 214 static __inline void 215 insw(u_int port, void *addr, size_t count) 216 { 217 __asm __volatile("cld; rep; insw" 218 : "+D" (addr), "+c" (count) 219 : "d" (port) 220 : "memory"); 221 } 222 223 static __inline void 224 insl(u_int port, void *addr, size_t count) 225 { 226 __asm __volatile("cld; rep; insl" 227 : "+D" (addr), "+c" (count) 228 : "d" (port) 229 : "memory"); 230 } 231 232 static __inline void 233 invd(void) 234 { 235 __asm __volatile("invd"); 236 } 237 238 static __inline u_short 239 inw(u_int port) 240 { 241 u_short data; 242 243 __asm volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 244 return (data); 245 } 246 247 static __inline void 248 outb(u_int port, u_char data) 249 { 250 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 251 } 252 253 static __inline void 254 outl(u_int port, u_int data) 255 { 256 __asm volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 257 } 258 259 static __inline void 260 outsb(u_int port, const void *addr, size_t count) 261 { 262 __asm __volatile("cld; rep; outsb" 263 : "+S" (addr), "+c" (count) 264 : "d" (port)); 265 } 266 267 static __inline void 268 outsw(u_int port, const void *addr, size_t count) 269 { 270 __asm __volatile("cld; rep; outsw" 271 : "+S" (addr), "+c" (count) 272 : "d" (port)); 273 } 274 275 static __inline void 276 outsl(u_int port, const void *addr, size_t count) 277 { 278 __asm __volatile("cld; rep; outsl" 279 : "+S" (addr), "+c" (count) 280 : "d" (port)); 281 } 282 283 static __inline void 284 outw(u_int port, u_short data) 285 { 286 __asm volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 287 } 288 289 static __inline void 290 ia32_pause(void) 291 { 292 __asm __volatile("pause"); 293 } 294 295 static __inline u_int 296 #ifdef XEN 297 _read_eflags(void) 298 #else 299 read_eflags(void) 300 #endif 301 { 302 u_int ef; 303 304 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 305 return (ef); 306 } 307 308 static __inline uint64_t 309 rdmsr(u_int msr) 310 { 311 uint64_t rv; 312 313 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 314 return (rv); 315 } 316 317 static __inline uint64_t 318 rdpmc(u_int pmc) 319 { 320 uint64_t rv; 321 322 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 323 return (rv); 324 } 325 326 static __inline uint64_t 327 rdtsc(void) 328 { 329 uint64_t rv; 330 331 __asm __volatile("rdtsc" : "=A" (rv)); 332 return (rv); 333 } 334 335 static __inline void 336 wbinvd(void) 337 { 338 __asm __volatile("wbinvd"); 339 } 340 341 static __inline void 342 #ifdef XEN 343 _write_eflags(u_int ef) 344 #else 345 write_eflags(u_int ef) 346 #endif 347 { 348 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 349 } 350 351 static __inline void 352 wrmsr(u_int msr, uint64_t newval) 353 { 354 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 355 } 356 357 static __inline void 358 load_cr0(u_int data) 359 { 360 361 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 362 } 363 364 static __inline u_int 365 rcr0(void) 366 { 367 u_int data; 368 369 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 370 return (data); 371 } 372 373 static __inline u_int 374 rcr2(void) 375 { 376 u_int data; 377 378 #ifdef XEN 379 return (xen_rcr2()); 380 #endif 381 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 382 return (data); 383 } 384 385 static __inline void 386 load_cr3(u_int data) 387 { 388 #ifdef XEN 389 xen_load_cr3(data); 390 #else 391 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 392 #endif 393 } 394 395 static __inline u_int 396 rcr3(void) 397 { 398 u_int data; 399 400 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 401 return (data); 402 } 403 404 static __inline void 405 load_cr4(u_int data) 406 { 407 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 408 } 409 410 static __inline u_int 411 rcr4(void) 412 { 413 u_int data; 414 415 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 416 return (data); 417 } 418 419 /* 420 * Global TLB flush (except for thise for pages marked PG_G) 421 */ 422 static __inline void 423 invltlb(void) 424 { 425 #ifdef XEN 426 xen_tlb_flush(); 427 #else 428 load_cr3(rcr3()); 429 #endif 430 } 431 432 /* 433 * TLB flush for an individual page (even if it has PG_G). 434 * Only works on 486+ CPUs (i386 does not have PG_G). 435 */ 436 static __inline void 437 invlpg(u_int addr) 438 { 439 440 #ifdef XEN 441 xen_invlpg(addr); 442 #else 443 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 444 #endif 445 } 446 447 static __inline u_short 448 rfs(void) 449 { 450 u_short sel; 451 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 452 return (sel); 453 } 454 455 static __inline uint64_t 456 rgdt(void) 457 { 458 uint64_t gdtr; 459 __asm __volatile("sgdt %0" : "=m" (gdtr)); 460 return (gdtr); 461 } 462 463 static __inline u_short 464 rgs(void) 465 { 466 u_short sel; 467 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 468 return (sel); 469 } 470 471 static __inline uint64_t 472 ridt(void) 473 { 474 uint64_t idtr; 475 __asm __volatile("sidt %0" : "=m" (idtr)); 476 return (idtr); 477 } 478 479 static __inline u_short 480 rldt(void) 481 { 482 u_short ldtr; 483 __asm __volatile("sldt %0" : "=g" (ldtr)); 484 return (ldtr); 485 } 486 487 static __inline u_short 488 rss(void) 489 { 490 u_short sel; 491 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 492 return (sel); 493 } 494 495 static __inline u_short 496 rtr(void) 497 { 498 u_short tr; 499 __asm __volatile("str %0" : "=g" (tr)); 500 return (tr); 501 } 502 503 static __inline void 504 load_fs(u_short sel) 505 { 506 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 507 } 508 509 static __inline void 510 load_gs(u_short sel) 511 { 512 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 513 } 514 515 static __inline void 516 lidt(struct region_descriptor *addr) 517 { 518 __asm __volatile("lidt (%0)" : : "r" (addr)); 519 } 520 521 static __inline void 522 lldt(u_short sel) 523 { 524 __asm __volatile("lldt %0" : : "r" (sel)); 525 } 526 527 static __inline void 528 ltr(u_short sel) 529 { 530 __asm __volatile("ltr %0" : : "r" (sel)); 531 } 532 533 static __inline u_int 534 rdr0(void) 535 { 536 u_int data; 537 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 538 return (data); 539 } 540 541 static __inline void 542 load_dr0(u_int dr0) 543 { 544 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 545 } 546 547 static __inline u_int 548 rdr1(void) 549 { 550 u_int data; 551 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 552 return (data); 553 } 554 555 static __inline void 556 load_dr1(u_int dr1) 557 { 558 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 559 } 560 561 static __inline u_int 562 rdr2(void) 563 { 564 u_int data; 565 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 566 return (data); 567 } 568 569 static __inline void 570 load_dr2(u_int dr2) 571 { 572 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 573 } 574 575 static __inline u_int 576 rdr3(void) 577 { 578 u_int data; 579 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 580 return (data); 581 } 582 583 static __inline void 584 load_dr3(u_int dr3) 585 { 586 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 587 } 588 589 static __inline u_int 590 rdr4(void) 591 { 592 u_int data; 593 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 594 return (data); 595 } 596 597 static __inline void 598 load_dr4(u_int dr4) 599 { 600 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 601 } 602 603 static __inline u_int 604 rdr5(void) 605 { 606 u_int data; 607 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 608 return (data); 609 } 610 611 static __inline void 612 load_dr5(u_int dr5) 613 { 614 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 615 } 616 617 static __inline u_int 618 rdr6(void) 619 { 620 u_int data; 621 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 622 return (data); 623 } 624 625 static __inline void 626 load_dr6(u_int dr6) 627 { 628 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 629 } 630 631 static __inline u_int 632 rdr7(void) 633 { 634 u_int data; 635 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 636 return (data); 637 } 638 639 static __inline void 640 load_dr7(u_int dr7) 641 { 642 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 643 } 644 645 static __inline u_char 646 read_cyrix_reg(u_char reg) 647 { 648 outb(0x22, reg); 649 return inb(0x23); 650 } 651 652 static __inline void 653 write_cyrix_reg(u_char reg, u_char data) 654 { 655 outb(0x22, reg); 656 outb(0x23, data); 657 } 658 659 static __inline register_t 660 intr_disable(void) 661 { 662 register_t eflags; 663 664 eflags = read_eflags(); 665 disable_intr(); 666 return (eflags); 667 } 668 669 static __inline void 670 intr_restore(register_t eflags) 671 { 672 write_eflags(eflags); 673 } 674 675 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 676 677 int breakpoint(void); 678 u_int bsfl(u_int mask); 679 u_int bsrl(u_int mask); 680 void disable_intr(void); 681 void do_cpuid(u_int ax, u_int *p); 682 void enable_intr(void); 683 void halt(void); 684 void ia32_pause(void); 685 u_char inb(u_int port); 686 u_int inl(u_int port); 687 void insb(u_int port, void *addr, size_t count); 688 void insl(u_int port, void *addr, size_t count); 689 void insw(u_int port, void *addr, size_t count); 690 register_t intr_disable(void); 691 void intr_restore(register_t ef); 692 void invd(void); 693 void invlpg(u_int addr); 694 void invltlb(void); 695 u_short inw(u_int port); 696 void lidt(struct region_descriptor *addr); 697 void lldt(u_short sel); 698 void load_cr0(u_int cr0); 699 void load_cr3(u_int cr3); 700 void load_cr4(u_int cr4); 701 void load_dr0(u_int dr0); 702 void load_dr1(u_int dr1); 703 void load_dr2(u_int dr2); 704 void load_dr3(u_int dr3); 705 void load_dr4(u_int dr4); 706 void load_dr5(u_int dr5); 707 void load_dr6(u_int dr6); 708 void load_dr7(u_int dr7); 709 void load_fs(u_short sel); 710 void load_gs(u_short sel); 711 void ltr(u_short sel); 712 void outb(u_int port, u_char data); 713 void outl(u_int port, u_int data); 714 void outsb(u_int port, const void *addr, size_t count); 715 void outsl(u_int port, const void *addr, size_t count); 716 void outsw(u_int port, const void *addr, size_t count); 717 void outw(u_int port, u_short data); 718 u_int rcr0(void); 719 u_int rcr2(void); 720 u_int rcr3(void); 721 u_int rcr4(void); 722 uint64_t rdmsr(u_int msr); 723 uint64_t rdpmc(u_int pmc); 724 u_int rdr0(void); 725 u_int rdr1(void); 726 u_int rdr2(void); 727 u_int rdr3(void); 728 u_int rdr4(void); 729 u_int rdr5(void); 730 u_int rdr6(void); 731 u_int rdr7(void); 732 uint64_t rdtsc(void); 733 u_char read_cyrix_reg(u_char reg); 734 u_int read_eflags(void); 735 u_int rfs(void); 736 uint64_t rgdt(void); 737 u_int rgs(void); 738 uint64_t ridt(void); 739 u_short rldt(void); 740 u_short rtr(void); 741 void wbinvd(void); 742 void write_cyrix_reg(u_char reg, u_char data); 743 void write_eflags(u_int ef); 744 void wrmsr(u_int msr, uint64_t newval); 745 746 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 747 748 void reset_dbregs(void); 749 750 #ifdef _KERNEL 751 int rdmsr_safe(u_int msr, uint64_t *val); 752 int wrmsr_safe(u_int msr, uint64_t newval); 753 #endif 754 755 #endif /* !_MACHINE_CPUFUNC_H_ */ 756