1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 /* 35 * Functions to provide access to special i386 instructions. 36 * This in included in sys/systm.h, and that file should be 37 * used in preference to this. 38 */ 39 40 #ifndef _MACHINE_CPUFUNC_H_ 41 #define _MACHINE_CPUFUNC_H_ 42 43 #ifndef _SYS_CDEFS_H_ 44 #error this file needs sys/cdefs.h as a prerequisite 45 #endif 46 47 struct region_descriptor; 48 49 #define readb(va) (*(volatile uint8_t *) (va)) 50 #define readw(va) (*(volatile uint16_t *) (va)) 51 #define readl(va) (*(volatile uint32_t *) (va)) 52 53 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 54 #define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 55 #define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 56 57 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 58 59 static __inline void 60 breakpoint(void) 61 { 62 __asm __volatile("int $3"); 63 } 64 65 static __inline __pure2 u_int 66 bsfl(u_int mask) 67 { 68 u_int result; 69 70 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 71 return (result); 72 } 73 74 static __inline __pure2 u_int 75 bsrl(u_int mask) 76 { 77 u_int result; 78 79 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81 } 82 83 static __inline void 84 clflush(u_long addr) 85 { 86 87 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 88 } 89 90 static __inline void 91 clflushopt(u_long addr) 92 { 93 94 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 95 } 96 97 static __inline void 98 clts(void) 99 { 100 101 __asm __volatile("clts"); 102 } 103 104 static __inline void 105 disable_intr(void) 106 { 107 108 __asm __volatile("cli" : : : "memory"); 109 } 110 111 static __inline void 112 do_cpuid(u_int ax, u_int *p) 113 { 114 __asm __volatile("cpuid" 115 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 116 : "0" (ax)); 117 } 118 119 static __inline void 120 cpuid_count(u_int ax, u_int cx, u_int *p) 121 { 122 __asm __volatile("cpuid" 123 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 124 : "0" (ax), "c" (cx)); 125 } 126 127 static __inline void 128 enable_intr(void) 129 { 130 131 __asm __volatile("sti"); 132 } 133 134 static __inline void 135 cpu_monitor(const void *addr, u_long extensions, u_int hints) 136 { 137 138 __asm __volatile("monitor" 139 : : "a" (addr), "c" (extensions), "d" (hints)); 140 } 141 142 static __inline void 143 cpu_mwait(u_long extensions, u_int hints) 144 { 145 146 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 147 } 148 149 static __inline void 150 lfence(void) 151 { 152 153 __asm __volatile("lfence" : : : "memory"); 154 } 155 156 static __inline void 157 mfence(void) 158 { 159 160 __asm __volatile("mfence" : : : "memory"); 161 } 162 163 static __inline void 164 sfence(void) 165 { 166 167 __asm __volatile("sfence" : : : "memory"); 168 } 169 170 #ifdef _KERNEL 171 172 #define HAVE_INLINE_FFS 173 174 static __inline __pure2 int 175 ffs(int mask) 176 { 177 /* 178 * Note that gcc-2's builtin ffs would be used if we didn't declare 179 * this inline or turn off the builtin. The builtin is faster but 180 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 181 * versions. 182 */ 183 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 184 } 185 186 #define HAVE_INLINE_FFSL 187 188 static __inline __pure2 int 189 ffsl(long mask) 190 { 191 return (ffs((int)mask)); 192 } 193 194 #define HAVE_INLINE_FLS 195 196 static __inline __pure2 int 197 fls(int mask) 198 { 199 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 200 } 201 202 #define HAVE_INLINE_FLSL 203 204 static __inline __pure2 int 205 flsl(long mask) 206 { 207 return (fls((int)mask)); 208 } 209 210 #endif /* _KERNEL */ 211 212 static __inline void 213 halt(void) 214 { 215 __asm __volatile("hlt"); 216 } 217 218 static __inline u_char 219 inb(u_int port) 220 { 221 u_char data; 222 223 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 224 return (data); 225 } 226 227 static __inline u_int 228 inl(u_int port) 229 { 230 u_int data; 231 232 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 233 return (data); 234 } 235 236 static __inline void 237 insb(u_int port, void *addr, size_t count) 238 { 239 __asm __volatile("cld; rep; insb" 240 : "+D" (addr), "+c" (count) 241 : "d" (port) 242 : "memory"); 243 } 244 245 static __inline void 246 insw(u_int port, void *addr, size_t count) 247 { 248 __asm __volatile("cld; rep; insw" 249 : "+D" (addr), "+c" (count) 250 : "d" (port) 251 : "memory"); 252 } 253 254 static __inline void 255 insl(u_int port, void *addr, size_t count) 256 { 257 __asm __volatile("cld; rep; insl" 258 : "+D" (addr), "+c" (count) 259 : "d" (port) 260 : "memory"); 261 } 262 263 static __inline void 264 invd(void) 265 { 266 __asm __volatile("invd"); 267 } 268 269 static __inline u_short 270 inw(u_int port) 271 { 272 u_short data; 273 274 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 275 return (data); 276 } 277 278 static __inline void 279 outb(u_int port, u_char data) 280 { 281 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 282 } 283 284 static __inline void 285 outl(u_int port, u_int data) 286 { 287 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 288 } 289 290 static __inline void 291 outsb(u_int port, const void *addr, size_t count) 292 { 293 __asm __volatile("cld; rep; outsb" 294 : "+S" (addr), "+c" (count) 295 : "d" (port)); 296 } 297 298 static __inline void 299 outsw(u_int port, const void *addr, size_t count) 300 { 301 __asm __volatile("cld; rep; outsw" 302 : "+S" (addr), "+c" (count) 303 : "d" (port)); 304 } 305 306 static __inline void 307 outsl(u_int port, const void *addr, size_t count) 308 { 309 __asm __volatile("cld; rep; outsl" 310 : "+S" (addr), "+c" (count) 311 : "d" (port)); 312 } 313 314 static __inline void 315 outw(u_int port, u_short data) 316 { 317 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 318 } 319 320 static __inline void 321 ia32_pause(void) 322 { 323 __asm __volatile("pause"); 324 } 325 326 static __inline u_int 327 read_eflags(void) 328 { 329 u_int ef; 330 331 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 332 return (ef); 333 } 334 335 static __inline uint64_t 336 rdmsr(u_int msr) 337 { 338 uint64_t rv; 339 340 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 341 return (rv); 342 } 343 344 static __inline uint32_t 345 rdmsr32(u_int msr) 346 { 347 uint32_t low; 348 349 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 350 return (low); 351 } 352 353 static __inline uint64_t 354 rdpmc(u_int pmc) 355 { 356 uint64_t rv; 357 358 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 359 return (rv); 360 } 361 362 static __inline uint64_t 363 rdtsc(void) 364 { 365 uint64_t rv; 366 367 __asm __volatile("rdtsc" : "=A" (rv)); 368 return (rv); 369 } 370 371 static __inline uint32_t 372 rdtsc32(void) 373 { 374 uint32_t rv; 375 376 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 377 return (rv); 378 } 379 380 static __inline void 381 wbinvd(void) 382 { 383 __asm __volatile("wbinvd"); 384 } 385 386 static __inline void 387 write_eflags(u_int ef) 388 { 389 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 390 } 391 392 static __inline void 393 wrmsr(u_int msr, uint64_t newval) 394 { 395 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 396 } 397 398 static __inline void 399 load_cr0(u_int data) 400 { 401 402 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 403 } 404 405 static __inline u_int 406 rcr0(void) 407 { 408 u_int data; 409 410 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 411 return (data); 412 } 413 414 static __inline u_int 415 rcr2(void) 416 { 417 u_int data; 418 419 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 420 return (data); 421 } 422 423 static __inline void 424 load_cr3(u_int data) 425 { 426 427 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 428 } 429 430 static __inline u_int 431 rcr3(void) 432 { 433 u_int data; 434 435 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 436 return (data); 437 } 438 439 static __inline void 440 load_cr4(u_int data) 441 { 442 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 443 } 444 445 static __inline u_int 446 rcr4(void) 447 { 448 u_int data; 449 450 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 451 return (data); 452 } 453 454 static __inline uint64_t 455 rxcr(u_int reg) 456 { 457 u_int low, high; 458 459 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 460 return (low | ((uint64_t)high << 32)); 461 } 462 463 static __inline void 464 load_xcr(u_int reg, uint64_t val) 465 { 466 u_int low, high; 467 468 low = val; 469 high = val >> 32; 470 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 471 } 472 473 /* 474 * Global TLB flush (except for thise for pages marked PG_G) 475 */ 476 static __inline void 477 invltlb(void) 478 { 479 480 load_cr3(rcr3()); 481 } 482 483 /* 484 * TLB flush for an individual page (even if it has PG_G). 485 * Only works on 486+ CPUs (i386 does not have PG_G). 486 */ 487 static __inline void 488 invlpg(u_int addr) 489 { 490 491 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 492 } 493 494 static __inline u_short 495 rfs(void) 496 { 497 u_short sel; 498 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 499 return (sel); 500 } 501 502 static __inline uint64_t 503 rgdt(void) 504 { 505 uint64_t gdtr; 506 __asm __volatile("sgdt %0" : "=m" (gdtr)); 507 return (gdtr); 508 } 509 510 static __inline u_short 511 rgs(void) 512 { 513 u_short sel; 514 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 515 return (sel); 516 } 517 518 static __inline uint64_t 519 ridt(void) 520 { 521 uint64_t idtr; 522 __asm __volatile("sidt %0" : "=m" (idtr)); 523 return (idtr); 524 } 525 526 static __inline u_short 527 rldt(void) 528 { 529 u_short ldtr; 530 __asm __volatile("sldt %0" : "=g" (ldtr)); 531 return (ldtr); 532 } 533 534 static __inline u_short 535 rss(void) 536 { 537 u_short sel; 538 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 539 return (sel); 540 } 541 542 static __inline u_short 543 rtr(void) 544 { 545 u_short tr; 546 __asm __volatile("str %0" : "=g" (tr)); 547 return (tr); 548 } 549 550 static __inline void 551 load_fs(u_short sel) 552 { 553 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 554 } 555 556 static __inline void 557 load_gs(u_short sel) 558 { 559 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 560 } 561 562 static __inline void 563 lidt(struct region_descriptor *addr) 564 { 565 __asm __volatile("lidt (%0)" : : "r" (addr)); 566 } 567 568 static __inline void 569 lldt(u_short sel) 570 { 571 __asm __volatile("lldt %0" : : "r" (sel)); 572 } 573 574 static __inline void 575 ltr(u_short sel) 576 { 577 __asm __volatile("ltr %0" : : "r" (sel)); 578 } 579 580 static __inline u_int 581 rdr0(void) 582 { 583 u_int data; 584 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 585 return (data); 586 } 587 588 static __inline void 589 load_dr0(u_int dr0) 590 { 591 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 592 } 593 594 static __inline u_int 595 rdr1(void) 596 { 597 u_int data; 598 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 599 return (data); 600 } 601 602 static __inline void 603 load_dr1(u_int dr1) 604 { 605 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 606 } 607 608 static __inline u_int 609 rdr2(void) 610 { 611 u_int data; 612 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 613 return (data); 614 } 615 616 static __inline void 617 load_dr2(u_int dr2) 618 { 619 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 620 } 621 622 static __inline u_int 623 rdr3(void) 624 { 625 u_int data; 626 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 627 return (data); 628 } 629 630 static __inline void 631 load_dr3(u_int dr3) 632 { 633 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 634 } 635 636 static __inline u_int 637 rdr6(void) 638 { 639 u_int data; 640 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 641 return (data); 642 } 643 644 static __inline void 645 load_dr6(u_int dr6) 646 { 647 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 648 } 649 650 static __inline u_int 651 rdr7(void) 652 { 653 u_int data; 654 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 655 return (data); 656 } 657 658 static __inline void 659 load_dr7(u_int dr7) 660 { 661 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 662 } 663 664 static __inline u_char 665 read_cyrix_reg(u_char reg) 666 { 667 outb(0x22, reg); 668 return inb(0x23); 669 } 670 671 static __inline void 672 write_cyrix_reg(u_char reg, u_char data) 673 { 674 outb(0x22, reg); 675 outb(0x23, data); 676 } 677 678 static __inline register_t 679 intr_disable(void) 680 { 681 register_t eflags; 682 683 eflags = read_eflags(); 684 disable_intr(); 685 return (eflags); 686 } 687 688 static __inline void 689 intr_restore(register_t eflags) 690 { 691 write_eflags(eflags); 692 } 693 694 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 695 696 int breakpoint(void); 697 u_int bsfl(u_int mask); 698 u_int bsrl(u_int mask); 699 void clflush(u_long addr); 700 void clts(void); 701 void cpuid_count(u_int ax, u_int cx, u_int *p); 702 void disable_intr(void); 703 void do_cpuid(u_int ax, u_int *p); 704 void enable_intr(void); 705 void halt(void); 706 void ia32_pause(void); 707 u_char inb(u_int port); 708 u_int inl(u_int port); 709 void insb(u_int port, void *addr, size_t count); 710 void insl(u_int port, void *addr, size_t count); 711 void insw(u_int port, void *addr, size_t count); 712 register_t intr_disable(void); 713 void intr_restore(register_t ef); 714 void invd(void); 715 void invlpg(u_int addr); 716 void invltlb(void); 717 u_short inw(u_int port); 718 void lidt(struct region_descriptor *addr); 719 void lldt(u_short sel); 720 void load_cr0(u_int cr0); 721 void load_cr3(u_int cr3); 722 void load_cr4(u_int cr4); 723 void load_dr0(u_int dr0); 724 void load_dr1(u_int dr1); 725 void load_dr2(u_int dr2); 726 void load_dr3(u_int dr3); 727 void load_dr6(u_int dr6); 728 void load_dr7(u_int dr7); 729 void load_fs(u_short sel); 730 void load_gs(u_short sel); 731 void ltr(u_short sel); 732 void outb(u_int port, u_char data); 733 void outl(u_int port, u_int data); 734 void outsb(u_int port, const void *addr, size_t count); 735 void outsl(u_int port, const void *addr, size_t count); 736 void outsw(u_int port, const void *addr, size_t count); 737 void outw(u_int port, u_short data); 738 u_int rcr0(void); 739 u_int rcr2(void); 740 u_int rcr3(void); 741 u_int rcr4(void); 742 uint64_t rdmsr(u_int msr); 743 uint64_t rdpmc(u_int pmc); 744 u_int rdr0(void); 745 u_int rdr1(void); 746 u_int rdr2(void); 747 u_int rdr3(void); 748 u_int rdr6(void); 749 u_int rdr7(void); 750 uint64_t rdtsc(void); 751 u_char read_cyrix_reg(u_char reg); 752 u_int read_eflags(void); 753 u_int rfs(void); 754 uint64_t rgdt(void); 755 u_int rgs(void); 756 uint64_t ridt(void); 757 u_short rldt(void); 758 u_short rtr(void); 759 void wbinvd(void); 760 void write_cyrix_reg(u_char reg, u_char data); 761 void write_eflags(u_int ef); 762 void wrmsr(u_int msr, uint64_t newval); 763 764 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 765 766 void reset_dbregs(void); 767 768 #ifdef _KERNEL 769 int rdmsr_safe(u_int msr, uint64_t *val); 770 int wrmsr_safe(u_int msr, uint64_t newval); 771 #endif 772 773 #endif /* !_MACHINE_CPUFUNC_H_ */ 774