1/* $NetBSD: cpufunc.S,v 1.25 2014/02/12 23:24:09 dsl Exp $ */ 2 3/*- 4 * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Charles M. Hannum, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Functions to provide access to i386-specific instructions. 34 */ 35 36#include <sys/errno.h> 37 38#include <machine/asm.h> 39#include <machine/frameasm.h> 40#include <machine/specialreg.h> 41#include <machine/segments.h> 42 43#include "opt_xen.h" 44 45#include "assym.h" 46 47/* Small and slow, so align less. */ 48#undef _ALIGN_TEXT 49#define _ALIGN_TEXT .align 8 50 51ENTRY(x86_lfence) 52 lfence 53 ret 54 55ENTRY(x86_sfence) 56 sfence 57 ret 58 59ENTRY(x86_mfence) 60 mfence 61 ret 62 63#ifndef XEN 64ENTRY(invlpg) 65 invlpg (%rdi) 66 ret 67 68ENTRY(lidt) 69 lidt (%rdi) 70 ret 71 72ENTRY(lldt) 73 cmpl %edi, CPUVAR(CURLDT) 74 jne 1f 75 ret 761: 77 movl %edi, CPUVAR(CURLDT) 78 lldt %di 79 ret 80 81ENTRY(ltr) 82 ltr %di 83 ret 84 85ENTRY(lcr0) 86 movq %rdi, %cr0 87 ret 88 89ENTRY(rcr0) 90 movq %cr0, %rax 91 ret 92 93ENTRY(lcr2) 94 movq %rdi, %cr2 95 ret 96 97ENTRY(rcr2) 98 movq %cr2, %rax 99 ret 100 101ENTRY(lcr3) 102 movq %rdi, %cr3 103 ret 104 105ENTRY(rcr3) 106 movq %cr3, %rax 107 ret 108#endif 109 110ENTRY(lcr4) 111 movq %rdi, %cr4 112 ret 113 114ENTRY(rcr4) 115 movq %cr4, %rax 116 ret 117 118ENTRY(lcr8) 119 movq %rdi, %cr8 120 ret 121 122ENTRY(rcr8) 123 movq %cr8, %rax 124 ret 125 126/* 127 * Big hammer: flush all TLB entries, including ones from PTE's 128 * with the G bit set. This should only be necessary if TLB 129 * shootdown falls far behind. 130 * 131 * Intel Architecture Software Developer's Manual, Volume 3, 132 * System Programming, section 9.10, "Invalidating the 133 * Translation Lookaside Buffers (TLBS)": 134 * "The following operations invalidate all TLB entries, irrespective 135 * of the setting of the G flag: 136 * ... 137 * "(P6 family processors only): Writing to control register CR4 to 138 * modify the PSE, PGE, or PAE flag." 139 * 140 * (the alternatives not quoted above are not an option here.) 141 * 142 * If PGE is not in use, we reload CR3. 143 */ 144#ifndef XEN 145ENTRY(tlbflushg) 146 movq %cr4, %rax 147 testq $CR4_PGE, %rax 148 jz 1f 149 movq %rax, %rdx 150 andq $~CR4_PGE, %rdx 151 movq %rdx, %cr4 152 movq %rax, %cr4 153 ret 154 155ENTRY(tlbflush) 1561: 157 movq %cr3, %rax 158 movq %rax, %cr3 159 ret 160 161ENTRY(ldr6) 162 movq %rdi, %dr6 163 ret 164 165ENTRY(rdr6) 166 movq %dr6, %rdi 167 ret 168 169ENTRY(x86_disable_intr) 170 cli 171 ret 172 173ENTRY(x86_enable_intr) 174 sti 175 ret 176 177ENTRY(x86_read_flags) 178 pushfq 179 popq %rax 180 ret 181 182STRONG_ALIAS(x86_read_psl,x86_read_flags) 183 184ENTRY(x86_write_flags) 185 pushq %rdi 186 popfq 187 ret 188 189STRONG_ALIAS(x86_write_psl,x86_write_flags) 190#endif /* XEN */ 191 192ENTRY(rdmsr) 193 movq %rdi, %rcx 194 xorq %rax, %rax 195 rdmsr 196 shlq $32, %rdx 197 orq %rdx, %rax 198 ret 199 200ENTRY(wrmsr) 201 movq %rdi, %rcx 202 movq %rsi, %rax 203 movq %rsi, %rdx 204 shrq $32, %rdx 205 wrmsr 206 ret 207 208ENTRY(rdmsr_locked) 209 movq %rdi, %rcx 210 xorq %rax, %rax 211 movl $OPTERON_MSR_PASSCODE, %edi 212 rdmsr 213 shlq $32, %rdx 214 orq %rdx, %rax 215 ret 216 217ENTRY(wrmsr_locked) 218 movq %rdi, %rcx 219 movq %rsi, %rax 220 movq %rsi, %rdx 221 shrq $32, %rdx 222 movl $OPTERON_MSR_PASSCODE, %edi 223 wrmsr 224 ret 225 226/* 227 * Support for reading MSRs in the safe manner (returns EFAULT on fault) 228 */ 229/* int rdmsr_safe(u_int msr, uint64_t *data) */ 230ENTRY(rdmsr_safe) 231 movq CPUVAR(CURLWP), %r8 232 movq L_PCB(%r8), %r8 233 movq $_C_LABEL(msr_onfault), PCB_ONFAULT(%r8) 234 235 movl %edi, %ecx /* u_int msr */ 236 rdmsr /* Read MSR pointed by %ecx. Returns 237 hi byte in edx, lo in %eax */ 238 salq $32, %rdx /* sign-shift %rdx left */ 239 movl %eax, %eax /* zero-extend %eax -> %rax */ 240 orq %rdx, %rax 241 movq %rax, (%rsi) /* *data */ 242 xorq %rax, %rax /* "no error" */ 243 244 movq %rax, PCB_ONFAULT(%r8) 245 ret 246 247ENTRY(rdxcr) 248 movq %rdi, %rcx 249 xgetbv 250 shlq $32, %rdx 251 orq %rdx, %rax 252 ret 253 254ENTRY(wrxcr) 255 movq %rdi, %rcx 256 movq %rsi, %rax 257 movq %rsi, %rdx 258 shrq $32, %rdx 259 xsetbv 260 ret 261 262/* 263 * MSR operations fault handler 264 */ 265NENTRY(msr_onfault) 266 movq CPUVAR(CURLWP), %r8 267 movq L_PCB(%r8), %r8 268 movq $0, PCB_ONFAULT(%r8) 269 movl $EFAULT, %eax 270 ret 271 272#ifndef XEN 273ENTRY(wbinvd) 274 wbinvd 275 ret 276#endif 277 278ENTRY(cpu_counter) 279 xorq %rax, %rax 280 rdtsc 281 shlq $32, %rdx 282 orq %rdx, %rax 283 addq CPUVAR(CC_SKEW), %rax 284 ret 285 286ENTRY(cpu_counter32) 287 rdtsc 288 addl CPUVAR(CC_SKEW), %eax 289 ret 290 291ENTRY(rdpmc) 292 movq %rdi, %rcx 293 xorq %rax, %rax 294 rdpmc 295 shlq $32, %rdx 296 orq %rdx, %rax 297 ret 298 299ENTRY(breakpoint) 300 pushq %rbp 301 movq %rsp, %rbp 302 int $0x03 /* paranoid, not 'int3' */ 303 leave 304 ret 305 306ENTRY(x86_curcpu) 307 movq %gs:(CPU_INFO_SELF), %rax 308 ret 309 310ENTRY(x86_curlwp) 311 movq %gs:(CPU_INFO_CURLWP), %rax 312 ret 313 314ENTRY(cpu_set_curpri) 315 movl %edi, %gs:(CPU_INFO_CURPRIORITY) 316 ret 317 318ENTRY(__byte_swap_u32_variable) 319 movl %edi, %eax 320 bswapl %eax 321 ret 322 323ENTRY(__byte_swap_u16_variable) 324 movl %edi, %eax 325 xchgb %al, %ah 326 ret 327 328/* 329 * void lgdt(struct region_descriptor *rdp); 330 * 331 * Load a new GDT pointer (and do any necessary cleanup). 332 * XXX It's somewhat questionable whether reloading all the segment registers 333 * is necessary, since the actual descriptor data is not changed except by 334 * process creation and exit, both of which clean up via task switches. OTOH, 335 * this only happens at run time when the GDT is resized. 336 */ 337#ifndef XEN 338ENTRY(lgdt) 339 /* Reload the descriptor table. */ 340 movq %rdi,%rax 341 lgdt (%rax) 342 /* Flush the prefetch q. */ 343 jmp 1f 344 nop 3451: /* Reload "stale" selectors. */ 346#else /* XEN */ 347/* 348 * void lgdt_finish(void); 349 * Reload segments after a GDT change 350 */ 351ENTRY(lgdt_finish) 352#endif /* XEN */ 353 movl $GSEL(GDATA_SEL, SEL_KPL),%eax 354 movl %eax,%ds 355 movl %eax,%es 356 movl %eax,%ss 357 /* FALLTHROUGH */ 358 359/* 360 * void x86_flush() 361 * 362 * Flush instruction pipelines by doing an intersegment (far) return. 363 */ 364ENTRY(x86_flush) 365 popq %rax 366 pushq $GSEL(GCODE_SEL, SEL_KPL) 367 pushq %rax 368 lretq 369 370/* Waits - set up stack frame. */ 371ENTRY(x86_hlt) 372 pushq %rbp 373 movq %rsp, %rbp 374 hlt 375 leave 376 ret 377 378/* Waits - set up stack frame. */ 379ENTRY(x86_stihlt) 380 pushq %rbp 381 movq %rsp, %rbp 382 sti 383 hlt 384 leave 385 ret 386 387ENTRY(x86_monitor) 388 movq %rdi, %rax 389 movq %rsi, %rcx 390 monitor %rax, %rcx, %rdx 391 ret 392 393/* Waits - set up stack frame. */ 394ENTRY(x86_mwait) 395 pushq %rbp 396 movq %rsp, %rbp 397 movq %rdi, %rax 398 movq %rsi, %rcx 399 mwait %rax, %rcx 400 leave 401 ret 402 403NENTRY(x86_pause) 404 pause 405 ret 406 407ENTRY(x86_cpuid2) 408 movq %rbx, %r8 409 movq %rdi, %rax 410 movq %rsi, %rcx 411 movq %rdx, %rsi 412 cpuid 413 movl %eax, 0(%rsi) 414 movl %ebx, 4(%rsi) 415 movl %ecx, 8(%rsi) 416 movl %edx, 12(%rsi) 417 movq %r8, %rbx 418 ret 419 420ENTRY(x86_getss) 421 movl %ss, %eax 422 ret 423 424ENTRY(fldcw) 425 fldcw (%rdi) 426 ret 427 428ENTRY(fnclex) 429 fnclex 430 ret 431 432ENTRY(fninit) 433 fninit 434 ret 435 436ENTRY(fnsave) 437 fnsave (%rdi) 438 ret 439 440ENTRY(fnstcw) 441 fnstcw (%rdi) 442 ret 443 444ENTRY(fngetsw) 445 fnstsw %ax 446 ret 447 448ENTRY(fnstsw) 449 fnstsw (%rdi) 450 ret 451 452ENTRY(fp_divide_by_0) 453 fldz 454 fld1 455 fdiv %st, %st(1) 456 fwait 457 ret 458 459ENTRY(frstor) 460 frstor (%rdi) 461 ret 462 463ENTRY(fwait) 464 fwait 465 ret 466 467ENTRY(clts) 468 clts 469 ret 470 471ENTRY(stts) 472 movq %cr0, %rax 473 orq $CR0_TS, %rax 474 movq %rax, %cr0 475 ret 476 477ENTRY(fxsave) 478 fxsave (%rdi) 479 ret 480 481ENTRY(fxrstor) 482 fxrstor (%rdi) 483 ret 484 485ENTRY(fldummy) 486 ffree %st(7) 487 fldz 488 ret 489 490ENTRY(xsave) 491 movq %rsi, %rax 492 movq %rsi, %rdx 493 shrq $32, %rdx 494 xsave (%rdi) 495 ret 496 497ENTRY(xsaveopt) 498 movq %rsi, %rax 499 movq %rsi, %rdx 500 shrq $32, %rdx 501 xsaveopt (%rdi) 502 ret 503 504ENTRY(xrstor) 505 movq %rsi, %rax 506 movq %rsi, %rdx 507 shrq $32, %rdx 508 xrstor (%rdi) 509 ret 510 511ENTRY(x86_stmxcsr) 512 stmxcsr (%rdi) 513 ret 514 515ENTRY(x86_ldmxcsr) 516 ldmxcsr (%rdi) 517 ret 518 519ENTRY(inb) 520 movq %rdi, %rdx 521 xorq %rax, %rax 522 inb %dx, %al 523 ret 524 525ENTRY(insb) 526 movl %edx, %ecx 527 movl %edi, %edx 528 movq %rsi, %rdi 529 rep 530 insb 531 ret 532 533ENTRY(inw) 534 movq %rdi, %rdx 535 xorq %rax, %rax 536 inw %dx, %ax 537 ret 538 539ENTRY(insw) 540 movl %edx, %ecx 541 movl %edi, %edx 542 movq %rsi, %rdi 543 rep 544 insw 545 ret 546 547ENTRY(inl) 548 movq %rdi, %rdx 549 xorq %rax, %rax 550 inl %dx, %eax 551 ret 552 553ENTRY(insl) 554 movl %edx, %ecx 555 movl %edi, %edx 556 movq %rsi, %rdi 557 rep 558 insl 559 ret 560 561ENTRY(outb) 562 movq %rdi, %rdx 563 movq %rsi, %rax 564 outb %al, %dx 565 ret 566 567ENTRY(outsb) 568 movl %edx, %ecx 569 movl %edi, %edx 570 rep 571 outsb 572 ret 573 574ENTRY(outw) 575 movq %rdi, %rdx 576 movq %rsi, %rax 577 outw %ax, %dx 578 ret 579 580ENTRY(outsw) 581 movl %edx, %ecx 582 movl %edi, %edx 583 rep 584 outsw 585 ret 586 587ENTRY(outl) 588 movq %rdi, %rdx 589 movq %rsi, %rax 590 outl %eax, %dx 591 ret 592 593ENTRY(outsl) 594 movl %edx, %ecx 595 movl %edi, %edx 596 rep 597 outsl 598 ret 599 600ENTRY(setfs) 601 movw %di, %fs 602 ret 603 604ENTRY(setusergs) 605 CLI(ax) 606 swapgs 607 movw %di, %gs 608 swapgs 609 STI(ax) 610 ret 611