1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * Copyright (c) 2007 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * Portions of this software were developed by A. Joseph Koshy under 8 * sponsorship from the FreeBSD Foundation and Google, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD$ 35 */ 36 37#include "opt_atpic.h" 38#include "opt_compat.h" 39#include "opt_hwpmc_hooks.h" 40 41#include <machine/asmacros.h> 42#include <machine/psl.h> 43#include <machine/trap.h> 44#include <machine/specialreg.h> 45 46#include "assym.s" 47 48#ifdef KDTRACE_HOOKS 49 .bss 50 .globl dtrace_invop_jump_addr 51 .align 8 52 .type dtrace_invop_jump_addr,@object 53 .size dtrace_invop_jump_addr,8 54dtrace_invop_jump_addr: 55 .zero 8 56 .globl dtrace_invop_calltrap_addr 57 .align 8 58 .type dtrace_invop_calltrap_addr,@object 59 .size dtrace_invop_calltrap_addr,8 60dtrace_invop_calltrap_addr: 61 .zero 8 62#endif 63 .text 64#ifdef HWPMC_HOOKS 65 ENTRY(start_exceptions) 66#endif 67 68/*****************************************************************************/ 69/* Trap handling */ 70/*****************************************************************************/ 71/* 72 * Trap and fault vector routines. 73 * 74 * All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes 75 * state on the stack but also disables interrupts. This is important for 76 * us for the use of the swapgs instruction. We cannot be interrupted 77 * until the GS.base value is correct. For most traps, we automatically 78 * then enable interrupts if the interrupted context had them enabled. 79 * This is equivalent to the i386 port's use of SDT_SYS386TGT. 80 * 81 * The cpu will push a certain amount of state onto the kernel stack for 82 * the current process. See amd64/include/frame.h. 83 * This includes the current RFLAGS (status register, which includes 84 * the interrupt disable state prior to the trap), the code segment register, 85 * and the return instruction pointer are pushed by the cpu. The cpu 86 * will also push an 'error' code for certain traps. We push a dummy 87 * error code for those traps where the cpu doesn't in order to maintain 88 * a consistent frame. We also push a contrived 'trap number'. 89 * 90 * The CPU does not push the general registers, so we must do that, and we 91 * must restore them prior to calling 'iret'. The CPU adjusts %cs and %ss 92 * but does not mess with %ds, %es, %gs or %fs. We swap the %gs base for 93 * for the kernel mode operation shortly, without changes to the selector 94 * loaded. Since superuser long mode works with any selectors loaded into 95 * segment registers other then %cs, which makes them mostly unused in long 96 * mode, and kernel does not reference %fs, leave them alone. The segment 97 * registers are reloaded on return to the usermode. 98 */ 99 100MCOUNT_LABEL(user) 101MCOUNT_LABEL(btrap) 102 103/* Traps that we leave interrupts disabled for.. */ 104#define TRAP_NOEN(a) \ 105 subq $TF_RIP,%rsp; \ 106 movl $(a),TF_TRAPNO(%rsp) ; \ 107 movq $0,TF_ADDR(%rsp) ; \ 108 movq $0,TF_ERR(%rsp) ; \ 109 jmp alltraps_noen 110IDTVEC(dbg) 111 TRAP_NOEN(T_TRCTRAP) 112IDTVEC(bpt) 113 TRAP_NOEN(T_BPTFLT) 114#ifdef KDTRACE_HOOKS 115IDTVEC(dtrace_ret) 116 TRAP_NOEN(T_DTRACE_RET) 117#endif 118 119/* Regular traps; The cpu does not supply tf_err for these. */ 120#define TRAP(a) \ 121 subq $TF_RIP,%rsp; \ 122 movl $(a),TF_TRAPNO(%rsp) ; \ 123 movq $0,TF_ADDR(%rsp) ; \ 124 movq $0,TF_ERR(%rsp) ; \ 125 jmp alltraps 126IDTVEC(div) 127 TRAP(T_DIVIDE) 128IDTVEC(ofl) 129 TRAP(T_OFLOW) 130IDTVEC(bnd) 131 TRAP(T_BOUND) 132IDTVEC(ill) 133 TRAP(T_PRIVINFLT) 134IDTVEC(dna) 135 TRAP(T_DNA) 136IDTVEC(fpusegm) 137 TRAP(T_FPOPFLT) 138IDTVEC(mchk) 139 TRAP(T_MCHK) 140IDTVEC(rsvd) 141 TRAP(T_RESERVED) 142IDTVEC(fpu) 143 TRAP(T_ARITHTRAP) 144IDTVEC(xmm) 145 TRAP(T_XMMFLT) 146 147/* This group of traps have tf_err already pushed by the cpu */ 148#define TRAP_ERR(a) \ 149 subq $TF_ERR,%rsp; \ 150 movl $(a),TF_TRAPNO(%rsp) ; \ 151 movq $0,TF_ADDR(%rsp) ; \ 152 jmp alltraps 153IDTVEC(tss) 154 TRAP_ERR(T_TSSFLT) 155IDTVEC(missing) 156 subq $TF_ERR,%rsp 157 movl $T_SEGNPFLT,TF_TRAPNO(%rsp) 158 jmp prot_addrf 159IDTVEC(stk) 160 subq $TF_ERR,%rsp 161 movl $T_STKFLT,TF_TRAPNO(%rsp) 162 jmp prot_addrf 163IDTVEC(align) 164 TRAP_ERR(T_ALIGNFLT) 165 166 /* 167 * alltraps entry point. Use swapgs if this is the first time in the 168 * kernel from userland. Reenable interrupts if they were enabled 169 * before the trap. This approximates SDT_SYS386TGT on the i386 port. 170 */ 171 SUPERALIGN_TEXT 172 .globl alltraps 173 .type alltraps,@function 174alltraps: 175 movq %rdi,TF_RDI(%rsp) 176 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 177 jz alltraps_testi /* already running with kernel GS.base */ 178 swapgs 179 movq PCPU(CURPCB),%rdi 180 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 181 movw %fs,TF_FS(%rsp) 182 movw %gs,TF_GS(%rsp) 183 movw %es,TF_ES(%rsp) 184 movw %ds,TF_DS(%rsp) 185alltraps_testi: 186 testl $PSL_I,TF_RFLAGS(%rsp) 187 jz alltraps_pushregs_no_rdi 188 sti 189alltraps_pushregs_no_rdi: 190 movq %rsi,TF_RSI(%rsp) 191 movq %rdx,TF_RDX(%rsp) 192 movq %rcx,TF_RCX(%rsp) 193 movq %r8,TF_R8(%rsp) 194 movq %r9,TF_R9(%rsp) 195 movq %rax,TF_RAX(%rsp) 196 movq %rbx,TF_RBX(%rsp) 197 movq %rbp,TF_RBP(%rsp) 198 movq %r10,TF_R10(%rsp) 199 movq %r11,TF_R11(%rsp) 200 movq %r12,TF_R12(%rsp) 201 movq %r13,TF_R13(%rsp) 202 movq %r14,TF_R14(%rsp) 203 movq %r15,TF_R15(%rsp) 204 movl $TF_HASSEGS,TF_FLAGS(%rsp) 205 cld 206 FAKE_MCOUNT(TF_RIP(%rsp)) 207#ifdef KDTRACE_HOOKS 208 /* 209 * DTrace Function Boundary Trace (fbt) probes are triggered 210 * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint 211 * interrupt. For all other trap types, just handle them in 212 * the usual way. 213 */ 214 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 215 jnz calltrap /* ignore userland traps */ 216 cmpl $T_BPTFLT,TF_TRAPNO(%rsp) 217 jne calltrap 218 219 /* Check if there is no DTrace hook registered. */ 220 cmpq $0,dtrace_invop_jump_addr 221 je calltrap 222 223 /* 224 * Set our jump address for the jump back in the event that 225 * the breakpoint wasn't caused by DTrace at all. 226 */ 227 movq $calltrap,dtrace_invop_calltrap_addr(%rip) 228 229 /* Jump to the code hooked in by DTrace. */ 230 jmpq *dtrace_invop_jump_addr 231#endif 232 .globl calltrap 233 .type calltrap,@function 234calltrap: 235 movq %rsp,%rdi 236 call trap_check 237 MEXITCOUNT 238 jmp doreti /* Handle any pending ASTs */ 239 240 /* 241 * alltraps_noen entry point. Unlike alltraps above, we want to 242 * leave the interrupts disabled. This corresponds to 243 * SDT_SYS386IGT on the i386 port. 244 */ 245 SUPERALIGN_TEXT 246 .globl alltraps_noen 247 .type alltraps_noen,@function 248alltraps_noen: 249 movq %rdi,TF_RDI(%rsp) 250 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 251 jz 1f /* already running with kernel GS.base */ 252 swapgs 253 movq PCPU(CURPCB),%rdi 254 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 2551: movw %fs,TF_FS(%rsp) 256 movw %gs,TF_GS(%rsp) 257 movw %es,TF_ES(%rsp) 258 movw %ds,TF_DS(%rsp) 259 jmp alltraps_pushregs_no_rdi 260 261IDTVEC(dblfault) 262 subq $TF_ERR,%rsp 263 movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) 264 movq $0,TF_ADDR(%rsp) 265 movq $0,TF_ERR(%rsp) 266 movq %rdi,TF_RDI(%rsp) 267 movq %rsi,TF_RSI(%rsp) 268 movq %rdx,TF_RDX(%rsp) 269 movq %rcx,TF_RCX(%rsp) 270 movq %r8,TF_R8(%rsp) 271 movq %r9,TF_R9(%rsp) 272 movq %rax,TF_RAX(%rsp) 273 movq %rbx,TF_RBX(%rsp) 274 movq %rbp,TF_RBP(%rsp) 275 movq %r10,TF_R10(%rsp) 276 movq %r11,TF_R11(%rsp) 277 movq %r12,TF_R12(%rsp) 278 movq %r13,TF_R13(%rsp) 279 movq %r14,TF_R14(%rsp) 280 movq %r15,TF_R15(%rsp) 281 movw %fs,TF_FS(%rsp) 282 movw %gs,TF_GS(%rsp) 283 movw %es,TF_ES(%rsp) 284 movw %ds,TF_DS(%rsp) 285 movl $TF_HASSEGS,TF_FLAGS(%rsp) 286 cld 287 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 288 jz 1f /* already running with kernel GS.base */ 289 swapgs 2901: 291 movq %rsp,%rdi 292 call dblfault_handler 2932: 294 hlt 295 jmp 2b 296 297IDTVEC(page) 298 subq $TF_ERR,%rsp 299 movl $T_PAGEFLT,TF_TRAPNO(%rsp) 300 movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 301 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 302 jz 1f /* already running with kernel GS.base */ 303 swapgs 304 movq PCPU(CURPCB),%rdi 305 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 3061: movq %cr2,%rdi /* preserve %cr2 before .. */ 307 movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ 308 movw %fs,TF_FS(%rsp) 309 movw %gs,TF_GS(%rsp) 310 movw %es,TF_ES(%rsp) 311 movw %ds,TF_DS(%rsp) 312 testl $PSL_I,TF_RFLAGS(%rsp) 313 jz alltraps_pushregs_no_rdi 314 sti 315 jmp alltraps_pushregs_no_rdi 316 317 /* 318 * We have to special-case this one. If we get a trap in doreti() at 319 * the iretq stage, we'll reenter with the wrong gs state. We'll have 320 * to do a special the swapgs in this case even coming from the kernel. 321 * XXX linux has a trap handler for their equivalent of load_gs(). 322 */ 323IDTVEC(prot) 324 subq $TF_ERR,%rsp 325 movl $T_PROTFLT,TF_TRAPNO(%rsp) 326prot_addrf: 327 movq $0,TF_ADDR(%rsp) 328 movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 329 leaq doreti_iret(%rip),%rdi 330 cmpq %rdi,TF_RIP(%rsp) 331 je 1f /* kernel but with user gsbase!! */ 332 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 333 jz 2f /* already running with kernel GS.base */ 3341: swapgs 3352: movq PCPU(CURPCB),%rdi 336 orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) /* always full iret from GPF */ 337 movw %fs,TF_FS(%rsp) 338 movw %gs,TF_GS(%rsp) 339 movw %es,TF_ES(%rsp) 340 movw %ds,TF_DS(%rsp) 341 testl $PSL_I,TF_RFLAGS(%rsp) 342 jz alltraps_pushregs_no_rdi 343 sti 344 jmp alltraps_pushregs_no_rdi 345 346/* 347 * Fast syscall entry point. We enter here with just our new %cs/%ss set, 348 * and the new privilige level. We are still running on the old user stack 349 * pointer. We have to juggle a few things around to find our stack etc. 350 * swapgs gives us access to our PCPU space only. 351 * 352 * We do not support invoking this from a custom %cs or %ss (e.g. using 353 * entries from an LDT). 354 */ 355IDTVEC(fast_syscall) 356 swapgs 357 movq %rsp,PCPU(SCRATCH_RSP) 358 movq PCPU(RSP0),%rsp 359 /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ 360 subq $TF_SIZE,%rsp 361 /* defer TF_RSP till we have a spare register */ 362 movq %r11,TF_RFLAGS(%rsp) 363 movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ 364 movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ 365 movq %r11,TF_RSP(%rsp) /* user stack pointer */ 366 movw %fs,TF_FS(%rsp) 367 movw %gs,TF_GS(%rsp) 368 movw %es,TF_ES(%rsp) 369 movw %ds,TF_DS(%rsp) 370 movq PCPU(CURPCB),%r11 371 andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) 372 sti 373 movq $KUDSEL,TF_SS(%rsp) 374 movq $KUCSEL,TF_CS(%rsp) 375 movq $2,TF_ERR(%rsp) 376 movq %rdi,TF_RDI(%rsp) /* arg 1 */ 377 movq %rsi,TF_RSI(%rsp) /* arg 2 */ 378 movq %rdx,TF_RDX(%rsp) /* arg 3 */ 379 movq %r10,TF_RCX(%rsp) /* arg 4 */ 380 movq %r8,TF_R8(%rsp) /* arg 5 */ 381 movq %r9,TF_R9(%rsp) /* arg 6 */ 382 movq %rax,TF_RAX(%rsp) /* syscall number */ 383 movq %rbx,TF_RBX(%rsp) /* C preserved */ 384 movq %rbp,TF_RBP(%rsp) /* C preserved */ 385 movq %r12,TF_R12(%rsp) /* C preserved */ 386 movq %r13,TF_R13(%rsp) /* C preserved */ 387 movq %r14,TF_R14(%rsp) /* C preserved */ 388 movq %r15,TF_R15(%rsp) /* C preserved */ 389 movl $TF_HASSEGS,TF_FLAGS(%rsp) 390 cld 391 FAKE_MCOUNT(TF_RIP(%rsp)) 392 movq PCPU(CURTHREAD),%rdi 393 movq %rsp,TD_FRAME(%rdi) 394 movl TF_RFLAGS(%rsp),%esi 395 andl $PSL_T,%esi 396 call amd64_syscall 3971: movq PCPU(CURPCB),%rax 398 /* Disable interrupts before testing PCB_FULL_IRET. */ 399 cli 400 testl $PCB_FULL_IRET,PCB_FLAGS(%rax) 401 jnz 3f 402 /* Check for and handle AST's on return to userland. */ 403 movq PCPU(CURTHREAD),%rax 404 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 405 jne 2f 406 /* Restore preserved registers. */ 407 MEXITCOUNT 408 movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ 409 movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ 410 movq TF_RDX(%rsp),%rdx /* return value 2 */ 411 movq TF_RAX(%rsp),%rax /* return value 1 */ 412 movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ 413 movq TF_RIP(%rsp),%rcx /* original %rip */ 414 movq TF_RSP(%rsp),%rsp /* user stack pointer */ 415 swapgs 416 sysretq 417 4182: /* AST scheduled. */ 419 sti 420 movq %rsp,%rdi 421 call ast 422 jmp 1b 423 4243: /* Requested full context restore, use doreti for that. */ 425 MEXITCOUNT 426 jmp doreti 427 428/* 429 * Here for CYA insurance, in case a "syscall" instruction gets 430 * issued from 32 bit compatibility mode. MSR_CSTAR has to point 431 * to *something* if EFER_SCE is enabled. 432 */ 433IDTVEC(fast_syscall32) 434 sysret 435 436/* 437 * NMI handling is special. 438 * 439 * First, NMIs do not respect the state of the processor's RFLAGS.IF 440 * bit. The NMI handler may be entered at any time, including when 441 * the processor is in a critical section with RFLAGS.IF == 0. 442 * The processor's GS.base value could be invalid on entry to the 443 * handler. 444 * 445 * Second, the processor treats NMIs specially, blocking further NMIs 446 * until an 'iretq' instruction is executed. We thus need to execute 447 * the NMI handler with interrupts disabled, to prevent a nested interrupt 448 * from executing an 'iretq' instruction and inadvertently taking the 449 * processor out of NMI mode. 450 * 451 * Third, the NMI handler runs on its own stack (tss_ist2). The canonical 452 * GS.base value for the processor is stored just above the bottom of its 453 * NMI stack. For NMIs taken from kernel mode, the current value in 454 * the processor's GS.base is saved at entry to C-preserved register %r12, 455 * the canonical value for GS.base is then loaded into the processor, and 456 * the saved value is restored at exit time. For NMIs taken from user mode, 457 * the cheaper 'SWAPGS' instructions are used for swapping GS.base. 458 */ 459 460IDTVEC(nmi) 461 subq $TF_RIP,%rsp 462 movl $(T_NMI),TF_TRAPNO(%rsp) 463 movq $0,TF_ADDR(%rsp) 464 movq $0,TF_ERR(%rsp) 465 movq %rdi,TF_RDI(%rsp) 466 movq %rsi,TF_RSI(%rsp) 467 movq %rdx,TF_RDX(%rsp) 468 movq %rcx,TF_RCX(%rsp) 469 movq %r8,TF_R8(%rsp) 470 movq %r9,TF_R9(%rsp) 471 movq %rax,TF_RAX(%rsp) 472 movq %rbx,TF_RBX(%rsp) 473 movq %rbp,TF_RBP(%rsp) 474 movq %r10,TF_R10(%rsp) 475 movq %r11,TF_R11(%rsp) 476 movq %r12,TF_R12(%rsp) 477 movq %r13,TF_R13(%rsp) 478 movq %r14,TF_R14(%rsp) 479 movq %r15,TF_R15(%rsp) 480 movw %fs,TF_FS(%rsp) 481 movw %gs,TF_GS(%rsp) 482 movw %es,TF_ES(%rsp) 483 movw %ds,TF_DS(%rsp) 484 movl $TF_HASSEGS,TF_FLAGS(%rsp) 485 cld 486 xorl %ebx,%ebx 487 testb $SEL_RPL_MASK,TF_CS(%rsp) 488 jnz nmi_fromuserspace 489 /* 490 * We've interrupted the kernel. Preserve GS.base in %r12. 491 */ 492 movl $MSR_GSBASE,%ecx 493 rdmsr 494 movq %rax,%r12 495 shlq $32,%rdx 496 orq %rdx,%r12 497 /* Retrieve and load the canonical value for GS.base. */ 498 movq TF_SIZE(%rsp),%rdx 499 movl %edx,%eax 500 shrq $32,%rdx 501 wrmsr 502 jmp nmi_calltrap 503nmi_fromuserspace: 504 incl %ebx 505 swapgs 506/* Note: this label is also used by ddb and gdb: */ 507nmi_calltrap: 508 FAKE_MCOUNT(TF_RIP(%rsp)) 509 movq %rsp,%rdi 510 call trap 511 MEXITCOUNT 512#ifdef HWPMC_HOOKS 513 /* 514 * Capture a userspace callchain if needed. 515 * 516 * - Check if the current trap was from user mode. 517 * - Check if the current thread is valid. 518 * - Check if the thread requires a user call chain to be 519 * captured. 520 * 521 * We are still in NMI mode at this point. 522 */ 523 testl %ebx,%ebx 524 jz nocallchain /* not from userspace */ 525 movq PCPU(CURTHREAD),%rax 526 orq %rax,%rax /* curthread present? */ 527 jz nocallchain 528 testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ 529 jz nocallchain 530 /* 531 * A user callchain is to be captured, so: 532 * - Move execution to the regular kernel stack, to allow for 533 * nested NMI interrupts. 534 * - Take the processor out of "NMI" mode by faking an "iret". 535 * - Enable interrupts, so that copyin() can work. 536 */ 537 movq %rsp,%rsi /* source stack pointer */ 538 movq $TF_SIZE,%rcx 539 movq PCPU(RSP0),%rdx 540 subq %rcx,%rdx 541 movq %rdx,%rdi /* destination stack pointer */ 542 543 shrq $3,%rcx /* trap frame size in long words */ 544 cld 545 rep 546 movsq /* copy trapframe */ 547 548 movl %ss,%eax 549 pushq %rax /* tf_ss */ 550 pushq %rdx /* tf_rsp (on kernel stack) */ 551 pushfq /* tf_rflags */ 552 movl %cs,%eax 553 pushq %rax /* tf_cs */ 554 pushq $outofnmi /* tf_rip */ 555 iretq 556outofnmi: 557 /* 558 * At this point the processor has exited NMI mode and is running 559 * with interrupts turned off on the normal kernel stack. 560 * 561 * If a pending NMI gets recognized at or after this point, it 562 * will cause a kernel callchain to be traced. 563 * 564 * We turn interrupts back on, and call the user callchain capture hook. 565 */ 566 movq pmc_hook,%rax 567 orq %rax,%rax 568 jz nocallchain 569 movq PCPU(CURTHREAD),%rdi /* thread */ 570 movq $PMC_FN_USER_CALLCHAIN,%rsi /* command */ 571 movq %rsp,%rdx /* frame */ 572 sti 573 call *%rax 574 cli 575nocallchain: 576#endif 577 testl %ebx,%ebx 578 jnz doreti_exit 579nmi_kernelexit: 580 /* 581 * Put back the preserved MSR_GSBASE value. 582 */ 583 movl $MSR_GSBASE,%ecx 584 movq %r12,%rdx 585 movl %edx,%eax 586 shrq $32,%rdx 587 wrmsr 588nmi_restoreregs: 589 movq TF_RDI(%rsp),%rdi 590 movq TF_RSI(%rsp),%rsi 591 movq TF_RDX(%rsp),%rdx 592 movq TF_RCX(%rsp),%rcx 593 movq TF_R8(%rsp),%r8 594 movq TF_R9(%rsp),%r9 595 movq TF_RAX(%rsp),%rax 596 movq TF_RBX(%rsp),%rbx 597 movq TF_RBP(%rsp),%rbp 598 movq TF_R10(%rsp),%r10 599 movq TF_R11(%rsp),%r11 600 movq TF_R12(%rsp),%r12 601 movq TF_R13(%rsp),%r13 602 movq TF_R14(%rsp),%r14 603 movq TF_R15(%rsp),%r15 604 addq $TF_RIP,%rsp 605 jmp doreti_iret 606 607ENTRY(fork_trampoline) 608 movq %r12,%rdi /* function */ 609 movq %rbx,%rsi /* arg1 */ 610 movq %rsp,%rdx /* trapframe pointer */ 611 call fork_exit 612 MEXITCOUNT 613 jmp doreti /* Handle any ASTs */ 614 615/* 616 * To efficiently implement classification of trap and interrupt handlers 617 * for profiling, there must be only trap handlers between the labels btrap 618 * and bintr, and only interrupt handlers between the labels bintr and 619 * eintr. This is implemented (partly) by including files that contain 620 * some of the handlers. Before including the files, set up a normal asm 621 * environment so that the included files doen't need to know that they are 622 * included. 623 */ 624 625#ifdef COMPAT_FREEBSD32 626 .data 627 .p2align 4 628 .text 629 SUPERALIGN_TEXT 630 631#include <amd64/ia32/ia32_exception.S> 632#endif 633 634 .data 635 .p2align 4 636 .text 637 SUPERALIGN_TEXT 638MCOUNT_LABEL(bintr) 639 640#include <amd64/amd64/apic_vector.S> 641 642#ifdef DEV_ATPIC 643 .data 644 .p2align 4 645 .text 646 SUPERALIGN_TEXT 647 648#include <amd64/amd64/atpic_vector.S> 649#endif 650 651 .text 652MCOUNT_LABEL(eintr) 653 654/* 655 * void doreti(struct trapframe) 656 * 657 * Handle return from interrupts, traps and syscalls. 658 */ 659 .text 660 SUPERALIGN_TEXT 661 .type doreti,@function 662 .globl doreti 663doreti: 664 FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */ 665 /* 666 * Check if ASTs can be handled now. 667 */ 668 testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */ 669 jz doreti_exit /* can't handle ASTs now if not */ 670 671doreti_ast: 672 /* 673 * Check for ASTs atomically with returning. Disabling CPU 674 * interrupts provides sufficient locking even in the SMP case, 675 * since we will be informed of any new ASTs by an IPI. 676 */ 677 cli 678 movq PCPU(CURTHREAD),%rax 679 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 680 je doreti_exit 681 sti 682 movq %rsp,%rdi /* pass a pointer to the trapframe */ 683 call ast 684 jmp doreti_ast 685 686 /* 687 * doreti_exit: pop registers, iret. 688 * 689 * The segment register pop is a special case, since it may 690 * fault if (for example) a sigreturn specifies bad segment 691 * registers. The fault is handled in trap.c. 692 */ 693doreti_exit: 694 MEXITCOUNT 695 movq PCPU(CURPCB),%r8 696 697 /* 698 * Do not reload segment registers for kernel. 699 * Since we do not reload segments registers with sane 700 * values on kernel entry, descriptors referenced by 701 * segments registers might be not valid. This is fatal 702 * for user mode, but is not a problem for the kernel. 703 */ 704 testb $SEL_RPL_MASK,TF_CS(%rsp) 705 jz ld_regs 706 testl $PCB_FULL_IRET,PCB_FLAGS(%r8) 707 jz ld_regs 708 testl $TF_HASSEGS,TF_FLAGS(%rsp) 709 je set_segs 710 711do_segs: 712 /* Restore %fs and fsbase */ 713 movw TF_FS(%rsp),%ax 714 .globl ld_fs 715ld_fs: 716 movw %ax,%fs 717 cmpw $KUF32SEL,%ax 718 jne 1f 719 movl $MSR_FSBASE,%ecx 720 movl PCB_FSBASE(%r8),%eax 721 movl PCB_FSBASE+4(%r8),%edx 722 .globl ld_fsbase 723ld_fsbase: 724 wrmsr 7251: 726 /* Restore %gs and gsbase */ 727 movw TF_GS(%rsp),%si 728 pushfq 729 cli 730 movl $MSR_GSBASE,%ecx 731 /* Save current kernel %gs base into %r12d:%r13d */ 732 rdmsr 733 movl %eax,%r12d 734 movl %edx,%r13d 735 .globl ld_gs 736ld_gs: 737 movw %si,%gs 738 /* Save user %gs base into %r14d:%r15d */ 739 rdmsr 740 movl %eax,%r14d 741 movl %edx,%r15d 742 /* Restore kernel %gs base */ 743 movl %r12d,%eax 744 movl %r13d,%edx 745 wrmsr 746 popfq 747 /* 748 * Restore user %gs base, either from PCB if used for TLS, or 749 * from the previously saved msr read. 750 */ 751 movl $MSR_KGSBASE,%ecx 752 cmpw $KUG32SEL,%si 753 jne 1f 754 movl PCB_GSBASE(%r8),%eax 755 movl PCB_GSBASE+4(%r8),%edx 756 jmp ld_gsbase 7571: 758 movl %r14d,%eax 759 movl %r15d,%edx 760 .globl ld_gsbase 761ld_gsbase: 762 wrmsr /* May trap if non-canonical, but only for TLS. */ 763 .globl ld_es 764ld_es: 765 movw TF_ES(%rsp),%es 766 .globl ld_ds 767ld_ds: 768 movw TF_DS(%rsp),%ds 769ld_regs: 770 movq TF_RDI(%rsp),%rdi 771 movq TF_RSI(%rsp),%rsi 772 movq TF_RDX(%rsp),%rdx 773 movq TF_RCX(%rsp),%rcx 774 movq TF_R8(%rsp),%r8 775 movq TF_R9(%rsp),%r9 776 movq TF_RAX(%rsp),%rax 777 movq TF_RBX(%rsp),%rbx 778 movq TF_RBP(%rsp),%rbp 779 movq TF_R10(%rsp),%r10 780 movq TF_R11(%rsp),%r11 781 movq TF_R12(%rsp),%r12 782 movq TF_R13(%rsp),%r13 783 movq TF_R14(%rsp),%r14 784 movq TF_R15(%rsp),%r15 785 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 786 jz 1f /* keep running with kernel GS.base */ 787 cli 788 swapgs 7891: 790 addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ 791 .globl doreti_iret 792doreti_iret: 793 iretq 794 795set_segs: 796 movw $KUDSEL,%ax 797 movw %ax,TF_DS(%rsp) 798 movw %ax,TF_ES(%rsp) 799 movw $KUF32SEL,TF_FS(%rsp) 800 movw $KUG32SEL,TF_GS(%rsp) 801 jmp do_segs 802 803 /* 804 * doreti_iret_fault. Alternative return code for 805 * the case where we get a fault in the doreti_exit code 806 * above. trap() (amd64/amd64/trap.c) catches this specific 807 * case, sends the process a signal and continues in the 808 * corresponding place in the code below. 809 */ 810 ALIGN_TEXT 811 .globl doreti_iret_fault 812doreti_iret_fault: 813 subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ 814 testl $PSL_I,TF_RFLAGS(%rsp) 815 jz 1f 816 sti 8171: 818 movw %fs,TF_FS(%rsp) 819 movw %gs,TF_GS(%rsp) 820 movw %es,TF_ES(%rsp) 821 movw %ds,TF_DS(%rsp) 822 movl $TF_HASSEGS,TF_FLAGS(%rsp) 823 movq %rdi,TF_RDI(%rsp) 824 movq %rsi,TF_RSI(%rsp) 825 movq %rdx,TF_RDX(%rsp) 826 movq %rcx,TF_RCX(%rsp) 827 movq %r8,TF_R8(%rsp) 828 movq %r9,TF_R9(%rsp) 829 movq %rax,TF_RAX(%rsp) 830 movq %rbx,TF_RBX(%rsp) 831 movq %rbp,TF_RBP(%rsp) 832 movq %r10,TF_R10(%rsp) 833 movq %r11,TF_R11(%rsp) 834 movq %r12,TF_R12(%rsp) 835 movq %r13,TF_R13(%rsp) 836 movq %r14,TF_R14(%rsp) 837 movq %r15,TF_R15(%rsp) 838 movl $T_PROTFLT,TF_TRAPNO(%rsp) 839 movq $0,TF_ERR(%rsp) /* XXX should be the error code */ 840 movq $0,TF_ADDR(%rsp) 841 FAKE_MCOUNT(TF_RIP(%rsp)) 842 jmp calltrap 843 844 ALIGN_TEXT 845 .globl ds_load_fault 846ds_load_fault: 847 movl $T_PROTFLT,TF_TRAPNO(%rsp) 848 testl $PSL_I,TF_RFLAGS(%rsp) 849 jz 1f 850 sti 8511: 852 movq %rsp,%rdi 853 call trap 854 movw $KUDSEL,TF_DS(%rsp) 855 jmp doreti 856 857 ALIGN_TEXT 858 .globl es_load_fault 859es_load_fault: 860 movl $T_PROTFLT,TF_TRAPNO(%rsp) 861 testl $PSL_I,TF_RFLAGS(%rsp) 862 jz 1f 863 sti 8641: 865 movq %rsp,%rdi 866 call trap 867 movw $KUDSEL,TF_ES(%rsp) 868 jmp doreti 869 870 ALIGN_TEXT 871 .globl fs_load_fault 872fs_load_fault: 873 testl $PSL_I,TF_RFLAGS(%rsp) 874 jz 1f 875 sti 8761: 877 movl $T_PROTFLT,TF_TRAPNO(%rsp) 878 movq %rsp,%rdi 879 call trap 880 movw $KUF32SEL,TF_FS(%rsp) 881 jmp doreti 882 883 ALIGN_TEXT 884 .globl gs_load_fault 885gs_load_fault: 886 popfq 887 movl $T_PROTFLT,TF_TRAPNO(%rsp) 888 testl $PSL_I,TF_RFLAGS(%rsp) 889 jz 1f 890 sti 8911: 892 movq %rsp,%rdi 893 call trap 894 movw $KUG32SEL,TF_GS(%rsp) 895 jmp doreti 896 897 ALIGN_TEXT 898 .globl fsbase_load_fault 899fsbase_load_fault: 900 movl $T_PROTFLT,TF_TRAPNO(%rsp) 901 testl $PSL_I,TF_RFLAGS(%rsp) 902 jz 1f 903 sti 9041: 905 movq %rsp,%rdi 906 call trap 907 movq PCPU(CURTHREAD),%r8 908 movq TD_PCB(%r8),%r8 909 movq $0,PCB_FSBASE(%r8) 910 jmp doreti 911 912 ALIGN_TEXT 913 .globl gsbase_load_fault 914gsbase_load_fault: 915 movl $T_PROTFLT,TF_TRAPNO(%rsp) 916 testl $PSL_I,TF_RFLAGS(%rsp) 917 jz 1f 918 sti 9191: 920 movq %rsp,%rdi 921 call trap 922 movq PCPU(CURTHREAD),%r8 923 movq TD_PCB(%r8),%r8 924 movq $0,PCB_GSBASE(%r8) 925 jmp doreti 926 927#ifdef HWPMC_HOOKS 928 ENTRY(end_exceptions) 929#endif 930