1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * Copyright (c) 2007 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * Portions of this software were developed by A. Joseph Koshy under 8 * sponsorship from the FreeBSD Foundation and Google, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD$ 35 */ 36 37#include "opt_atpic.h" 38#include "opt_compat.h" 39#include "opt_hwpmc_hooks.h" 40#include "opt_kdtrace.h" 41 42#include <machine/asmacros.h> 43#include <machine/psl.h> 44#include <machine/trap.h> 45#include <machine/specialreg.h> 46 47#include "assym.s" 48 49#ifdef KDTRACE_HOOKS 50 .bss 51 .globl dtrace_invop_jump_addr 52 .align 8 53 .type dtrace_invop_jump_addr,@object 54 .size dtrace_invop_jump_addr,8 55dtrace_invop_jump_addr: 56 .zero 8 57 .globl dtrace_invop_calltrap_addr 58 .align 8 59 .type dtrace_invop_calltrap_addr,@object 60 .size dtrace_invop_calltrap_addr,8 61dtrace_invop_calltrap_addr: 62 .zero 8 63#endif 64 .text 65#ifdef HWPMC_HOOKS 66 ENTRY(start_exceptions) 67#endif 68 69/*****************************************************************************/ 70/* Trap handling */ 71/*****************************************************************************/ 72/* 73 * Trap and fault vector routines. 74 * 75 * All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes 76 * state on the stack but also disables interrupts. This is important for 77 * us for the use of the swapgs instruction. We cannot be interrupted 78 * until the GS.base value is correct. For most traps, we automatically 79 * then enable interrupts if the interrupted context had them enabled. 80 * This is equivalent to the i386 port's use of SDT_SYS386TGT. 81 * 82 * The cpu will push a certain amount of state onto the kernel stack for 83 * the current process. See amd64/include/frame.h. 84 * This includes the current RFLAGS (status register, which includes 85 * the interrupt disable state prior to the trap), the code segment register, 86 * and the return instruction pointer are pushed by the cpu. The cpu 87 * will also push an 'error' code for certain traps. We push a dummy 88 * error code for those traps where the cpu doesn't in order to maintain 89 * a consistent frame. We also push a contrived 'trap number'. 90 * 91 * The cpu does not push the general registers, we must do that, and we 92 * must restore them prior to calling 'iret'. The cpu adjusts the %cs and 93 * %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we 94 * must load them with appropriate values for supervisor mode operation. 95 */ 96 97MCOUNT_LABEL(user) 98MCOUNT_LABEL(btrap) 99 100/* Traps that we leave interrupts disabled for.. */ 101#define TRAP_NOEN(a) \ 102 subq $TF_RIP,%rsp; \ 103 movl $(a),TF_TRAPNO(%rsp) ; \ 104 movq $0,TF_ADDR(%rsp) ; \ 105 movq $0,TF_ERR(%rsp) ; \ 106 jmp alltraps_noen 107IDTVEC(dbg) 108 TRAP_NOEN(T_TRCTRAP) 109IDTVEC(bpt) 110 TRAP_NOEN(T_BPTFLT) 111#ifdef KDTRACE_HOOKS 112IDTVEC(dtrace_ret) 113 TRAP_NOEN(T_DTRACE_RET) 114#endif 115 116/* Regular traps; The cpu does not supply tf_err for these. */ 117#define TRAP(a) \ 118 subq $TF_RIP,%rsp; \ 119 movl $(a),TF_TRAPNO(%rsp) ; \ 120 movq $0,TF_ADDR(%rsp) ; \ 121 movq $0,TF_ERR(%rsp) ; \ 122 jmp alltraps 123IDTVEC(div) 124 TRAP(T_DIVIDE) 125IDTVEC(ofl) 126 TRAP(T_OFLOW) 127IDTVEC(bnd) 128 TRAP(T_BOUND) 129IDTVEC(ill) 130 TRAP(T_PRIVINFLT) 131IDTVEC(dna) 132 TRAP(T_DNA) 133IDTVEC(fpusegm) 134 TRAP(T_FPOPFLT) 135IDTVEC(mchk) 136 TRAP(T_MCHK) 137IDTVEC(rsvd) 138 TRAP(T_RESERVED) 139IDTVEC(fpu) 140 TRAP(T_ARITHTRAP) 141IDTVEC(xmm) 142 TRAP(T_XMMFLT) 143 144/* This group of traps have tf_err already pushed by the cpu */ 145#define TRAP_ERR(a) \ 146 subq $TF_ERR,%rsp; \ 147 movl $(a),TF_TRAPNO(%rsp) ; \ 148 movq $0,TF_ADDR(%rsp) ; \ 149 jmp alltraps 150IDTVEC(tss) 151 TRAP_ERR(T_TSSFLT) 152IDTVEC(missing) 153 TRAP_ERR(T_SEGNPFLT) 154IDTVEC(stk) 155 TRAP_ERR(T_STKFLT) 156IDTVEC(align) 157 TRAP_ERR(T_ALIGNFLT) 158 159 /* 160 * alltraps entry point. Use swapgs if this is the first time in the 161 * kernel from userland. Reenable interrupts if they were enabled 162 * before the trap. This approximates SDT_SYS386TGT on the i386 port. 163 */ 164 SUPERALIGN_TEXT 165 .globl alltraps 166 .type alltraps,@function 167alltraps: 168 movq %rdi,TF_RDI(%rsp) 169 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 170 jz alltraps_testi /* already running with kernel GS.base */ 171 swapgs 172 movq PCPU(CURPCB),%rdi 173 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 174 movw %fs,TF_FS(%rsp) 175 movw %gs,TF_GS(%rsp) 176 movw %es,TF_ES(%rsp) 177 movw %ds,TF_DS(%rsp) 178alltraps_testi: 179 testl $PSL_I,TF_RFLAGS(%rsp) 180 jz alltraps_pushregs_no_rdi 181 sti 182alltraps_pushregs_no_rdi: 183 movq %rsi,TF_RSI(%rsp) 184 movq %rdx,TF_RDX(%rsp) 185 movq %rcx,TF_RCX(%rsp) 186 movq %r8,TF_R8(%rsp) 187 movq %r9,TF_R9(%rsp) 188 movq %rax,TF_RAX(%rsp) 189 movq %rbx,TF_RBX(%rsp) 190 movq %rbp,TF_RBP(%rsp) 191 movq %r10,TF_R10(%rsp) 192 movq %r11,TF_R11(%rsp) 193 movq %r12,TF_R12(%rsp) 194 movq %r13,TF_R13(%rsp) 195 movq %r14,TF_R14(%rsp) 196 movq %r15,TF_R15(%rsp) 197 movl $TF_HASSEGS,TF_FLAGS(%rsp) 198 cld 199 FAKE_MCOUNT(TF_RIP(%rsp)) 200#ifdef KDTRACE_HOOKS 201 /* 202 * DTrace Function Boundary Trace (fbt) probes are triggered 203 * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint 204 * interrupt. For all other trap types, just handle them in 205 * the usual way. 206 */ 207 cmpl $T_BPTFLT,TF_TRAPNO(%rsp) 208 jne calltrap 209 210 /* Check if there is no DTrace hook registered. */ 211 cmpq $0,dtrace_invop_jump_addr 212 je calltrap 213 214 /* 215 * Set our jump address for the jump back in the event that 216 * the breakpoint wasn't caused by DTrace at all. 217 */ 218 movq $calltrap,dtrace_invop_calltrap_addr(%rip) 219 220 /* Jump to the code hooked in by DTrace. */ 221 movq dtrace_invop_jump_addr,%rax 222 jmpq *dtrace_invop_jump_addr 223#endif 224 .globl calltrap 225 .type calltrap,@function 226calltrap: 227 movq %rsp,%rdi 228 call trap 229 MEXITCOUNT 230 jmp doreti /* Handle any pending ASTs */ 231 232 /* 233 * alltraps_noen entry point. Unlike alltraps above, we want to 234 * leave the interrupts disabled. This corresponds to 235 * SDT_SYS386IGT on the i386 port. 236 */ 237 SUPERALIGN_TEXT 238 .globl alltraps_noen 239 .type alltraps_noen,@function 240alltraps_noen: 241 movq %rdi,TF_RDI(%rsp) 242 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 243 jz 1f /* already running with kernel GS.base */ 244 swapgs 245 movq PCPU(CURPCB),%rdi 246 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 2471: movw %fs,TF_FS(%rsp) 248 movw %gs,TF_GS(%rsp) 249 movw %es,TF_ES(%rsp) 250 movw %ds,TF_DS(%rsp) 251 jmp alltraps_pushregs_no_rdi 252 253IDTVEC(dblfault) 254 subq $TF_ERR,%rsp 255 movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) 256 movq $0,TF_ADDR(%rsp) 257 movq $0,TF_ERR(%rsp) 258 movq %rdi,TF_RDI(%rsp) 259 movq %rsi,TF_RSI(%rsp) 260 movq %rdx,TF_RDX(%rsp) 261 movq %rcx,TF_RCX(%rsp) 262 movq %r8,TF_R8(%rsp) 263 movq %r9,TF_R9(%rsp) 264 movq %rax,TF_RAX(%rsp) 265 movq %rbx,TF_RBX(%rsp) 266 movq %rbp,TF_RBP(%rsp) 267 movq %r10,TF_R10(%rsp) 268 movq %r11,TF_R11(%rsp) 269 movq %r12,TF_R12(%rsp) 270 movq %r13,TF_R13(%rsp) 271 movq %r14,TF_R14(%rsp) 272 movq %r15,TF_R15(%rsp) 273 movw %fs,TF_FS(%rsp) 274 movw %gs,TF_GS(%rsp) 275 movw %es,TF_ES(%rsp) 276 movw %ds,TF_DS(%rsp) 277 movl $TF_HASSEGS,TF_FLAGS(%rsp) 278 cld 279 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 280 jz 1f /* already running with kernel GS.base */ 281 swapgs 2821: 283 movq %rsp,%rdi 284 call dblfault_handler 2852: 286 hlt 287 jmp 2b 288 289IDTVEC(page) 290 subq $TF_ERR,%rsp 291 movl $T_PAGEFLT,TF_TRAPNO(%rsp) 292 movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 293 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 294 jz 1f /* already running with kernel GS.base */ 295 swapgs 296 movq PCPU(CURPCB),%rdi 297 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 2981: movq %cr2,%rdi /* preserve %cr2 before .. */ 299 movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ 300 movw %fs,TF_FS(%rsp) 301 movw %gs,TF_GS(%rsp) 302 movw %es,TF_ES(%rsp) 303 movw %ds,TF_DS(%rsp) 304 testl $PSL_I,TF_RFLAGS(%rsp) 305 jz alltraps_pushregs_no_rdi 306 sti 307 jmp alltraps_pushregs_no_rdi 308 309 /* 310 * We have to special-case this one. If we get a trap in doreti() at 311 * the iretq stage, we'll reenter with the wrong gs state. We'll have 312 * to do a special the swapgs in this case even coming from the kernel. 313 * XXX linux has a trap handler for their equivalent of load_gs(). 314 */ 315IDTVEC(prot) 316 subq $TF_ERR,%rsp 317 movl $T_PROTFLT,TF_TRAPNO(%rsp) 318 movq $0,TF_ADDR(%rsp) 319 movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 320 leaq doreti_iret(%rip),%rdi 321 cmpq %rdi,TF_RIP(%rsp) 322 je 1f /* kernel but with user gsbase!! */ 323 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 324 jz 2f /* already running with kernel GS.base */ 3251: swapgs 3262: movq PCPU(CURPCB),%rdi 327 orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) /* always full iret from GPF */ 328 movw %fs,TF_FS(%rsp) 329 movw %gs,TF_GS(%rsp) 330 movw %es,TF_ES(%rsp) 331 movw %ds,TF_DS(%rsp) 332 testl $PSL_I,TF_RFLAGS(%rsp) 333 jz alltraps_pushregs_no_rdi 334 sti 335 jmp alltraps_pushregs_no_rdi 336 337/* 338 * Fast syscall entry point. We enter here with just our new %cs/%ss set, 339 * and the new privilige level. We are still running on the old user stack 340 * pointer. We have to juggle a few things around to find our stack etc. 341 * swapgs gives us access to our PCPU space only. 342 */ 343IDTVEC(fast_syscall) 344 swapgs 345 movq %rsp,PCPU(SCRATCH_RSP) 346 movq PCPU(RSP0),%rsp 347 /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ 348 subq $TF_SIZE,%rsp 349 /* defer TF_RSP till we have a spare register */ 350 movq %r11,TF_RFLAGS(%rsp) 351 movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ 352 movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ 353 movq %r11,TF_RSP(%rsp) /* user stack pointer */ 354 movw %fs,TF_FS(%rsp) 355 movw %gs,TF_GS(%rsp) 356 movw %es,TF_ES(%rsp) 357 movw %ds,TF_DS(%rsp) 358 movq PCPU(CURPCB),%r11 359 andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) 360 sti 361 movq $KUDSEL,TF_SS(%rsp) 362 movq $KUCSEL,TF_CS(%rsp) 363 movq $2,TF_ERR(%rsp) 364 movq %rdi,TF_RDI(%rsp) /* arg 1 */ 365 movq %rsi,TF_RSI(%rsp) /* arg 2 */ 366 movq %rdx,TF_RDX(%rsp) /* arg 3 */ 367 movq %r10,TF_RCX(%rsp) /* arg 4 */ 368 movq %r8,TF_R8(%rsp) /* arg 5 */ 369 movq %r9,TF_R9(%rsp) /* arg 6 */ 370 movq %rax,TF_RAX(%rsp) /* syscall number */ 371 movq %rbx,TF_RBX(%rsp) /* C preserved */ 372 movq %rbp,TF_RBP(%rsp) /* C preserved */ 373 movq %r12,TF_R12(%rsp) /* C preserved */ 374 movq %r13,TF_R13(%rsp) /* C preserved */ 375 movq %r14,TF_R14(%rsp) /* C preserved */ 376 movq %r15,TF_R15(%rsp) /* C preserved */ 377 movl $TF_HASSEGS,TF_FLAGS(%rsp) 378 cld 379 FAKE_MCOUNT(TF_RIP(%rsp)) 380 movq %rsp,%rdi 381 call syscall 382 movq PCPU(CURPCB),%rax 383 MEXITCOUNT 384 jmp doreti 385 386/* 387 * Here for CYA insurance, in case a "syscall" instruction gets 388 * issued from 32 bit compatability mode. MSR_CSTAR has to point 389 * to *something* if EFER_SCE is enabled. 390 */ 391IDTVEC(fast_syscall32) 392 sysret 393 394/* 395 * NMI handling is special. 396 * 397 * First, NMIs do not respect the state of the processor's RFLAGS.IF 398 * bit. The NMI handler may be entered at any time, including when 399 * the processor is in a critical section with RFLAGS.IF == 0. 400 * The processor's GS.base value could be invalid on entry to the 401 * handler. 402 * 403 * Second, the processor treats NMIs specially, blocking further NMIs 404 * until an 'iretq' instruction is executed. We thus need to execute 405 * the NMI handler with interrupts disabled, to prevent a nested interrupt 406 * from executing an 'iretq' instruction and inadvertently taking the 407 * processor out of NMI mode. 408 * 409 * Third, the NMI handler runs on its own stack (tss_ist2). The canonical 410 * GS.base value for the processor is stored just above the bottom of its 411 * NMI stack. For NMIs taken from kernel mode, the current value in 412 * the processor's GS.base is saved at entry to C-preserved register %r12, 413 * the canonical value for GS.base is then loaded into the processor, and 414 * the saved value is restored at exit time. For NMIs taken from user mode, 415 * the cheaper 'SWAPGS' instructions are used for swapping GS.base. 416 */ 417 418IDTVEC(nmi) 419 subq $TF_RIP,%rsp 420 movl $(T_NMI),TF_TRAPNO(%rsp) 421 movq $0,TF_ADDR(%rsp) 422 movq $0,TF_ERR(%rsp) 423 movq %rdi,TF_RDI(%rsp) 424 movq %rsi,TF_RSI(%rsp) 425 movq %rdx,TF_RDX(%rsp) 426 movq %rcx,TF_RCX(%rsp) 427 movq %r8,TF_R8(%rsp) 428 movq %r9,TF_R9(%rsp) 429 movq %rax,TF_RAX(%rsp) 430 movq %rbx,TF_RBX(%rsp) 431 movq %rbp,TF_RBP(%rsp) 432 movq %r10,TF_R10(%rsp) 433 movq %r11,TF_R11(%rsp) 434 movq %r12,TF_R12(%rsp) 435 movq %r13,TF_R13(%rsp) 436 movq %r14,TF_R14(%rsp) 437 movq %r15,TF_R15(%rsp) 438 movw %fs,TF_FS(%rsp) 439 movw %gs,TF_GS(%rsp) 440 movw %es,TF_ES(%rsp) 441 movw %ds,TF_DS(%rsp) 442 movl $TF_HASSEGS,TF_FLAGS(%rsp) 443 cld 444 xorl %ebx,%ebx 445 testb $SEL_RPL_MASK,TF_CS(%rsp) 446 jnz nmi_fromuserspace 447 /* 448 * We've interrupted the kernel. Preserve GS.base in %r12. 449 */ 450 movl $MSR_GSBASE,%ecx 451 rdmsr 452 movq %rax,%r12 453 shlq $32,%rdx 454 orq %rdx,%r12 455 /* Retrieve and load the canonical value for GS.base. */ 456 movq TF_SIZE(%rsp),%rdx 457 movl %edx,%eax 458 shrq $32,%rdx 459 wrmsr 460 jmp nmi_calltrap 461nmi_fromuserspace: 462 incl %ebx 463 swapgs 464/* Note: this label is also used by ddb and gdb: */ 465nmi_calltrap: 466 FAKE_MCOUNT(TF_RIP(%rsp)) 467 movq %rsp,%rdi 468 call trap 469 MEXITCOUNT 470#ifdef HWPMC_HOOKS 471 /* 472 * Capture a userspace callchain if needed. 473 * 474 * - Check if the current trap was from user mode. 475 * - Check if the current thread is valid. 476 * - Check if the thread requires a user call chain to be 477 * captured. 478 * 479 * We are still in NMI mode at this point. 480 */ 481 testl %ebx,%ebx 482 jz nocallchain /* not from userspace */ 483 movq PCPU(CURTHREAD),%rax 484 orq %rax,%rax /* curthread present? */ 485 jz nocallchain 486 testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ 487 jz nocallchain 488 /* 489 * A user callchain is to be captured, so: 490 * - Move execution to the regular kernel stack, to allow for 491 * nested NMI interrupts. 492 * - Take the processor out of "NMI" mode by faking an "iret". 493 * - Enable interrupts, so that copyin() can work. 494 */ 495 movq %rsp,%rsi /* source stack pointer */ 496 movq $TF_SIZE,%rcx 497 movq PCPU(RSP0),%rdx 498 subq %rcx,%rdx 499 movq %rdx,%rdi /* destination stack pointer */ 500 501 shrq $3,%rcx /* trap frame size in long words */ 502 cld 503 rep 504 movsq /* copy trapframe */ 505 506 movl %ss,%eax 507 pushq %rax /* tf_ss */ 508 pushq %rdx /* tf_rsp (on kernel stack) */ 509 pushfq /* tf_rflags */ 510 movl %cs,%eax 511 pushq %rax /* tf_cs */ 512 pushq $outofnmi /* tf_rip */ 513 iretq 514outofnmi: 515 /* 516 * At this point the processor has exited NMI mode and is running 517 * with interrupts turned off on the normal kernel stack. 518 * 519 * If a pending NMI gets recognized at or after this point, it 520 * will cause a kernel callchain to be traced. 521 * 522 * We turn interrupts back on, and call the user callchain capture hook. 523 */ 524 movq pmc_hook,%rax 525 orq %rax,%rax 526 jz nocallchain 527 movq PCPU(CURTHREAD),%rdi /* thread */ 528 movq $PMC_FN_USER_CALLCHAIN,%rsi /* command */ 529 movq %rsp,%rdx /* frame */ 530 sti 531 call *%rax 532 cli 533nocallchain: 534#endif 535 testl %ebx,%ebx 536 jnz doreti_exit 537nmi_kernelexit: 538 /* 539 * Put back the preserved MSR_GSBASE value. 540 */ 541 movl $MSR_GSBASE,%ecx 542 movq %r12,%rdx 543 movl %edx,%eax 544 shrq $32,%rdx 545 wrmsr 546nmi_restoreregs: 547 movq TF_RDI(%rsp),%rdi 548 movq TF_RSI(%rsp),%rsi 549 movq TF_RDX(%rsp),%rdx 550 movq TF_RCX(%rsp),%rcx 551 movq TF_R8(%rsp),%r8 552 movq TF_R9(%rsp),%r9 553 movq TF_RAX(%rsp),%rax 554 movq TF_RBX(%rsp),%rbx 555 movq TF_RBP(%rsp),%rbp 556 movq TF_R10(%rsp),%r10 557 movq TF_R11(%rsp),%r11 558 movq TF_R12(%rsp),%r12 559 movq TF_R13(%rsp),%r13 560 movq TF_R14(%rsp),%r14 561 movq TF_R15(%rsp),%r15 562 addq $TF_RIP,%rsp 563 jmp doreti_iret 564 565ENTRY(fork_trampoline) 566 movq %r12,%rdi /* function */ 567 movq %rbx,%rsi /* arg1 */ 568 movq %rsp,%rdx /* trapframe pointer */ 569 call fork_exit 570 MEXITCOUNT 571 jmp doreti /* Handle any ASTs */ 572 573/* 574 * To efficiently implement classification of trap and interrupt handlers 575 * for profiling, there must be only trap handlers between the labels btrap 576 * and bintr, and only interrupt handlers between the labels bintr and 577 * eintr. This is implemented (partly) by including files that contain 578 * some of the handlers. Before including the files, set up a normal asm 579 * environment so that the included files doen't need to know that they are 580 * included. 581 */ 582 583#ifdef COMPAT_FREEBSD32 584 .data 585 .p2align 4 586 .text 587 SUPERALIGN_TEXT 588 589#include <amd64/ia32/ia32_exception.S> 590#endif 591 592 .data 593 .p2align 4 594 .text 595 SUPERALIGN_TEXT 596MCOUNT_LABEL(bintr) 597 598#include <amd64/amd64/apic_vector.S> 599 600#ifdef DEV_ATPIC 601 .data 602 .p2align 4 603 .text 604 SUPERALIGN_TEXT 605 606#include <amd64/amd64/atpic_vector.S> 607#endif 608 609 .text 610MCOUNT_LABEL(eintr) 611 612/* 613 * void doreti(struct trapframe) 614 * 615 * Handle return from interrupts, traps and syscalls. 616 */ 617 .text 618 SUPERALIGN_TEXT 619 .type doreti,@function 620doreti: 621 FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */ 622 /* 623 * Check if ASTs can be handled now. 624 */ 625 testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */ 626 jz doreti_exit /* can't handle ASTs now if not */ 627 628doreti_ast: 629 /* 630 * Check for ASTs atomically with returning. Disabling CPU 631 * interrupts provides sufficient locking eve in the SMP case, 632 * since we will be informed of any new ASTs by an IPI. 633 */ 634 cli 635 movq PCPU(CURTHREAD),%rax 636 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 637 je doreti_exit 638 sti 639 movq %rsp,%rdi /* pass a pointer to the trapframe */ 640 call ast 641 jmp doreti_ast 642 643 /* 644 * doreti_exit: pop registers, iret. 645 * 646 * The segment register pop is a special case, since it may 647 * fault if (for example) a sigreturn specifies bad segment 648 * registers. The fault is handled in trap.c. 649 */ 650doreti_exit: 651 MEXITCOUNT 652 movq PCPU(CURTHREAD),%r8 653 movq TD_PCB(%r8),%r8 654 655 /* 656 * Do not reload segment registers for kernel. 657 * Since we do not reload segments registers with sane 658 * values on kernel entry, descriptors referenced by 659 * segments registers might be not valid. This is fatal 660 * for user mode, but is not a problem for the kernel. 661 */ 662 testb $SEL_RPL_MASK,TF_CS(%rsp) 663 jz ld_regs 664 testl $PCB_FULL_IRET,PCB_FLAGS(%r8) 665 jz ld_regs 666 testl $TF_HASSEGS,TF_FLAGS(%rsp) 667 je set_segs 668 669do_segs: 670 /* Restore %fs and fsbase */ 671 movw TF_FS(%rsp),%ax 672 .globl ld_fs 673ld_fs: 674 movw %ax,%fs 675 cmpw $KUF32SEL,%ax 676 jne 1f 677 movl $MSR_FSBASE,%ecx 678 movl PCB_FSBASE(%r8),%eax 679 movl PCB_FSBASE+4(%r8),%edx 680 .globl ld_fsbase 681ld_fsbase: 682 wrmsr 6831: 684 /* Restore %gs and gsbase */ 685 movw TF_GS(%rsp),%si 686 pushfq 687 cli 688 movl $MSR_GSBASE,%ecx 689 rdmsr 690 .globl ld_gs 691ld_gs: 692 movw %si,%gs 693 wrmsr 694 popfq 695 cmpw $KUG32SEL,%si 696 jne 1f 697 movl $MSR_KGSBASE,%ecx 698 movl PCB_GSBASE(%r8),%eax 699 movl PCB_GSBASE+4(%r8),%edx 700 .globl ld_gsbase 701ld_gsbase: 702 wrmsr 7031: 704 .globl ld_es 705ld_es: 706 movw TF_ES(%rsp),%es 707 .globl ld_ds 708ld_ds: 709 movw TF_DS(%rsp),%ds 710ld_regs: 711 movq TF_RDI(%rsp),%rdi 712 movq TF_RSI(%rsp),%rsi 713 movq TF_RDX(%rsp),%rdx 714 movq TF_RCX(%rsp),%rcx 715 movq TF_R8(%rsp),%r8 716 movq TF_R9(%rsp),%r9 717 movq TF_RAX(%rsp),%rax 718 movq TF_RBX(%rsp),%rbx 719 movq TF_RBP(%rsp),%rbp 720 movq TF_R10(%rsp),%r10 721 movq TF_R11(%rsp),%r11 722 movq TF_R12(%rsp),%r12 723 movq TF_R13(%rsp),%r13 724 movq TF_R14(%rsp),%r14 725 movq TF_R15(%rsp),%r15 726 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 727 jz 1f /* keep running with kernel GS.base */ 728 cli 729 swapgs 7301: 731 addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ 732 .globl doreti_iret 733doreti_iret: 734 iretq 735 736set_segs: 737 movw $KUDSEL,%ax 738 movw %ax,TF_DS(%rsp) 739 movw %ax,TF_ES(%rsp) 740 movw $KUF32SEL,TF_FS(%rsp) 741 movw $KUG32SEL,TF_GS(%rsp) 742 jmp do_segs 743 744 /* 745 * doreti_iret_fault. Alternative return code for 746 * the case where we get a fault in the doreti_exit code 747 * above. trap() (amd64/amd64/trap.c) catches this specific 748 * case, sends the process a signal and continues in the 749 * corresponding place in the code below. 750 */ 751 ALIGN_TEXT 752 .globl doreti_iret_fault 753doreti_iret_fault: 754 subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ 755 testl $PSL_I,TF_RFLAGS(%rsp) 756 jz 1f 757 sti 7581: 759 movw %fs,TF_FS(%rsp) 760 movw %gs,TF_GS(%rsp) 761 movw %es,TF_ES(%rsp) 762 movw %ds,TF_DS(%rsp) 763 movl $TF_HASSEGS,TF_FLAGS(%rsp) 764 movq %rdi,TF_RDI(%rsp) 765 movq %rsi,TF_RSI(%rsp) 766 movq %rdx,TF_RDX(%rsp) 767 movq %rcx,TF_RCX(%rsp) 768 movq %r8,TF_R8(%rsp) 769 movq %r9,TF_R9(%rsp) 770 movq %rax,TF_RAX(%rsp) 771 movq %rbx,TF_RBX(%rsp) 772 movq %rbp,TF_RBP(%rsp) 773 movq %r10,TF_R10(%rsp) 774 movq %r11,TF_R11(%rsp) 775 movq %r12,TF_R12(%rsp) 776 movq %r13,TF_R13(%rsp) 777 movq %r14,TF_R14(%rsp) 778 movq %r15,TF_R15(%rsp) 779 movl $T_PROTFLT,TF_TRAPNO(%rsp) 780 movq $0,TF_ERR(%rsp) /* XXX should be the error code */ 781 movq $0,TF_ADDR(%rsp) 782 FAKE_MCOUNT(TF_RIP(%rsp)) 783 jmp calltrap 784 785 ALIGN_TEXT 786 .globl ds_load_fault 787ds_load_fault: 788 movl $T_PROTFLT,TF_TRAPNO(%rsp) 789 movq %rsp,%rdi 790 call trap 791 movw $KUDSEL,TF_DS(%rsp) 792 jmp doreti 793 794 ALIGN_TEXT 795 .globl es_load_fault 796es_load_fault: 797 movl $T_PROTFLT,TF_TRAPNO(%rsp) 798 movq %rsp,%rdi 799 call trap 800 movw $KUDSEL,TF_ES(%rsp) 801 jmp doreti 802 803 ALIGN_TEXT 804 .globl fs_load_fault 805fs_load_fault: 806 movl $T_PROTFLT,TF_TRAPNO(%rsp) 807 movq %rsp,%rdi 808 call trap 809 movw $KUF32SEL,TF_FS(%rsp) 810 jmp doreti 811 812 ALIGN_TEXT 813 .globl gs_load_fault 814gs_load_fault: 815 popfq 816 movl $T_PROTFLT,TF_TRAPNO(%rsp) 817 movq %rsp,%rdi 818 call trap 819 movw $KUG32SEL,TF_GS(%rsp) 820 jmp doreti 821 822 ALIGN_TEXT 823 .globl fsbase_load_fault 824fsbase_load_fault: 825 movl $T_PROTFLT,TF_TRAPNO(%rsp) 826 movq %rsp,%rdi 827 call trap 828 movq PCPU(CURTHREAD),%r8 829 movq TD_PCB(%r8),%r8 830 movq $0,PCB_FSBASE(%r8) 831 jmp doreti 832 833 ALIGN_TEXT 834 .globl gsbase_load_fault 835gsbase_load_fault: 836 movl $T_PROTFLT,TF_TRAPNO(%rsp) 837 movq %rsp,%rdi 838 call trap 839 movq PCPU(CURTHREAD),%r8 840 movq TD_PCB(%r8),%r8 841 movq $0,PCB_GSBASE(%r8) 842 jmp doreti 843 844#ifdef HWPMC_HOOKS 845 ENTRY(end_exceptions) 846#endif 847