10967373eSDavid Greenman/*- 2fcfe57d6SPeter Wemm * Copyright (c) 2003 Peter Wemm. 30967373eSDavid Greenman * Copyright (c) 1990 The Regents of the University of California. 40967373eSDavid Greenman * All rights reserved. 50967373eSDavid Greenman * 60967373eSDavid Greenman * This code is derived from software contributed to Berkeley by 70967373eSDavid Greenman * William Jolitz. 80967373eSDavid Greenman * 90967373eSDavid Greenman * Redistribution and use in source and binary forms, with or without 100967373eSDavid Greenman * modification, are permitted provided that the following conditions 110967373eSDavid Greenman * are met: 120967373eSDavid Greenman * 1. Redistributions of source code must retain the above copyright 130967373eSDavid Greenman * notice, this list of conditions and the following disclaimer. 140967373eSDavid Greenman * 2. Redistributions in binary form must reproduce the above copyright 150967373eSDavid Greenman * notice, this list of conditions and the following disclaimer in the 160967373eSDavid Greenman * documentation and/or other materials provided with the distribution. 170967373eSDavid Greenman * 4. Neither the name of the University nor the names of its contributors 180967373eSDavid Greenman * may be used to endorse or promote products derived from this software 190967373eSDavid Greenman * without specific prior written permission. 200967373eSDavid Greenman * 210967373eSDavid Greenman * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 220967373eSDavid Greenman * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 230967373eSDavid Greenman * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 240967373eSDavid Greenman * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 250967373eSDavid Greenman * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 260967373eSDavid Greenman * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 270967373eSDavid Greenman * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 280967373eSDavid Greenman * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 290967373eSDavid Greenman * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 300967373eSDavid Greenman * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 310967373eSDavid Greenman * SUCH DAMAGE. 320967373eSDavid Greenman * 33c3aac50fSPeter Wemm * $FreeBSD$ 340967373eSDavid Greenman */ 350967373eSDavid Greenman 360513ce7fSBruce Evans#include <machine/asmacros.h> 37db527225SPeter Wemm#include <machine/specialreg.h> 38477a642cSPeter Wemm 390513ce7fSBruce Evans#include "assym.s" 400513ce7fSBruce Evans 410967373eSDavid Greenman/*****************************************************************************/ 420967373eSDavid Greenman/* Scheduling */ 430967373eSDavid Greenman/*****************************************************************************/ 440967373eSDavid Greenman 450967373eSDavid Greenman .text 460967373eSDavid Greenman 4751621230SPeter Wemm#ifdef SMP 4851621230SPeter Wemm#define LK lock ; 4951621230SPeter Wemm#else 5051621230SPeter Wemm#define LK 5151621230SPeter Wemm#endif 5251621230SPeter Wemm 530967373eSDavid Greenman/* 540384fff8SJason Evans * cpu_throw() 55e602ba25SJulian Elischer * 562c87e001SPeter Wemm * This is the second half of cpu_switch(). It is used when the current 57e602ba25SJulian Elischer * thread is either a dummy or slated to die, and we no longer care 58cc66ebe2SPeter Wemm * about its state. This is only a slight optimization and is probably 59cc66ebe2SPeter Wemm * not worth it anymore. Note that we need to clear the pm_active bits so 60cc66ebe2SPeter Wemm * we do need the old proc if it still exists. 61afa88623SPeter Wemm * %rdi = oldtd 62afa88623SPeter Wemm * %rsi = newtd 630384fff8SJason Evans */ 640384fff8SJason EvansENTRY(cpu_throw) 65afa88623SPeter Wemm movl PCPU(CPUID), %eax 66afa88623SPeter Wemm testq %rdi,%rdi /* no thread? */ 67cc66ebe2SPeter Wemm jz 1f 68cc66ebe2SPeter Wemm /* release bit from old pm_active */ 69afa88623SPeter Wemm movq TD_PROC(%rdi), %rdx /* oldtd->td_proc */ 70afa88623SPeter Wemm movq P_VMSPACE(%rdx), %rdx /* proc->p_vmspace */ 7151621230SPeter Wemm LK btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ 72cc66ebe2SPeter Wemm1: 73afa88623SPeter Wemm movq TD_PCB(%rsi),%rdx /* newtd->td_proc */ 74afa88623SPeter Wemm movq PCB_CR3(%rdx),%rdx 75afa88623SPeter Wemm movq %rdx,%cr3 /* new address space */ 76cc66ebe2SPeter Wemm /* set bit in new pm_active */ 77afa88623SPeter Wemm movq TD_PROC(%rsi),%rdx 78afa88623SPeter Wemm movq P_VMSPACE(%rdx), %rdx 7951621230SPeter Wemm LK btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */ 800384fff8SJason Evans jmp sw1 810384fff8SJason Evans 820384fff8SJason Evans/* 83cc66ebe2SPeter Wemm * cpu_switch(old, new) 84e602ba25SJulian Elischer * 85e602ba25SJulian Elischer * Save the current thread state, then select the next thread to run 86e602ba25SJulian Elischer * and load its state. 87afa88623SPeter Wemm * %rdi = oldtd 88afa88623SPeter Wemm * %rsi = newtd 890967373eSDavid Greenman */ 9026f9a767SRodney W. GrimesENTRY(cpu_switch) 91cc66ebe2SPeter Wemm /* Switch to new thread. First, save context. */ 92bf1e8974SPeter Wemm movq TD_PCB(%rdi),%r8 930967373eSDavid Greenman 94afa88623SPeter Wemm movq (%rsp),%rax /* Hardware registers */ 95bf1e8974SPeter Wemm movq %rax,PCB_RIP(%r8) 96bf1e8974SPeter Wemm movq %rbx,PCB_RBX(%r8) 97bf1e8974SPeter Wemm movq %rsp,PCB_RSP(%r8) 98bf1e8974SPeter Wemm movq %rbp,PCB_RBP(%r8) 99bf1e8974SPeter Wemm movq %r12,PCB_R12(%r8) 100bf1e8974SPeter Wemm movq %r13,PCB_R13(%r8) 101bf1e8974SPeter Wemm movq %r14,PCB_R14(%r8) 102bf1e8974SPeter Wemm movq %r15,PCB_R15(%r8) 103afa88623SPeter Wemm pushfq /* PSL */ 104bf1e8974SPeter Wemm popq PCB_RFLAGS(%r8) 105bf1e8974SPeter Wemm 106df4fd277SPeter Wemm testl $PCB_32BIT,PCB_FLAGS(%r8) 107df4fd277SPeter Wemm jz 1f /* no, skip over */ 108df4fd277SPeter Wemm 109df4fd277SPeter Wemm /* Save segment selector numbers */ 110df4fd277SPeter Wemm movl %ds,PCB_DS(%r8) 111df4fd277SPeter Wemm movl %es,PCB_ES(%r8) 112df4fd277SPeter Wemm movl %fs,PCB_FS(%r8) 113df4fd277SPeter Wemm movl %gs,PCB_GS(%r8) 114df4fd277SPeter Wemm jmp 2f 115df4fd277SPeter Wemm1: 116df4fd277SPeter Wemm 117bf1e8974SPeter Wemm /* Save userland %fs */ 118bf1e8974SPeter Wemm movl $MSR_FSBASE,%ecx 119bf1e8974SPeter Wemm rdmsr 120bf1e8974SPeter Wemm movl %eax,PCB_FSBASE(%r8) 121bf1e8974SPeter Wemm movl %edx,PCB_FSBASE+4(%r8) 122bf1e8974SPeter Wemm 123bf1e8974SPeter Wemm /* Save userland %gs */ 124bf1e8974SPeter Wemm movl $MSR_KGSBASE,%ecx 125bf1e8974SPeter Wemm rdmsr 126bf1e8974SPeter Wemm movl %eax,PCB_GSBASE(%r8) 127bf1e8974SPeter Wemm movl %edx,PCB_GSBASE+4(%r8) 128df4fd277SPeter Wemm2: 129d85631c4SPeter Wemm 130db527225SPeter Wemm /* Test if debug registers should be saved. */ 131db527225SPeter Wemm testl $PCB_DBREGS,PCB_FLAGS(%r8) 132db527225SPeter Wemm jz 1f /* no, skip over */ 133db527225SPeter Wemm movq %dr7,%rax /* yes, do the save */ 134db527225SPeter Wemm movq %rax,PCB_DR7(%r8) 135db527225SPeter Wemm andq $0x0000fc00, %rax /* disable all watchpoints */ 136db527225SPeter Wemm movq %rax,%dr7 137db527225SPeter Wemm movq %dr6,%rax 138db527225SPeter Wemm movq %rax,PCB_DR6(%r8) 139db527225SPeter Wemm movq %dr3,%rax 140db527225SPeter Wemm movq %rax,PCB_DR3(%r8) 141db527225SPeter Wemm movq %dr2,%rax 142db527225SPeter Wemm movq %rax,PCB_DR2(%r8) 143db527225SPeter Wemm movq %dr1,%rax 144db527225SPeter Wemm movq %rax,PCB_DR1(%r8) 145db527225SPeter Wemm movq %dr0,%rax 146db527225SPeter Wemm movq %rax,PCB_DR0(%r8) 147db527225SPeter Wemm1: 148db527225SPeter Wemm 1490967373eSDavid Greenman /* have we used fp, and need a save? */ 150afa88623SPeter Wemm cmpq %rdi,PCPU(FPCURTHREAD) 1510967373eSDavid Greenman jne 1f 152db527225SPeter Wemm addq $PCB_SAVEFPU,%r8 153db527225SPeter Wemm clts 154db527225SPeter Wemm fxsave (%r8) 155db527225SPeter Wemm smsw %ax 156db527225SPeter Wemm orb $CR0_TS,%al 157db527225SPeter Wemm lmsw %ax 158db527225SPeter Wemm xorq %rax,%rax 159db527225SPeter Wemm movq %rax,PCPU(FPCURTHREAD) 1600967373eSDavid Greenman1: 1610967373eSDavid Greenman 162cc66ebe2SPeter Wemm /* Save is done. Now fire up new thread. Leave old vmspace. */ 163bf1e8974SPeter Wemm movq TD_PCB(%rsi),%r8 164b40ce416SJulian Elischer 1650967373eSDavid Greenman /* switch address space */ 166bf1e8974SPeter Wemm movq PCB_CR3(%r8),%rdx 167db527225SPeter Wemm movq %cr3,%rax 168db527225SPeter Wemm cmpq %rdx,%rax /* Same address space? */ 169db527225SPeter Wemm je sw1 170afa88623SPeter Wemm movq %rdx,%cr3 /* new address space */ 171163fd6fbSJohn Baldwin 172db527225SPeter Wemm movl PCPU(CPUID), %eax 173cc66ebe2SPeter Wemm /* Release bit from old pmap->pm_active */ 174afa88623SPeter Wemm movq TD_PROC(%rdi), %rdx /* oldproc */ 175afa88623SPeter Wemm movq P_VMSPACE(%rdx), %rdx 17651621230SPeter Wemm LK btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ 177cc66ebe2SPeter Wemm 178cc66ebe2SPeter Wemm /* Set bit in new pmap->pm_active */ 179afa88623SPeter Wemm movq TD_PROC(%rsi),%rdx /* newproc */ 180afa88623SPeter Wemm movq P_VMSPACE(%rdx), %rdx 18151621230SPeter Wemm LK btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */ 182cc66ebe2SPeter Wemm 183cc66ebe2SPeter Wemmsw1: 184cc66ebe2SPeter Wemm /* 185cc66ebe2SPeter Wemm * At this point, we've switched address spaces and are ready 186cc66ebe2SPeter Wemm * to load up the rest of the next context. 187cc66ebe2SPeter Wemm */ 188bf1e8974SPeter Wemm movq TD_PCB(%rsi),%r8 189bf1e8974SPeter Wemm 190df4fd277SPeter Wemm testl $PCB_32BIT,PCB_FLAGS(%r8) 191df4fd277SPeter Wemm jz 1f /* no, skip over */ 192df4fd277SPeter Wemm 193d85631c4SPeter Wemm /* Restore segment selector numbers */ 194d85631c4SPeter Wemm movl PCB_DS(%r8),%ds 195d85631c4SPeter Wemm movl PCB_ES(%r8),%es 196d85631c4SPeter Wemm movl PCB_FS(%r8),%fs 197d85631c4SPeter Wemm 198c0a54ff6SPeter Wemm /* Restore userland %gs while preserving kernel gsbase */ 199d85631c4SPeter Wemm movl $MSR_GSBASE,%ecx 200c0a54ff6SPeter Wemm rdmsr 201c0a54ff6SPeter Wemm movl PCB_GS(%r8),%gs 202d85631c4SPeter Wemm wrmsr 203df4fd277SPeter Wemm jmp 2f 204df4fd277SPeter Wemm1: 205d85631c4SPeter Wemm 206bf1e8974SPeter Wemm /* Restore userland %fs */ 207bf1e8974SPeter Wemm movl $MSR_FSBASE,%ecx 208bf1e8974SPeter Wemm movl PCB_FSBASE(%r8),%eax 209bf1e8974SPeter Wemm movl PCB_FSBASE+4(%r8),%edx 210bf1e8974SPeter Wemm wrmsr 211bf1e8974SPeter Wemm 212bf1e8974SPeter Wemm /* Restore userland %gs */ 213bf1e8974SPeter Wemm movl $MSR_KGSBASE,%ecx 214bf1e8974SPeter Wemm movl PCB_GSBASE(%r8),%eax 215bf1e8974SPeter Wemm movl PCB_GSBASE+4(%r8),%edx 216bf1e8974SPeter Wemm wrmsr 217df4fd277SPeter Wemm2: 2189869fa1dSJohn Baldwin 219afa88623SPeter Wemm /* Update the TSS_RSP0 pointer for the next interrupt */ 2200d2a2989SPeter Wemm movq PCPU(TSSP), %rax 2210d2a2989SPeter Wemm addq $COMMON_TSS_RSP0, %rax 222bf1e8974SPeter Wemm leaq -16(%r8), %rbx 2230d2a2989SPeter Wemm movq %rbx, (%rax) 2240d2a2989SPeter Wemm movq %rbx, PCPU(RSP0) 225a2a1c95cSPeter Wemm 2269869fa1dSJohn Baldwin /* Restore context. */ 227bf1e8974SPeter Wemm movq PCB_RBX(%r8),%rbx 228bf1e8974SPeter Wemm movq PCB_RSP(%r8),%rsp 229bf1e8974SPeter Wemm movq PCB_RBP(%r8),%rbp 230bf1e8974SPeter Wemm movq PCB_R12(%r8),%r12 231bf1e8974SPeter Wemm movq PCB_R13(%r8),%r13 232bf1e8974SPeter Wemm movq PCB_R14(%r8),%r14 233bf1e8974SPeter Wemm movq PCB_R15(%r8),%r15 234bf1e8974SPeter Wemm movq PCB_RIP(%r8),%rax 235afa88623SPeter Wemm movq %rax,(%rsp) 236bf1e8974SPeter Wemm pushq PCB_RFLAGS(%r8) 237afa88623SPeter Wemm popfq 2380967373eSDavid Greenman 239bf1e8974SPeter Wemm movq %r8, PCPU(CURPCB) 240afa88623SPeter Wemm movq %rsi, PCPU(CURTHREAD) /* into next thread */ 241477a642cSPeter Wemm 242db527225SPeter Wemm /* Test if debug registers should be restored. */ 243db527225SPeter Wemm testl $PCB_DBREGS,PCB_FLAGS(%r8) 244db527225SPeter Wemm jz 1f 245db527225SPeter Wemm movq PCB_DR6(%r8),%rax 246db527225SPeter Wemm movq %rax,%dr6 247db527225SPeter Wemm movq PCB_DR3(%r8),%rax 248db527225SPeter Wemm movq %rax,%dr3 249db527225SPeter Wemm movq PCB_DR2(%r8),%rax 250db527225SPeter Wemm movq %rax,%dr2 251db527225SPeter Wemm movq PCB_DR1(%r8),%rax 252db527225SPeter Wemm movq %rax,%dr1 253db527225SPeter Wemm movq PCB_DR0(%r8),%rax 254db527225SPeter Wemm movq %rax,%dr0 255db527225SPeter Wemm /* But preserve reserved bits in %dr7 */ 256db527225SPeter Wemm movq %dr7,%rax 257db527225SPeter Wemm andq $0x0000fc00,%rax 258db527225SPeter Wemm movq PCB_DR7(%r8),%rcx 259db527225SPeter Wemm andq $~0x0000fc00,%rcx 260db527225SPeter Wemm orq %rcx,%rax 261db527225SPeter Wemm movq %rax,%dr7 262db527225SPeter Wemm1: 2630967373eSDavid Greenman ret 2640967373eSDavid Greenman 2650967373eSDavid Greenman/* 2662924d491SDavid Greenman * savectx(pcb) 2672924d491SDavid Greenman * Update pcb, saving current processor state. 2680967373eSDavid Greenman */ 2690967373eSDavid GreenmanENTRY(savectx) 2709869fa1dSJohn Baldwin /* Fetch PCB. */ 271afa88623SPeter Wemm movq %rdi,%rcx 2722924d491SDavid Greenman 273afa88623SPeter Wemm /* Save caller's return address. */ 274afa88623SPeter Wemm movq (%rsp),%rax 275afa88623SPeter Wemm movq %rax,PCB_RIP(%rcx) 2762924d491SDavid Greenman 277afa88623SPeter Wemm movq %cr3,%rax 278afa88623SPeter Wemm movq %rax,PCB_CR3(%rcx) 279ac5f943cSPeter Wemm 280afa88623SPeter Wemm movq %rbx,PCB_RBX(%rcx) 281afa88623SPeter Wemm movq %rsp,PCB_RSP(%rcx) 282afa88623SPeter Wemm movq %rbp,PCB_RBP(%rcx) 283afa88623SPeter Wemm movq %r12,PCB_R12(%rcx) 284afa88623SPeter Wemm movq %r13,PCB_R13(%rcx) 285afa88623SPeter Wemm movq %r14,PCB_R14(%rcx) 286afa88623SPeter Wemm movq %r15,PCB_R15(%rcx) 287afa88623SPeter Wemm pushfq 288afa88623SPeter Wemm popq PCB_RFLAGS(%rcx) 2890967373eSDavid Greenman 2900967373eSDavid Greenman /* 291bf2f09eeSPeter Wemm * If fpcurthread == NULL, then the fpu h/w state is irrelevant and the 2920967373eSDavid Greenman * state had better already be in the pcb. This is true for forks 2930967373eSDavid Greenman * but not for dumps (the old book-keeping with FP flags in the pcb 2940967373eSDavid Greenman * always lost for dumps because the dump pcb has 0 flags). 2950967373eSDavid Greenman * 296bf2f09eeSPeter Wemm * If fpcurthread != NULL, then we have to save the fpu h/w state to 2970bbc8826SJohn Baldwin * fpcurthread's pcb and copy it to the requested pcb, or save to the 2980967373eSDavid Greenman * requested pcb and reload. Copying is easier because we would 2990967373eSDavid Greenman * have to handle h/w bugs for reloading. We used to lose the 300bf2f09eeSPeter Wemm * parent's fpu state for forks by forgetting to reload. 3010967373eSDavid Greenman */ 302afa88623SPeter Wemm pushfq 303c2b095abSBruce Evans cli 304afa88623SPeter Wemm movq PCPU(FPCURTHREAD),%rax 305afa88623SPeter Wemm testq %rax,%rax 3060967373eSDavid Greenman je 1f 3070967373eSDavid Greenman 308afa88623SPeter Wemm movq TD_PCB(%rax),%rdi 309afa88623SPeter Wemm leaq PCB_SAVEFPU(%rdi),%rdi 310db527225SPeter Wemm clts 311db527225SPeter Wemm fxsave (%rdi) 312db527225SPeter Wemm smsw %ax 313db527225SPeter Wemm orb $CR0_TS,%al 314db527225SPeter Wemm lmsw %ax 3150967373eSDavid Greenman 316afa88623SPeter Wemm movq $PCB_SAVEFPU_SIZE,%rdx /* arg 3 */ 317afa88623SPeter Wemm leaq PCB_SAVEFPU(%rcx),%rsi /* arg 2 */ 318db527225SPeter Wemm /* arg 1 (%rdi) already loaded */ 31902318dacSJake Burkholder call bcopy 320c2b095abSBruce Evans1: 321afa88623SPeter Wemm popfq 3220967373eSDavid Greenman 3230967373eSDavid Greenman ret 324