10967373eSDavid Greenman/*- 2fcfe57d6SPeter Wemm * Copyright (c) 2003 Peter Wemm. 30967373eSDavid Greenman * Copyright (c) 1990 The Regents of the University of California. 40967373eSDavid Greenman * All rights reserved. 50967373eSDavid Greenman * 60967373eSDavid Greenman * This code is derived from software contributed to Berkeley by 70967373eSDavid Greenman * William Jolitz. 80967373eSDavid Greenman * 90967373eSDavid Greenman * Redistribution and use in source and binary forms, with or without 100967373eSDavid Greenman * modification, are permitted provided that the following conditions 110967373eSDavid Greenman * are met: 120967373eSDavid Greenman * 1. Redistributions of source code must retain the above copyright 130967373eSDavid Greenman * notice, this list of conditions and the following disclaimer. 140967373eSDavid Greenman * 2. Redistributions in binary form must reproduce the above copyright 150967373eSDavid Greenman * notice, this list of conditions and the following disclaimer in the 160967373eSDavid Greenman * documentation and/or other materials provided with the distribution. 170967373eSDavid Greenman * 4. Neither the name of the University nor the names of its contributors 180967373eSDavid Greenman * may be used to endorse or promote products derived from this software 190967373eSDavid Greenman * without specific prior written permission. 200967373eSDavid Greenman * 210967373eSDavid Greenman * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 220967373eSDavid Greenman * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 230967373eSDavid Greenman * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 240967373eSDavid Greenman * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 250967373eSDavid Greenman * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 260967373eSDavid Greenman * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 270967373eSDavid Greenman * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 280967373eSDavid Greenman * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 290967373eSDavid Greenman * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 300967373eSDavid Greenman * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 310967373eSDavid Greenman * SUCH DAMAGE. 320967373eSDavid Greenman * 33c3aac50fSPeter Wemm * $FreeBSD$ 340967373eSDavid Greenman */ 350967373eSDavid Greenman 360513ce7fSBruce Evans#include <machine/asmacros.h> 37db527225SPeter Wemm#include <machine/specialreg.h> 38477a642cSPeter Wemm 390513ce7fSBruce Evans#include "assym.s" 4040380a6aSJeff Roberson#include "opt_sched.h" 410513ce7fSBruce Evans 420967373eSDavid Greenman/*****************************************************************************/ 430967373eSDavid Greenman/* Scheduling */ 440967373eSDavid Greenman/*****************************************************************************/ 450967373eSDavid Greenman 460967373eSDavid Greenman .text 470967373eSDavid Greenman 4851621230SPeter Wemm#ifdef SMP 4951621230SPeter Wemm#define LK lock ; 5051621230SPeter Wemm#else 5151621230SPeter Wemm#define LK 5251621230SPeter Wemm#endif 5351621230SPeter Wemm 5440380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 5540380a6aSJeff Roberson#define SETLK xchgq 5640380a6aSJeff Roberson#else 5740380a6aSJeff Roberson#define SETLK movq 5840380a6aSJeff Roberson#endif 5940380a6aSJeff Roberson 600967373eSDavid Greenman/* 610384fff8SJason Evans * cpu_throw() 62e602ba25SJulian Elischer * 632c87e001SPeter Wemm * This is the second half of cpu_switch(). It is used when the current 64e602ba25SJulian Elischer * thread is either a dummy or slated to die, and we no longer care 65cc66ebe2SPeter Wemm * about its state. This is only a slight optimization and is probably 66cc66ebe2SPeter Wemm * not worth it anymore. Note that we need to clear the pm_active bits so 67cc66ebe2SPeter Wemm * we do need the old proc if it still exists. 68afa88623SPeter Wemm * %rdi = oldtd 69afa88623SPeter Wemm * %rsi = newtd 700384fff8SJason Evans */ 710384fff8SJason EvansENTRY(cpu_throw) 72afa88623SPeter Wemm movl PCPU(CPUID), %eax 73afa88623SPeter Wemm testq %rdi,%rdi /* no thread? */ 74cc66ebe2SPeter Wemm jz 1f 75cc66ebe2SPeter Wemm /* release bit from old pm_active */ 76afa88623SPeter Wemm movq TD_PROC(%rdi), %rdx /* oldtd->td_proc */ 77afa88623SPeter Wemm movq P_VMSPACE(%rdx), %rdx /* proc->p_vmspace */ 7851621230SPeter Wemm LK btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ 79cc66ebe2SPeter Wemm1: 80afa88623SPeter Wemm movq TD_PCB(%rsi),%rdx /* newtd->td_proc */ 81afa88623SPeter Wemm movq PCB_CR3(%rdx),%rdx 82afa88623SPeter Wemm movq %rdx,%cr3 /* new address space */ 835d68dad3SJeff Roberson jmp swact 84ea497502SJoseph KoshyEND(cpu_throw) 850384fff8SJason Evans 860384fff8SJason Evans/* 875d68dad3SJeff Roberson * cpu_switch(old, new, mtx) 88e602ba25SJulian Elischer * 89e602ba25SJulian Elischer * Save the current thread state, then select the next thread to run 90e602ba25SJulian Elischer * and load its state. 91afa88623SPeter Wemm * %rdi = oldtd 92afa88623SPeter Wemm * %rsi = newtd 935d68dad3SJeff Roberson * %rdx = mtx 940967373eSDavid Greenman */ 9526f9a767SRodney W. GrimesENTRY(cpu_switch) 96cc66ebe2SPeter Wemm /* Switch to new thread. First, save context. */ 97bf1e8974SPeter Wemm movq TD_PCB(%rdi),%r8 980967373eSDavid Greenman 99afa88623SPeter Wemm movq (%rsp),%rax /* Hardware registers */ 100bf1e8974SPeter Wemm movq %rax,PCB_RIP(%r8) 101bf1e8974SPeter Wemm movq %rbx,PCB_RBX(%r8) 102bf1e8974SPeter Wemm movq %rsp,PCB_RSP(%r8) 103bf1e8974SPeter Wemm movq %rbp,PCB_RBP(%r8) 104bf1e8974SPeter Wemm movq %r12,PCB_R12(%r8) 105bf1e8974SPeter Wemm movq %r13,PCB_R13(%r8) 106bf1e8974SPeter Wemm movq %r14,PCB_R14(%r8) 107bf1e8974SPeter Wemm movq %r15,PCB_R15(%r8) 108bf1e8974SPeter Wemm 109df4fd277SPeter Wemm testl $PCB_32BIT,PCB_FLAGS(%r8) 110df4fd277SPeter Wemm jz 1f /* no, skip over */ 111df4fd277SPeter Wemm 1129c5b213eSJung-uk Kim /* Save userland %gs */ 113df4fd277SPeter Wemm movl %gs,PCB_GS(%r8) 1149c5b213eSJung-uk Kim movq PCB_GS32P(%r8),%rax 1159c5b213eSJung-uk Kim movq (%rax),%rax 1169c5b213eSJung-uk Kim movq %rax,PCB_GS32SD(%r8) 1179c5b213eSJung-uk Kim 118df4fd277SPeter Wemm1: 119db527225SPeter Wemm /* Test if debug registers should be saved. */ 120db527225SPeter Wemm testl $PCB_DBREGS,PCB_FLAGS(%r8) 121db527225SPeter Wemm jz 1f /* no, skip over */ 122db527225SPeter Wemm movq %dr7,%rax /* yes, do the save */ 123db527225SPeter Wemm movq %rax,PCB_DR7(%r8) 124db527225SPeter Wemm andq $0x0000fc00, %rax /* disable all watchpoints */ 125db527225SPeter Wemm movq %rax,%dr7 126db527225SPeter Wemm movq %dr6,%rax 127db527225SPeter Wemm movq %rax,PCB_DR6(%r8) 128db527225SPeter Wemm movq %dr3,%rax 129db527225SPeter Wemm movq %rax,PCB_DR3(%r8) 130db527225SPeter Wemm movq %dr2,%rax 131db527225SPeter Wemm movq %rax,PCB_DR2(%r8) 132db527225SPeter Wemm movq %dr1,%rax 133db527225SPeter Wemm movq %rax,PCB_DR1(%r8) 134db527225SPeter Wemm movq %dr0,%rax 135db527225SPeter Wemm movq %rax,PCB_DR0(%r8) 136db527225SPeter Wemm1: 137db527225SPeter Wemm 1380967373eSDavid Greenman /* have we used fp, and need a save? */ 139afa88623SPeter Wemm cmpq %rdi,PCPU(FPCURTHREAD) 1400967373eSDavid Greenman jne 1f 141db527225SPeter Wemm addq $PCB_SAVEFPU,%r8 142db527225SPeter Wemm clts 143db527225SPeter Wemm fxsave (%r8) 144db527225SPeter Wemm smsw %ax 145db527225SPeter Wemm orb $CR0_TS,%al 146db527225SPeter Wemm lmsw %ax 14798df9e00SPeter Wemm xorl %eax,%eax 148db527225SPeter Wemm movq %rax,PCPU(FPCURTHREAD) 1490967373eSDavid Greenman1: 1500967373eSDavid Greenman 151cc66ebe2SPeter Wemm /* Save is done. Now fire up new thread. Leave old vmspace. */ 152bf1e8974SPeter Wemm movq TD_PCB(%rsi),%r8 153b40ce416SJulian Elischer 1540967373eSDavid Greenman /* switch address space */ 1555d68dad3SJeff Roberson movq PCB_CR3(%r8),%rcx 156db527225SPeter Wemm movq %cr3,%rax 1575d68dad3SJeff Roberson cmpq %rcx,%rax /* Same address space? */ 1585d68dad3SJeff Roberson jne swinact 15940380a6aSJeff Roberson SETLK %rdx, TD_LOCK(%rdi) /* Release the old thread */ 1605d68dad3SJeff Roberson jmp sw1 1615d68dad3SJeff Robersonswinact: 1625d68dad3SJeff Roberson movq %rcx,%cr3 /* new address space */ 163db527225SPeter Wemm movl PCPU(CPUID), %eax 164cc66ebe2SPeter Wemm /* Release bit from old pmap->pm_active */ 1655d68dad3SJeff Roberson movq TD_PROC(%rdi), %rcx /* oldproc */ 1665d68dad3SJeff Roberson movq P_VMSPACE(%rcx), %rcx 1675d68dad3SJeff Roberson LK btrl %eax, VM_PMAP+PM_ACTIVE(%rcx) /* clear old */ 16840380a6aSJeff Roberson SETLK %rdx, TD_LOCK(%rdi) /* Release the old thread */ 1695d68dad3SJeff Robersonswact: 170cc66ebe2SPeter Wemm /* Set bit in new pmap->pm_active */ 171afa88623SPeter Wemm movq TD_PROC(%rsi),%rdx /* newproc */ 172afa88623SPeter Wemm movq P_VMSPACE(%rdx), %rdx 17351621230SPeter Wemm LK btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */ 174cc66ebe2SPeter Wemm 175cc66ebe2SPeter Wemmsw1: 17640380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 17740380a6aSJeff Roberson /* Wait for the new thread to become unblocked */ 17840380a6aSJeff Roberson movq $blocked_lock, %rdx 17940380a6aSJeff Roberson1: 18040380a6aSJeff Roberson movq TD_LOCK(%rsi),%rcx 18140380a6aSJeff Roberson cmpq %rcx, %rdx 18240380a6aSJeff Roberson pause 18340380a6aSJeff Roberson je 1b 18440380a6aSJeff Roberson lfence 18540380a6aSJeff Roberson#endif 186cc66ebe2SPeter Wemm /* 187cc66ebe2SPeter Wemm * At this point, we've switched address spaces and are ready 188cc66ebe2SPeter Wemm * to load up the rest of the next context. 189cc66ebe2SPeter Wemm */ 190bf1e8974SPeter Wemm movq TD_PCB(%rsi),%r8 191bf1e8974SPeter Wemm 192bf1e8974SPeter Wemm /* Restore userland %fs */ 193bf1e8974SPeter Wemm movl $MSR_FSBASE,%ecx 194bf1e8974SPeter Wemm movl PCB_FSBASE(%r8),%eax 195bf1e8974SPeter Wemm movl PCB_FSBASE+4(%r8),%edx 196bf1e8974SPeter Wemm wrmsr 197bf1e8974SPeter Wemm 198bf1e8974SPeter Wemm /* Restore userland %gs */ 199bf1e8974SPeter Wemm movl $MSR_KGSBASE,%ecx 200bf1e8974SPeter Wemm movl PCB_GSBASE(%r8),%eax 201bf1e8974SPeter Wemm movl PCB_GSBASE+4(%r8),%edx 202bf1e8974SPeter Wemm wrmsr 2039869fa1dSJohn Baldwin 204afa88623SPeter Wemm /* Update the TSS_RSP0 pointer for the next interrupt */ 2050d2a2989SPeter Wemm movq PCPU(TSSP), %rax 2060d2a2989SPeter Wemm addq $COMMON_TSS_RSP0, %rax 207bf1e8974SPeter Wemm leaq -16(%r8), %rbx 2080d2a2989SPeter Wemm movq %rbx, (%rax) 2090d2a2989SPeter Wemm movq %rbx, PCPU(RSP0) 210a2a1c95cSPeter Wemm 2114e32b7b3SDavid Xu movq %r8, PCPU(CURPCB) 2124e32b7b3SDavid Xu movq %rsi, PCPU(CURTHREAD) /* into next thread */ 2134e32b7b3SDavid Xu 2149c5b213eSJung-uk Kim testl $PCB_32BIT,PCB_FLAGS(%r8) 2159c5b213eSJung-uk Kim jz 1f /* no, skip over */ 2169c5b213eSJung-uk Kim 2179c5b213eSJung-uk Kim /* Restore userland %gs while preserving kernel gsbase */ 2189c5b213eSJung-uk Kim movq PCB_GS32P(%r8),%rax 2199c5b213eSJung-uk Kim movq PCB_GS32SD(%r8),%rbx 2209c5b213eSJung-uk Kim movq %rbx,(%rax) 2219c5b213eSJung-uk Kim movl $MSR_GSBASE,%ecx 2229c5b213eSJung-uk Kim rdmsr 2239c5b213eSJung-uk Kim movl PCB_GS(%r8),%gs 2249c5b213eSJung-uk Kim wrmsr 2259c5b213eSJung-uk Kim 2269c5b213eSJung-uk Kim1: 2279869fa1dSJohn Baldwin /* Restore context. */ 228bf1e8974SPeter Wemm movq PCB_RBX(%r8),%rbx 229bf1e8974SPeter Wemm movq PCB_RSP(%r8),%rsp 230bf1e8974SPeter Wemm movq PCB_RBP(%r8),%rbp 231bf1e8974SPeter Wemm movq PCB_R12(%r8),%r12 232bf1e8974SPeter Wemm movq PCB_R13(%r8),%r13 233bf1e8974SPeter Wemm movq PCB_R14(%r8),%r14 234bf1e8974SPeter Wemm movq PCB_R15(%r8),%r15 235bf1e8974SPeter Wemm movq PCB_RIP(%r8),%rax 236afa88623SPeter Wemm movq %rax,(%rsp) 2370967373eSDavid Greenman 238db527225SPeter Wemm /* Test if debug registers should be restored. */ 239db527225SPeter Wemm testl $PCB_DBREGS,PCB_FLAGS(%r8) 240db527225SPeter Wemm jz 1f 241db527225SPeter Wemm movq PCB_DR6(%r8),%rax 242db527225SPeter Wemm movq %rax,%dr6 243db527225SPeter Wemm movq PCB_DR3(%r8),%rax 244db527225SPeter Wemm movq %rax,%dr3 245db527225SPeter Wemm movq PCB_DR2(%r8),%rax 246db527225SPeter Wemm movq %rax,%dr2 247db527225SPeter Wemm movq PCB_DR1(%r8),%rax 248db527225SPeter Wemm movq %rax,%dr1 249db527225SPeter Wemm movq PCB_DR0(%r8),%rax 250db527225SPeter Wemm movq %rax,%dr0 251db527225SPeter Wemm /* But preserve reserved bits in %dr7 */ 252db527225SPeter Wemm movq %dr7,%rax 253db527225SPeter Wemm andq $0x0000fc00,%rax 254db527225SPeter Wemm movq PCB_DR7(%r8),%rcx 255db527225SPeter Wemm andq $~0x0000fc00,%rcx 256db527225SPeter Wemm orq %rcx,%rax 257db527225SPeter Wemm movq %rax,%dr7 258db527225SPeter Wemm1: 2590967373eSDavid Greenman ret 260ea497502SJoseph KoshyEND(cpu_switch) 2610967373eSDavid Greenman 2620967373eSDavid Greenman/* 2632924d491SDavid Greenman * savectx(pcb) 2642924d491SDavid Greenman * Update pcb, saving current processor state. 2650967373eSDavid Greenman */ 2660967373eSDavid GreenmanENTRY(savectx) 2679869fa1dSJohn Baldwin /* Fetch PCB. */ 268afa88623SPeter Wemm movq %rdi,%rcx 2692924d491SDavid Greenman 270afa88623SPeter Wemm /* Save caller's return address. */ 271afa88623SPeter Wemm movq (%rsp),%rax 272afa88623SPeter Wemm movq %rax,PCB_RIP(%rcx) 2732924d491SDavid Greenman 274afa88623SPeter Wemm movq %cr3,%rax 275afa88623SPeter Wemm movq %rax,PCB_CR3(%rcx) 276ac5f943cSPeter Wemm 277afa88623SPeter Wemm movq %rbx,PCB_RBX(%rcx) 278afa88623SPeter Wemm movq %rsp,PCB_RSP(%rcx) 279afa88623SPeter Wemm movq %rbp,PCB_RBP(%rcx) 280afa88623SPeter Wemm movq %r12,PCB_R12(%rcx) 281afa88623SPeter Wemm movq %r13,PCB_R13(%rcx) 282afa88623SPeter Wemm movq %r14,PCB_R14(%rcx) 283afa88623SPeter Wemm movq %r15,PCB_R15(%rcx) 2840967373eSDavid Greenman 2850967373eSDavid Greenman /* 286bf2f09eeSPeter Wemm * If fpcurthread == NULL, then the fpu h/w state is irrelevant and the 2870967373eSDavid Greenman * state had better already be in the pcb. This is true for forks 2880967373eSDavid Greenman * but not for dumps (the old book-keeping with FP flags in the pcb 2890967373eSDavid Greenman * always lost for dumps because the dump pcb has 0 flags). 2900967373eSDavid Greenman * 291bf2f09eeSPeter Wemm * If fpcurthread != NULL, then we have to save the fpu h/w state to 2920bbc8826SJohn Baldwin * fpcurthread's pcb and copy it to the requested pcb, or save to the 2930967373eSDavid Greenman * requested pcb and reload. Copying is easier because we would 2940967373eSDavid Greenman * have to handle h/w bugs for reloading. We used to lose the 295bf2f09eeSPeter Wemm * parent's fpu state for forks by forgetting to reload. 2960967373eSDavid Greenman */ 297afa88623SPeter Wemm pushfq 298c2b095abSBruce Evans cli 299afa88623SPeter Wemm movq PCPU(FPCURTHREAD),%rax 300afa88623SPeter Wemm testq %rax,%rax 3010967373eSDavid Greenman je 1f 3020967373eSDavid Greenman 303afa88623SPeter Wemm movq TD_PCB(%rax),%rdi 304afa88623SPeter Wemm leaq PCB_SAVEFPU(%rdi),%rdi 305db527225SPeter Wemm clts 306db527225SPeter Wemm fxsave (%rdi) 307db527225SPeter Wemm smsw %ax 308db527225SPeter Wemm orb $CR0_TS,%al 309db527225SPeter Wemm lmsw %ax 3100967373eSDavid Greenman 311afa88623SPeter Wemm movq $PCB_SAVEFPU_SIZE,%rdx /* arg 3 */ 312afa88623SPeter Wemm leaq PCB_SAVEFPU(%rcx),%rsi /* arg 2 */ 313db527225SPeter Wemm /* arg 1 (%rdi) already loaded */ 31402318dacSJake Burkholder call bcopy 315c2b095abSBruce Evans1: 316afa88623SPeter Wemm popfq 3170967373eSDavid Greenman 3180967373eSDavid Greenman ret 319ea497502SJoseph KoshyEND(savectx) 320