10967373eSDavid Greenman/*- 2fcfe57d6SPeter Wemm * Copyright (c) 2003 Peter Wemm. 30967373eSDavid Greenman * Copyright (c) 1990 The Regents of the University of California. 40967373eSDavid Greenman * All rights reserved. 50967373eSDavid Greenman * 60967373eSDavid Greenman * This code is derived from software contributed to Berkeley by 70967373eSDavid Greenman * William Jolitz. 80967373eSDavid Greenman * 90967373eSDavid Greenman * Redistribution and use in source and binary forms, with or without 100967373eSDavid Greenman * modification, are permitted provided that the following conditions 110967373eSDavid Greenman * are met: 120967373eSDavid Greenman * 1. Redistributions of source code must retain the above copyright 130967373eSDavid Greenman * notice, this list of conditions and the following disclaimer. 140967373eSDavid Greenman * 2. Redistributions in binary form must reproduce the above copyright 150967373eSDavid Greenman * notice, this list of conditions and the following disclaimer in the 160967373eSDavid Greenman * documentation and/or other materials provided with the distribution. 17fbbd9655SWarner Losh * 3. Neither the name of the University nor the names of its contributors 180967373eSDavid Greenman * may be used to endorse or promote products derived from this software 190967373eSDavid Greenman * without specific prior written permission. 200967373eSDavid Greenman * 210967373eSDavid Greenman * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 220967373eSDavid Greenman * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 230967373eSDavid Greenman * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 240967373eSDavid Greenman * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 250967373eSDavid Greenman * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 260967373eSDavid Greenman * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 270967373eSDavid Greenman * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 280967373eSDavid Greenman * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 290967373eSDavid Greenman * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 300967373eSDavid Greenman * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 310967373eSDavid Greenman * SUCH DAMAGE. 320967373eSDavid Greenman */ 330967373eSDavid Greenman 340513ce7fSBruce Evans#include <machine/asmacros.h> 35db527225SPeter Wemm#include <machine/specialreg.h> 36477a642cSPeter Wemm 37fc2a8776SEd Maste#include "assym.inc" 3840380a6aSJeff Roberson#include "opt_sched.h" 390513ce7fSBruce Evans 400967373eSDavid Greenman/*****************************************************************************/ 410967373eSDavid Greenman/* Scheduling */ 420967373eSDavid Greenman/*****************************************************************************/ 430967373eSDavid Greenman 440967373eSDavid Greenman .text 450967373eSDavid Greenman 460967373eSDavid Greenman/* 470384fff8SJason Evans * cpu_throw() 48e602ba25SJulian Elischer * 492c87e001SPeter Wemm * This is the second half of cpu_switch(). It is used when the current 50e602ba25SJulian Elischer * thread is either a dummy or slated to die, and we no longer care 51cc66ebe2SPeter Wemm * about its state. This is only a slight optimization and is probably 52cc66ebe2SPeter Wemm * not worth it anymore. Note that we need to clear the pm_active bits so 53cc66ebe2SPeter Wemm * we do need the old proc if it still exists. 54afa88623SPeter Wemm * %rdi = oldtd 55afa88623SPeter Wemm * %rsi = newtd 560384fff8SJason Evans */ 570384fff8SJason EvansENTRY(cpu_throw) 58a546448bSKonstantin Belousov movq %rsi,%r12 59a546448bSKonstantin Belousov movq %rsi,%rdi 60a546448bSKonstantin Belousov call pmap_activate_sw 61a546448bSKonstantin Belousov jmp sw1 62ea497502SJoseph KoshyEND(cpu_throw) 630384fff8SJason Evans 640384fff8SJason Evans/* 655d68dad3SJeff Roberson * cpu_switch(old, new, mtx) 66e602ba25SJulian Elischer * 67e602ba25SJulian Elischer * Save the current thread state, then select the next thread to run 68e602ba25SJulian Elischer * and load its state. 69afa88623SPeter Wemm * %rdi = oldtd 70afa88623SPeter Wemm * %rsi = newtd 715d68dad3SJeff Roberson * %rdx = mtx 720967373eSDavid Greenman */ 7326f9a767SRodney W. GrimesENTRY(cpu_switch) 74cc66ebe2SPeter Wemm /* Switch to new thread. First, save context. */ 755e921ff4SKonstantin Belousov leaq TD_MD_PCB(%rdi),%r8 760967373eSDavid Greenman 77afa88623SPeter Wemm movq (%rsp),%rax /* Hardware registers */ 78bf1e8974SPeter Wemm movq %r15,PCB_R15(%r8) 79f001eabfSPeter Wemm movq %r14,PCB_R14(%r8) 80f001eabfSPeter Wemm movq %r13,PCB_R13(%r8) 81f001eabfSPeter Wemm movq %r12,PCB_R12(%r8) 82f001eabfSPeter Wemm movq %rbp,PCB_RBP(%r8) 83f001eabfSPeter Wemm movq %rsp,PCB_RSP(%r8) 84f001eabfSPeter Wemm movq %rbx,PCB_RBX(%r8) 85f001eabfSPeter Wemm movq %rax,PCB_RIP(%r8) 86f98c3ea7SKonstantin Belousov 873e902b3dSKonstantin Belousov testl $PCB_FULL_IRET,PCB_FLAGS(%r8) 883e902b3dSKonstantin Belousov jnz 2f 893e902b3dSKonstantin Belousov orl $PCB_FULL_IRET,PCB_FLAGS(%r8) 903e902b3dSKonstantin Belousov testl $TDP_KTHREAD,TD_PFLAGS(%rdi) 913e902b3dSKonstantin Belousov jnz 2f 923e902b3dSKonstantin Belousov testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) 933e902b3dSKonstantin Belousov jz 2f 943e902b3dSKonstantin Belousov movl %fs,%eax 953e902b3dSKonstantin Belousov cmpl $KUF32SEL,%eax 963e902b3dSKonstantin Belousov jne 1f 970d4e7ec5SRyan Libby rdfsbase %rax 983e902b3dSKonstantin Belousov movq %rax,PCB_FSBASE(%r8) 993e902b3dSKonstantin Belousov1: movl %gs,%eax 1003e902b3dSKonstantin Belousov cmpl $KUG32SEL,%eax 1013e902b3dSKonstantin Belousov jne 2f 1023e902b3dSKonstantin Belousov movq %rdx,%r12 1033e902b3dSKonstantin Belousov movl $MSR_KGSBASE,%ecx /* Read user gs base */ 1043e902b3dSKonstantin Belousov rdmsr 1053e902b3dSKonstantin Belousov shlq $32,%rdx 1063e902b3dSKonstantin Belousov orq %rdx,%rax 1073e902b3dSKonstantin Belousov movq %rax,PCB_GSBASE(%r8) 1083e902b3dSKonstantin Belousov movq %r12,%rdx 1093e902b3dSKonstantin Belousov 1103e902b3dSKonstantin Belousov2: 11150e3cec3SJung-uk Kim testl $PCB_DBREGS,PCB_FLAGS(%r8) 112f001eabfSPeter Wemm jnz store_dr /* static predict not taken */ 113f001eabfSPeter Wemmdone_store_dr: 114db527225SPeter Wemm 1150967373eSDavid Greenman /* have we used fp, and need a save? */ 116afa88623SPeter Wemm cmpq %rdi,PCPU(FPCURTHREAD) 117df013409SKonstantin Belousov jne ctx_switch_fpusave_done 118df013409SKonstantin Belousov movq PCB_SAVEFPU(%r8),%r9 119db527225SPeter Wemm clts 120d1a07e31SKonstantin Belousov cmpl $0,use_xsave(%rip) 1218c6f8f3dSKonstantin Belousov jne 1f 122df013409SKonstantin Belousov fxsave (%r9) 123df013409SKonstantin Belousov jmp ctx_switch_fpusave_done 1248c6f8f3dSKonstantin Belousov1: movq %rdx,%rcx 1258c6f8f3dSKonstantin Belousov movl xsave_mask,%eax 1268c6f8f3dSKonstantin Belousov movl xsave_mask+4,%edx 127df013409SKonstantin Belousov testl $PCB_32BIT,PCB_FLAGS(%r8) 128df013409SKonstantin Belousov jne ctx_switch_xsave32 129333d0c60SKonstantin Belousov .globl ctx_switch_xsave 130333d0c60SKonstantin Belousovctx_switch_xsave: 131333d0c60SKonstantin Belousov /* This is patched to xsaveopt if supported, see fpuinit_bsp1() */ 132df013409SKonstantin Belousov xsave64 (%r9) 133df013409SKonstantin Belousovctx_switch_xsave_done: 1348c6f8f3dSKonstantin Belousov movq %rcx,%rdx 135df013409SKonstantin Belousovctx_switch_fpusave_done: 136cc66ebe2SPeter Wemm /* Save is done. Now fire up new thread. Leave old vmspace. */ 137a546448bSKonstantin Belousov movq %rsi,%r12 138a546448bSKonstantin Belousov movq %rdi,%r13 139a546448bSKonstantin Belousov movq %rdx,%r15 140a546448bSKonstantin Belousov movq %rsi,%rdi 141a546448bSKonstantin Belousov callq pmap_activate_sw 14273dd84a7SMateusz Guzik movq %r15,TD_LOCK(%r13) /* Release the old thread */ 143cc66ebe2SPeter Wemmsw1: 1445e921ff4SKonstantin Belousov leaq TD_MD_PCB(%r12),%r8 14540380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 14640380a6aSJeff Roberson movq $blocked_lock, %rdx 147a546448bSKonstantin Belousov movq TD_LOCK(%r12),%rcx 14840380a6aSJeff Roberson cmpq %rcx, %rdx 14973dd84a7SMateusz Guzik je sw1wait 15073dd84a7SMateusz Guziksw1cont: 15140380a6aSJeff Roberson#endif 152cc66ebe2SPeter Wemm /* 153cc66ebe2SPeter Wemm * At this point, we've switched address spaces and are ready 154cc66ebe2SPeter Wemm * to load up the rest of the next context. 155cc66ebe2SPeter Wemm */ 156bf1e8974SPeter Wemm 157843d5752SKonstantin Belousov /* Skip loading LDT and user fsbase/gsbase for kthreads */ 158a546448bSKonstantin Belousov testl $TDP_KTHREAD,TD_PFLAGS(%r12) 159f001eabfSPeter Wemm jnz do_kthread 160f001eabfSPeter Wemm 1612c66cccaSKonstantin Belousov /* 1622c66cccaSKonstantin Belousov * Load ldt register 1632c66cccaSKonstantin Belousov */ 164a546448bSKonstantin Belousov movq TD_PROC(%r12),%rcx 1652c66cccaSKonstantin Belousov cmpq $0, P_MD+MD_LDT(%rcx) 1662c66cccaSKonstantin Belousov jne do_ldt 1672c66cccaSKonstantin Belousov xorl %eax,%eax 1682c66cccaSKonstantin Belousovld_ldt: lldt %ax 1698f4a1f3aSKonstantin Belousov 1702c66cccaSKonstantin Belousov /* Restore fs base in GDT */ 171bf1e8974SPeter Wemm movl PCB_FSBASE(%r8),%eax 1722c66cccaSKonstantin Belousov movq PCPU(FS32P),%rdx 1732c66cccaSKonstantin Belousov movw %ax,2(%rdx) 1742c66cccaSKonstantin Belousov shrl $16,%eax 1752c66cccaSKonstantin Belousov movb %al,4(%rdx) 1762c66cccaSKonstantin Belousov shrl $8,%eax 1772c66cccaSKonstantin Belousov movb %al,7(%rdx) 1789869fa1dSJohn Baldwin 1792c66cccaSKonstantin Belousov /* Restore gs base in GDT */ 1802c66cccaSKonstantin Belousov movl PCB_GSBASE(%r8),%eax 1812c66cccaSKonstantin Belousov movq PCPU(GS32P),%rdx 1822c66cccaSKonstantin Belousov movw %ax,2(%rdx) 1832c66cccaSKonstantin Belousov shrl $16,%eax 1842c66cccaSKonstantin Belousov movb %al,4(%rdx) 1852c66cccaSKonstantin Belousov shrl $8,%eax 1862c66cccaSKonstantin Belousov movb %al,7(%rdx) 1872c66cccaSKonstantin Belousov 1882c66cccaSKonstantin Belousovdo_kthread: 1892c66cccaSKonstantin Belousov /* Do we need to reload tss ? */ 1900d2a2989SPeter Wemm movq PCPU(TSSP),%rax 1912c66cccaSKonstantin Belousov movq PCB_TSSP(%r8),%rdx 19298158c75SKonstantin Belousov movq PCPU(PRVSPACE),%r13 19398158c75SKonstantin Belousov addq $PC_COMMONTSS,%r13 1942c66cccaSKonstantin Belousov testq %rdx,%rdx 19598158c75SKonstantin Belousov cmovzq %r13,%rdx 1962c66cccaSKonstantin Belousov cmpq %rax,%rdx 1972c66cccaSKonstantin Belousov jne do_tss 1982c66cccaSKonstantin Belousovdone_tss: 1995e921ff4SKonstantin Belousov movq TD_MD_STACK_BASE(%r12),%r9 2005e921ff4SKonstantin Belousov movq %r9,PCPU(RSP0) 2014e32b7b3SDavid Xu movq %r8,PCPU(CURPCB) 20227275f8aSTycho Nightingale movq PCPU(PTI_RSP0),%rax 203a9262f49SKonstantin Belousov cmpq $~0,PCPU(UCR3) 2045e921ff4SKonstantin Belousov cmove %r9,%rax 20527275f8aSTycho Nightingale movq %rax,TSS_RSP0(%rdx) 206a9262f49SKonstantin Belousov movq %r12,PCPU(CURTHREAD) /* into next thread */ 207f001eabfSPeter Wemm 208f001eabfSPeter Wemm /* Test if debug registers should be restored. */ 20950e3cec3SJung-uk Kim testl $PCB_DBREGS,PCB_FLAGS(%r8) 210f001eabfSPeter Wemm jnz load_dr /* static predict not taken */ 211f001eabfSPeter Wemmdone_load_dr: 2124e32b7b3SDavid Xu 213f001eabfSPeter Wemm /* Restore context. */ 214f001eabfSPeter Wemm movq PCB_R15(%r8),%r15 215f001eabfSPeter Wemm movq PCB_R14(%r8),%r14 216f001eabfSPeter Wemm movq PCB_R13(%r8),%r13 217f001eabfSPeter Wemm movq PCB_R12(%r8),%r12 218f001eabfSPeter Wemm movq PCB_RBP(%r8),%rbp 219f001eabfSPeter Wemm movq PCB_RSP(%r8),%rsp 220f001eabfSPeter Wemm movq PCB_RBX(%r8),%rbx 221f001eabfSPeter Wemm movq PCB_RIP(%r8),%rax 222f001eabfSPeter Wemm movq %rax,(%rsp) 223d1a07e31SKonstantin Belousov movq PCPU(CURTHREAD),%rdi 224d1a07e31SKonstantin Belousov call fpu_activate_sw 225ea602083SKonstantin Belousov cmpb $0,cpu_flush_rsb_ctxsw(%rip) 226ea602083SKonstantin Belousov jne rsb_flush 227f001eabfSPeter Wemm ret 228f001eabfSPeter Wemm 229f001eabfSPeter Wemm /* 230f001eabfSPeter Wemm * We order these strangely for several reasons. 231f001eabfSPeter Wemm * 1: I wanted to use static branch prediction hints 232f001eabfSPeter Wemm * 2: Most athlon64/opteron cpus don't have them. They define 233f001eabfSPeter Wemm * a forward branch as 'predict not taken'. Intel cores have 234f001eabfSPeter Wemm * the 'rep' prefix to invert this. 235f001eabfSPeter Wemm * So, to make it work on both forms of cpu we do the detour. 236f001eabfSPeter Wemm * We use jumps rather than call in order to avoid the stack. 237f001eabfSPeter Wemm */ 238f001eabfSPeter Wemm 239f001eabfSPeter Wemmstore_dr: 240f001eabfSPeter Wemm movq %dr7,%rax /* yes, do the save */ 241f001eabfSPeter Wemm movq %dr0,%r15 242f001eabfSPeter Wemm movq %dr1,%r14 243f001eabfSPeter Wemm movq %dr2,%r13 244f001eabfSPeter Wemm movq %dr3,%r12 245f001eabfSPeter Wemm movq %dr6,%r11 246f001eabfSPeter Wemm movq %r15,PCB_DR0(%r8) 247f001eabfSPeter Wemm movq %r14,PCB_DR1(%r8) 248f001eabfSPeter Wemm movq %r13,PCB_DR2(%r8) 249f001eabfSPeter Wemm movq %r12,PCB_DR3(%r8) 250f001eabfSPeter Wemm movq %r11,PCB_DR6(%r8) 251f001eabfSPeter Wemm movq %rax,PCB_DR7(%r8) 252aaa95ccbSKonstantin Belousov andq $0x0000fc00, %rax /* disable all watchpoints */ 253db527225SPeter Wemm movq %rax,%dr7 254f001eabfSPeter Wemm jmp done_store_dr 255f001eabfSPeter Wemm 256f001eabfSPeter Wemmload_dr: 257f001eabfSPeter Wemm movq %dr7,%rax 258f001eabfSPeter Wemm movq PCB_DR0(%r8),%r15 259f001eabfSPeter Wemm movq PCB_DR1(%r8),%r14 260f001eabfSPeter Wemm movq PCB_DR2(%r8),%r13 261f001eabfSPeter Wemm movq PCB_DR3(%r8),%r12 262f001eabfSPeter Wemm movq PCB_DR6(%r8),%r11 263f001eabfSPeter Wemm movq PCB_DR7(%r8),%rcx 264f001eabfSPeter Wemm movq %r15,%dr0 265f001eabfSPeter Wemm movq %r14,%dr1 266f001eabfSPeter Wemm /* Preserve reserved bits in %dr7 */ 267f001eabfSPeter Wemm andq $0x0000fc00,%rax 268f001eabfSPeter Wemm andq $~0x0000fc00,%rcx 269f001eabfSPeter Wemm movq %r13,%dr2 270f001eabfSPeter Wemm movq %r12,%dr3 271f001eabfSPeter Wemm orq %rcx,%rax 272f001eabfSPeter Wemm movq %r11,%dr6 273f001eabfSPeter Wemm movq %rax,%dr7 274f001eabfSPeter Wemm jmp done_load_dr 2752c66cccaSKonstantin Belousov 2762c66cccaSKonstantin Belousovdo_tss: movq %rdx,PCPU(TSSP) 2772c66cccaSKonstantin Belousov movq %rdx,%rcx 2782c66cccaSKonstantin Belousov movq PCPU(TSS),%rax 279cfe92f33SDimitry Andric movw %cx,2(%rax) 2802c66cccaSKonstantin Belousov shrq $16,%rcx 2812c66cccaSKonstantin Belousov movb %cl,4(%rax) 2822c66cccaSKonstantin Belousov shrq $8,%rcx 2832c66cccaSKonstantin Belousov movb %cl,7(%rax) 2842c66cccaSKonstantin Belousov shrq $8,%rcx 2852c66cccaSKonstantin Belousov movl %ecx,8(%rax) 2862c66cccaSKonstantin Belousov movb $0x89,5(%rax) /* unset busy */ 28727275f8aSTycho Nightingale movl $TSSSEL,%eax 2882c66cccaSKonstantin Belousov ltr %ax 2892c66cccaSKonstantin Belousov jmp done_tss 2902c66cccaSKonstantin Belousov 2912c66cccaSKonstantin Belousovdo_ldt: movq PCPU(LDT),%rax 2922c66cccaSKonstantin Belousov movq P_MD+MD_LDT_SD(%rcx),%rdx 2932c66cccaSKonstantin Belousov movq %rdx,(%rax) 2942c66cccaSKonstantin Belousov movq P_MD+MD_LDT_SD+8(%rcx),%rdx 2952c66cccaSKonstantin Belousov movq %rdx,8(%rax) 2962c66cccaSKonstantin Belousov movl $LDTSEL,%eax 2972c66cccaSKonstantin Belousov jmp ld_ldt 298df013409SKonstantin Belousov 299df013409SKonstantin Belousov .globl ctx_switch_xsave32 300df013409SKonstantin Belousovctx_switch_xsave32: 301df013409SKonstantin Belousov xsave (%r9) 302df013409SKonstantin Belousov jmp ctx_switch_xsave_done 303ea497502SJoseph KoshyEND(cpu_switch) 3040967373eSDavid Greenman 3050967373eSDavid Greenman/* 3062924d491SDavid Greenman * savectx(pcb) 3072924d491SDavid Greenman * Update pcb, saving current processor state. 3080967373eSDavid Greenman */ 3090967373eSDavid GreenmanENTRY(savectx) 310afa88623SPeter Wemm /* Save caller's return address. */ 311afa88623SPeter Wemm movq (%rsp),%rax 3123ab42a25SJung-uk Kim movq %rax,PCB_RIP(%rdi) 3132924d491SDavid Greenman 3143ab42a25SJung-uk Kim movq %rbx,PCB_RBX(%rdi) 3153ab42a25SJung-uk Kim movq %rsp,PCB_RSP(%rdi) 3163ab42a25SJung-uk Kim movq %rbp,PCB_RBP(%rdi) 3173ab42a25SJung-uk Kim movq %r12,PCB_R12(%rdi) 3183ab42a25SJung-uk Kim movq %r13,PCB_R13(%rdi) 3193ab42a25SJung-uk Kim movq %r14,PCB_R14(%rdi) 3203ab42a25SJung-uk Kim movq %r15,PCB_R15(%rdi) 321a2d2c836SJung-uk Kim 32205acaa9fSJung-uk Kim movq %cr0,%rax 32305acaa9fSJung-uk Kim movq %rax,PCB_CR0(%rdi) 324a2d2c836SJung-uk Kim movq %cr2,%rax 3253ab42a25SJung-uk Kim movq %rax,PCB_CR2(%rdi) 326afa88623SPeter Wemm movq %cr3,%rax 3273ab42a25SJung-uk Kim movq %rax,PCB_CR3(%rdi) 328a2d2c836SJung-uk Kim movq %cr4,%rax 3293ab42a25SJung-uk Kim movq %rax,PCB_CR4(%rdi) 330ac5f943cSPeter Wemm 331a2d2c836SJung-uk Kim movq %dr0,%rax 3323ab42a25SJung-uk Kim movq %rax,PCB_DR0(%rdi) 333a2d2c836SJung-uk Kim movq %dr1,%rax 3343ab42a25SJung-uk Kim movq %rax,PCB_DR1(%rdi) 335a2d2c836SJung-uk Kim movq %dr2,%rax 3363ab42a25SJung-uk Kim movq %rax,PCB_DR2(%rdi) 337a2d2c836SJung-uk Kim movq %dr3,%rax 3383ab42a25SJung-uk Kim movq %rax,PCB_DR3(%rdi) 339a2d2c836SJung-uk Kim movq %dr6,%rax 3403ab42a25SJung-uk Kim movq %rax,PCB_DR6(%rdi) 341a2d2c836SJung-uk Kim movq %dr7,%rax 3423ab42a25SJung-uk Kim movq %rax,PCB_DR7(%rdi) 343a2d2c836SJung-uk Kim 344a2d2c836SJung-uk Kim movl $MSR_FSBASE,%ecx 345a2d2c836SJung-uk Kim rdmsr 346305c5c0aSJung-uk Kim movl %eax,PCB_FSBASE(%rdi) 347305c5c0aSJung-uk Kim movl %edx,PCB_FSBASE+4(%rdi) 348a2d2c836SJung-uk Kim movl $MSR_GSBASE,%ecx 349a2d2c836SJung-uk Kim rdmsr 350305c5c0aSJung-uk Kim movl %eax,PCB_GSBASE(%rdi) 351305c5c0aSJung-uk Kim movl %edx,PCB_GSBASE+4(%rdi) 352a2d2c836SJung-uk Kim movl $MSR_KGSBASE,%ecx 353a2d2c836SJung-uk Kim rdmsr 354305c5c0aSJung-uk Kim movl %eax,PCB_KGSBASE(%rdi) 355305c5c0aSJung-uk Kim movl %edx,PCB_KGSBASE+4(%rdi) 356fb864578SMitsuru IWASAKI movl $MSR_EFER,%ecx 357fb864578SMitsuru IWASAKI rdmsr 358fb864578SMitsuru IWASAKI movl %eax,PCB_EFER(%rdi) 359fb864578SMitsuru IWASAKI movl %edx,PCB_EFER+4(%rdi) 360fb864578SMitsuru IWASAKI movl $MSR_STAR,%ecx 361fb864578SMitsuru IWASAKI rdmsr 362fb864578SMitsuru IWASAKI movl %eax,PCB_STAR(%rdi) 363fb864578SMitsuru IWASAKI movl %edx,PCB_STAR+4(%rdi) 364fb864578SMitsuru IWASAKI movl $MSR_LSTAR,%ecx 365fb864578SMitsuru IWASAKI rdmsr 366fb864578SMitsuru IWASAKI movl %eax,PCB_LSTAR(%rdi) 367fb864578SMitsuru IWASAKI movl %edx,PCB_LSTAR+4(%rdi) 368fb864578SMitsuru IWASAKI movl $MSR_CSTAR,%ecx 369fb864578SMitsuru IWASAKI rdmsr 370fb864578SMitsuru IWASAKI movl %eax,PCB_CSTAR(%rdi) 371fb864578SMitsuru IWASAKI movl %edx,PCB_CSTAR+4(%rdi) 372fb864578SMitsuru IWASAKI movl $MSR_SF_MASK,%ecx 373fb864578SMitsuru IWASAKI rdmsr 374fb864578SMitsuru IWASAKI movl %eax,PCB_SFMASK(%rdi) 375fb864578SMitsuru IWASAKI movl %edx,PCB_SFMASK+4(%rdi) 376a2d2c836SJung-uk Kim 3773ab42a25SJung-uk Kim sgdt PCB_GDT(%rdi) 3783ab42a25SJung-uk Kim sidt PCB_IDT(%rdi) 3793ab42a25SJung-uk Kim sldt PCB_LDT(%rdi) 3803ab42a25SJung-uk Kim str PCB_TR(%rdi) 381a2d2c836SJung-uk Kim 382a2d2c836SJung-uk Kim movl $1,%eax 3830967373eSDavid Greenman ret 384ea497502SJoseph KoshyEND(savectx) 3858c6f8f3dSKonstantin Belousov 3868c6f8f3dSKonstantin Belousov/* 3876ad79910SJung-uk Kim * resumectx(pcb) 388fb864578SMitsuru IWASAKI * Resuming processor state from pcb. 389fb864578SMitsuru IWASAKI */ 390fb864578SMitsuru IWASAKIENTRY(resumectx) 391f446480bSKonstantin Belousov /* Switch to KPML5/4phys. */ 3926ad79910SJung-uk Kim movq KPML4phys,%rax 393f446480bSKonstantin Belousov movq KPML5phys,%rcx 394f446480bSKonstantin Belousov cmpl $0, la57 395f446480bSKonstantin Belousov cmovne %rcx, %rax 3966ad79910SJung-uk Kim movq %rax,%cr3 397fb864578SMitsuru IWASAKI 398fb864578SMitsuru IWASAKI /* Force kernel segment registers. */ 399fb864578SMitsuru IWASAKI movl $KDSEL,%eax 400fb864578SMitsuru IWASAKI movw %ax,%ds 401fb864578SMitsuru IWASAKI movw %ax,%es 402fb864578SMitsuru IWASAKI movw %ax,%ss 403fb864578SMitsuru IWASAKI movl $KUF32SEL,%eax 404fb864578SMitsuru IWASAKI movw %ax,%fs 405fb864578SMitsuru IWASAKI movl $KUG32SEL,%eax 406fb864578SMitsuru IWASAKI movw %ax,%gs 407fb864578SMitsuru IWASAKI 408fb864578SMitsuru IWASAKI movl $MSR_FSBASE,%ecx 409fb864578SMitsuru IWASAKI movl PCB_FSBASE(%rdi),%eax 410fb864578SMitsuru IWASAKI movl 4 + PCB_FSBASE(%rdi),%edx 411fb864578SMitsuru IWASAKI wrmsr 412fb864578SMitsuru IWASAKI movl $MSR_GSBASE,%ecx 413fb864578SMitsuru IWASAKI movl PCB_GSBASE(%rdi),%eax 414fb864578SMitsuru IWASAKI movl 4 + PCB_GSBASE(%rdi),%edx 415fb864578SMitsuru IWASAKI wrmsr 416fb864578SMitsuru IWASAKI movl $MSR_KGSBASE,%ecx 417fb864578SMitsuru IWASAKI movl PCB_KGSBASE(%rdi),%eax 418fb864578SMitsuru IWASAKI movl 4 + PCB_KGSBASE(%rdi),%edx 419fb864578SMitsuru IWASAKI wrmsr 420fb864578SMitsuru IWASAKI 421bd101a66SKonstantin Belousov /* Restore EFER one more time. */ 422fb864578SMitsuru IWASAKI movl $MSR_EFER,%ecx 423fb864578SMitsuru IWASAKI movl PCB_EFER(%rdi),%eax 424fb864578SMitsuru IWASAKI wrmsr 425fb864578SMitsuru IWASAKI 426fb864578SMitsuru IWASAKI /* Restore fast syscall stuff. */ 427fb864578SMitsuru IWASAKI movl $MSR_STAR,%ecx 428fb864578SMitsuru IWASAKI movl PCB_STAR(%rdi),%eax 429fb864578SMitsuru IWASAKI movl 4 + PCB_STAR(%rdi),%edx 430fb864578SMitsuru IWASAKI wrmsr 431fb864578SMitsuru IWASAKI movl $MSR_LSTAR,%ecx 432fb864578SMitsuru IWASAKI movl PCB_LSTAR(%rdi),%eax 433fb864578SMitsuru IWASAKI movl 4 + PCB_LSTAR(%rdi),%edx 434fb864578SMitsuru IWASAKI wrmsr 435fb864578SMitsuru IWASAKI movl $MSR_CSTAR,%ecx 436fb864578SMitsuru IWASAKI movl PCB_CSTAR(%rdi),%eax 437fb864578SMitsuru IWASAKI movl 4 + PCB_CSTAR(%rdi),%edx 438fb864578SMitsuru IWASAKI wrmsr 439fb864578SMitsuru IWASAKI movl $MSR_SF_MASK,%ecx 440fb864578SMitsuru IWASAKI movl PCB_SFMASK(%rdi),%eax 441fb864578SMitsuru IWASAKI wrmsr 442fb864578SMitsuru IWASAKI 443b1d735baSJohn Baldwin /* Restore CR0, CR2, CR4 and CR3. */ 444fb864578SMitsuru IWASAKI movq PCB_CR0(%rdi),%rax 445fb864578SMitsuru IWASAKI movq %rax,%cr0 446fb864578SMitsuru IWASAKI movq PCB_CR2(%rdi),%rax 447fb864578SMitsuru IWASAKI movq %rax,%cr2 448fb864578SMitsuru IWASAKI movq PCB_CR4(%rdi),%rax 449fb864578SMitsuru IWASAKI movq %rax,%cr4 450fb864578SMitsuru IWASAKI movq PCB_CR3(%rdi),%rax 451fb864578SMitsuru IWASAKI movq %rax,%cr3 452fb864578SMitsuru IWASAKI 453fb864578SMitsuru IWASAKI /* Restore descriptor tables. */ 454fb864578SMitsuru IWASAKI lidt PCB_IDT(%rdi) 455fb864578SMitsuru IWASAKI lldt PCB_LDT(%rdi) 456fb864578SMitsuru IWASAKI 457fb864578SMitsuru IWASAKI#define SDT_SYSTSS 9 458fb864578SMitsuru IWASAKI#define SDT_SYSBSY 11 459fb864578SMitsuru IWASAKI 460fb864578SMitsuru IWASAKI /* Clear "task busy" bit and reload TR. */ 461fb864578SMitsuru IWASAKI movq PCPU(TSS),%rax 462fb864578SMitsuru IWASAKI andb $(~SDT_SYSBSY | SDT_SYSTSS),5(%rax) 463fb864578SMitsuru IWASAKI movw PCB_TR(%rdi),%ax 464fb864578SMitsuru IWASAKI ltr %ax 465fb864578SMitsuru IWASAKI 466fb864578SMitsuru IWASAKI#undef SDT_SYSTSS 467fb864578SMitsuru IWASAKI#undef SDT_SYSBSY 468fb864578SMitsuru IWASAKI 469fb864578SMitsuru IWASAKI /* Restore debug registers. */ 470fb864578SMitsuru IWASAKI movq PCB_DR0(%rdi),%rax 471fb864578SMitsuru IWASAKI movq %rax,%dr0 472fb864578SMitsuru IWASAKI movq PCB_DR1(%rdi),%rax 473fb864578SMitsuru IWASAKI movq %rax,%dr1 474fb864578SMitsuru IWASAKI movq PCB_DR2(%rdi),%rax 475fb864578SMitsuru IWASAKI movq %rax,%dr2 476fb864578SMitsuru IWASAKI movq PCB_DR3(%rdi),%rax 477fb864578SMitsuru IWASAKI movq %rax,%dr3 478fb864578SMitsuru IWASAKI movq PCB_DR6(%rdi),%rax 479fb864578SMitsuru IWASAKI movq %rax,%dr6 480fb864578SMitsuru IWASAKI movq PCB_DR7(%rdi),%rax 481fb864578SMitsuru IWASAKI movq %rax,%dr7 482fb864578SMitsuru IWASAKI 483fb864578SMitsuru IWASAKI /* Restore other callee saved registers. */ 484fb864578SMitsuru IWASAKI movq PCB_R15(%rdi),%r15 485fb864578SMitsuru IWASAKI movq PCB_R14(%rdi),%r14 486fb864578SMitsuru IWASAKI movq PCB_R13(%rdi),%r13 487fb864578SMitsuru IWASAKI movq PCB_R12(%rdi),%r12 488fb864578SMitsuru IWASAKI movq PCB_RBP(%rdi),%rbp 489fb864578SMitsuru IWASAKI movq PCB_RSP(%rdi),%rsp 490fb864578SMitsuru IWASAKI movq PCB_RBX(%rdi),%rbx 491fb864578SMitsuru IWASAKI 492fb864578SMitsuru IWASAKI /* Restore return address. */ 493fb864578SMitsuru IWASAKI movq PCB_RIP(%rdi),%rax 494fb864578SMitsuru IWASAKI movq %rax,(%rsp) 495fb864578SMitsuru IWASAKI 496fb864578SMitsuru IWASAKI xorl %eax,%eax 497fb864578SMitsuru IWASAKI ret 498fb864578SMitsuru IWASAKIEND(resumectx) 49973dd84a7SMateusz Guzik 50073dd84a7SMateusz Guzik/* Wait for the new thread to become unblocked */ 50173dd84a7SMateusz Guzik#if defined(SCHED_ULE) && defined(SMP) 50273dd84a7SMateusz Guziksw1wait: 50373dd84a7SMateusz Guzik1: 50473dd84a7SMateusz Guzik pause 50573dd84a7SMateusz Guzik movq TD_LOCK(%r12),%rcx 50673dd84a7SMateusz Guzik cmpq %rcx, %rdx 50773dd84a7SMateusz Guzik je 1b 50873dd84a7SMateusz Guzik jmp sw1cont 50973dd84a7SMateusz Guzik#endif 510