10967373eSDavid Greenman/*- 2fcfe57d6SPeter Wemm * Copyright (c) 2003 Peter Wemm. 30967373eSDavid Greenman * Copyright (c) 1990 The Regents of the University of California. 40967373eSDavid Greenman * All rights reserved. 50967373eSDavid Greenman * 60967373eSDavid Greenman * This code is derived from software contributed to Berkeley by 70967373eSDavid Greenman * William Jolitz. 80967373eSDavid Greenman * 90967373eSDavid Greenman * Redistribution and use in source and binary forms, with or without 100967373eSDavid Greenman * modification, are permitted provided that the following conditions 110967373eSDavid Greenman * are met: 120967373eSDavid Greenman * 1. Redistributions of source code must retain the above copyright 130967373eSDavid Greenman * notice, this list of conditions and the following disclaimer. 140967373eSDavid Greenman * 2. Redistributions in binary form must reproduce the above copyright 150967373eSDavid Greenman * notice, this list of conditions and the following disclaimer in the 160967373eSDavid Greenman * documentation and/or other materials provided with the distribution. 17fbbd9655SWarner Losh * 3. Neither the name of the University nor the names of its contributors 180967373eSDavid Greenman * may be used to endorse or promote products derived from this software 190967373eSDavid Greenman * without specific prior written permission. 200967373eSDavid Greenman * 210967373eSDavid Greenman * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 220967373eSDavid Greenman * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 230967373eSDavid Greenman * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 240967373eSDavid Greenman * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 250967373eSDavid Greenman * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 260967373eSDavid Greenman * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 270967373eSDavid Greenman * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 280967373eSDavid Greenman * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 290967373eSDavid Greenman * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 300967373eSDavid Greenman * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 310967373eSDavid Greenman * SUCH DAMAGE. 320967373eSDavid Greenman * 33c3aac50fSPeter Wemm * $FreeBSD$ 340967373eSDavid Greenman */ 350967373eSDavid Greenman 360513ce7fSBruce Evans#include <machine/asmacros.h> 37db527225SPeter Wemm#include <machine/specialreg.h> 38477a642cSPeter Wemm 390513ce7fSBruce Evans#include "assym.s" 4040380a6aSJeff Roberson#include "opt_sched.h" 410513ce7fSBruce Evans 420967373eSDavid Greenman/*****************************************************************************/ 430967373eSDavid Greenman/* Scheduling */ 440967373eSDavid Greenman/*****************************************************************************/ 450967373eSDavid Greenman 460967373eSDavid Greenman .text 470967373eSDavid Greenman 4851621230SPeter Wemm#ifdef SMP 4951621230SPeter Wemm#define LK lock ; 5051621230SPeter Wemm#else 5151621230SPeter Wemm#define LK 5251621230SPeter Wemm#endif 5351621230SPeter Wemm 5440380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 5540380a6aSJeff Roberson#define SETLK xchgq 5640380a6aSJeff Roberson#else 5740380a6aSJeff Roberson#define SETLK movq 5840380a6aSJeff Roberson#endif 5940380a6aSJeff Roberson 600967373eSDavid Greenman/* 610384fff8SJason Evans * cpu_throw() 62e602ba25SJulian Elischer * 632c87e001SPeter Wemm * This is the second half of cpu_switch(). It is used when the current 64e602ba25SJulian Elischer * thread is either a dummy or slated to die, and we no longer care 65cc66ebe2SPeter Wemm * about its state. This is only a slight optimization and is probably 66cc66ebe2SPeter Wemm * not worth it anymore. Note that we need to clear the pm_active bits so 67cc66ebe2SPeter Wemm * we do need the old proc if it still exists. 68afa88623SPeter Wemm * %rdi = oldtd 69afa88623SPeter Wemm * %rsi = newtd 700384fff8SJason Evans */ 710384fff8SJason EvansENTRY(cpu_throw) 72a546448bSKonstantin Belousov movq %rsi,%r12 73a546448bSKonstantin Belousov movq %rsi,%rdi 74a546448bSKonstantin Belousov call pmap_activate_sw 75a546448bSKonstantin Belousov jmp sw1 76ea497502SJoseph KoshyEND(cpu_throw) 770384fff8SJason Evans 780384fff8SJason Evans/* 795d68dad3SJeff Roberson * cpu_switch(old, new, mtx) 80e602ba25SJulian Elischer * 81e602ba25SJulian Elischer * Save the current thread state, then select the next thread to run 82e602ba25SJulian Elischer * and load its state. 83afa88623SPeter Wemm * %rdi = oldtd 84afa88623SPeter Wemm * %rsi = newtd 855d68dad3SJeff Roberson * %rdx = mtx 860967373eSDavid Greenman */ 8726f9a767SRodney W. GrimesENTRY(cpu_switch) 88cc66ebe2SPeter Wemm /* Switch to new thread. First, save context. */ 89bf1e8974SPeter Wemm movq TD_PCB(%rdi),%r8 9050e3cec3SJung-uk Kim orl $PCB_FULL_IRET,PCB_FLAGS(%r8) 910967373eSDavid Greenman 92afa88623SPeter Wemm movq (%rsp),%rax /* Hardware registers */ 93bf1e8974SPeter Wemm movq %r15,PCB_R15(%r8) 94f001eabfSPeter Wemm movq %r14,PCB_R14(%r8) 95f001eabfSPeter Wemm movq %r13,PCB_R13(%r8) 96f001eabfSPeter Wemm movq %r12,PCB_R12(%r8) 97f001eabfSPeter Wemm movq %rbp,PCB_RBP(%r8) 98f001eabfSPeter Wemm movq %rsp,PCB_RSP(%r8) 99f001eabfSPeter Wemm movq %rbx,PCB_RBX(%r8) 100f001eabfSPeter Wemm movq %rax,PCB_RIP(%r8) 101f98c3ea7SKonstantin Belousov 10250e3cec3SJung-uk Kim testl $PCB_DBREGS,PCB_FLAGS(%r8) 103f001eabfSPeter Wemm jnz store_dr /* static predict not taken */ 104f001eabfSPeter Wemmdone_store_dr: 105db527225SPeter Wemm 1060967373eSDavid Greenman /* have we used fp, and need a save? */ 107afa88623SPeter Wemm cmpq %rdi,PCPU(FPCURTHREAD) 1088c6f8f3dSKonstantin Belousov jne 3f 1096cf9a08dSKonstantin Belousov movq PCB_SAVEFPU(%r8),%r8 110db527225SPeter Wemm clts 1118c6f8f3dSKonstantin Belousov cmpl $0,use_xsave 1128c6f8f3dSKonstantin Belousov jne 1f 113db527225SPeter Wemm fxsave (%r8) 1148c6f8f3dSKonstantin Belousov jmp 2f 1158c6f8f3dSKonstantin Belousov1: movq %rdx,%rcx 1168c6f8f3dSKonstantin Belousov movl xsave_mask,%eax 1178c6f8f3dSKonstantin Belousov movl xsave_mask+4,%edx 118333d0c60SKonstantin Belousov .globl ctx_switch_xsave 119333d0c60SKonstantin Belousovctx_switch_xsave: 120333d0c60SKonstantin Belousov /* This is patched to xsaveopt if supported, see fpuinit_bsp1() */ 121f18d5bf4SKonstantin Belousov xsave (%r8) 1228c6f8f3dSKonstantin Belousov movq %rcx,%rdx 1238c6f8f3dSKonstantin Belousov2: smsw %ax 124db527225SPeter Wemm orb $CR0_TS,%al 125db527225SPeter Wemm lmsw %ax 12698df9e00SPeter Wemm xorl %eax,%eax 127db527225SPeter Wemm movq %rax,PCPU(FPCURTHREAD) 1288c6f8f3dSKonstantin Belousov3: 129cc66ebe2SPeter Wemm /* Save is done. Now fire up new thread. Leave old vmspace. */ 130a546448bSKonstantin Belousov movq %rsi,%r12 131a546448bSKonstantin Belousov movq %rdi,%r13 132a546448bSKonstantin Belousov movq %rdx,%r15 133a546448bSKonstantin Belousov movq %rsi,%rdi 134a546448bSKonstantin Belousov callq pmap_activate_sw 135a546448bSKonstantin Belousov SETLK %r15,TD_LOCK(%r13) /* Release the old thread */ 136cc66ebe2SPeter Wemmsw1: 137a546448bSKonstantin Belousov movq TD_PCB(%r12),%r8 13840380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 13940380a6aSJeff Roberson /* Wait for the new thread to become unblocked */ 14040380a6aSJeff Roberson movq $blocked_lock, %rdx 14140380a6aSJeff Roberson1: 142a546448bSKonstantin Belousov movq TD_LOCK(%r12),%rcx 14340380a6aSJeff Roberson cmpq %rcx, %rdx 14440380a6aSJeff Roberson pause 14540380a6aSJeff Roberson je 1b 14640380a6aSJeff Roberson#endif 147cc66ebe2SPeter Wemm /* 148cc66ebe2SPeter Wemm * At this point, we've switched address spaces and are ready 149cc66ebe2SPeter Wemm * to load up the rest of the next context. 150cc66ebe2SPeter Wemm */ 151bf1e8974SPeter Wemm 152f001eabfSPeter Wemm /* Skip loading user fsbase/gsbase for kthreads */ 153a546448bSKonstantin Belousov testl $TDP_KTHREAD,TD_PFLAGS(%r12) 154f001eabfSPeter Wemm jnz do_kthread 155f001eabfSPeter Wemm 1562c66cccaSKonstantin Belousov /* 1572c66cccaSKonstantin Belousov * Load ldt register 1582c66cccaSKonstantin Belousov */ 159a546448bSKonstantin Belousov movq TD_PROC(%r12),%rcx 1602c66cccaSKonstantin Belousov cmpq $0, P_MD+MD_LDT(%rcx) 1612c66cccaSKonstantin Belousov jne do_ldt 1622c66cccaSKonstantin Belousov xorl %eax,%eax 1632c66cccaSKonstantin Belousovld_ldt: lldt %ax 1648f4a1f3aSKonstantin Belousov 1652c66cccaSKonstantin Belousov /* Restore fs base in GDT */ 166bf1e8974SPeter Wemm movl PCB_FSBASE(%r8),%eax 1672c66cccaSKonstantin Belousov movq PCPU(FS32P),%rdx 1682c66cccaSKonstantin Belousov movw %ax,2(%rdx) 1692c66cccaSKonstantin Belousov shrl $16,%eax 1702c66cccaSKonstantin Belousov movb %al,4(%rdx) 1712c66cccaSKonstantin Belousov shrl $8,%eax 1722c66cccaSKonstantin Belousov movb %al,7(%rdx) 1739869fa1dSJohn Baldwin 1742c66cccaSKonstantin Belousov /* Restore gs base in GDT */ 1752c66cccaSKonstantin Belousov movl PCB_GSBASE(%r8),%eax 1762c66cccaSKonstantin Belousov movq PCPU(GS32P),%rdx 1772c66cccaSKonstantin Belousov movw %ax,2(%rdx) 1782c66cccaSKonstantin Belousov shrl $16,%eax 1792c66cccaSKonstantin Belousov movb %al,4(%rdx) 1802c66cccaSKonstantin Belousov shrl $8,%eax 1812c66cccaSKonstantin Belousov movb %al,7(%rdx) 1822c66cccaSKonstantin Belousov 1832c66cccaSKonstantin Belousovdo_kthread: 1842c66cccaSKonstantin Belousov /* Do we need to reload tss ? */ 1850d2a2989SPeter Wemm movq PCPU(TSSP),%rax 1862c66cccaSKonstantin Belousov movq PCB_TSSP(%r8),%rdx 1872c66cccaSKonstantin Belousov testq %rdx,%rdx 1882c66cccaSKonstantin Belousov cmovzq PCPU(COMMONTSSP),%rdx 1892c66cccaSKonstantin Belousov cmpq %rax,%rdx 1902c66cccaSKonstantin Belousov jne do_tss 1912c66cccaSKonstantin Belousovdone_tss: 192f001eabfSPeter Wemm movq %r8,PCPU(RSP0) 1934e32b7b3SDavid Xu movq %r8,PCPU(CURPCB) 1942c66cccaSKonstantin Belousov /* Update the TSS_RSP0 pointer for the next interrupt */ 1952c66cccaSKonstantin Belousov movq %r8,COMMON_TSS_RSP0(%rdx) 196a546448bSKonstantin Belousov movq %r12,PCPU(CURTHREAD) /* into next thread */ 197f001eabfSPeter Wemm 198f001eabfSPeter Wemm /* Test if debug registers should be restored. */ 19950e3cec3SJung-uk Kim testl $PCB_DBREGS,PCB_FLAGS(%r8) 200f001eabfSPeter Wemm jnz load_dr /* static predict not taken */ 201f001eabfSPeter Wemmdone_load_dr: 2024e32b7b3SDavid Xu 203f001eabfSPeter Wemm /* Restore context. */ 204f001eabfSPeter Wemm movq PCB_R15(%r8),%r15 205f001eabfSPeter Wemm movq PCB_R14(%r8),%r14 206f001eabfSPeter Wemm movq PCB_R13(%r8),%r13 207f001eabfSPeter Wemm movq PCB_R12(%r8),%r12 208f001eabfSPeter Wemm movq PCB_RBP(%r8),%rbp 209f001eabfSPeter Wemm movq PCB_RSP(%r8),%rsp 210f001eabfSPeter Wemm movq PCB_RBX(%r8),%rbx 211f001eabfSPeter Wemm movq PCB_RIP(%r8),%rax 212f001eabfSPeter Wemm movq %rax,(%rsp) 213f001eabfSPeter Wemm ret 214f001eabfSPeter Wemm 215f001eabfSPeter Wemm /* 216f001eabfSPeter Wemm * We order these strangely for several reasons. 217f001eabfSPeter Wemm * 1: I wanted to use static branch prediction hints 218f001eabfSPeter Wemm * 2: Most athlon64/opteron cpus don't have them. They define 219f001eabfSPeter Wemm * a forward branch as 'predict not taken'. Intel cores have 220f001eabfSPeter Wemm * the 'rep' prefix to invert this. 221f001eabfSPeter Wemm * So, to make it work on both forms of cpu we do the detour. 222f001eabfSPeter Wemm * We use jumps rather than call in order to avoid the stack. 223f001eabfSPeter Wemm */ 224f001eabfSPeter Wemm 225f001eabfSPeter Wemmstore_dr: 226f001eabfSPeter Wemm movq %dr7,%rax /* yes, do the save */ 227f001eabfSPeter Wemm movq %dr0,%r15 228f001eabfSPeter Wemm movq %dr1,%r14 229f001eabfSPeter Wemm movq %dr2,%r13 230f001eabfSPeter Wemm movq %dr3,%r12 231f001eabfSPeter Wemm movq %dr6,%r11 232f001eabfSPeter Wemm movq %r15,PCB_DR0(%r8) 233f001eabfSPeter Wemm movq %r14,PCB_DR1(%r8) 234f001eabfSPeter Wemm movq %r13,PCB_DR2(%r8) 235f001eabfSPeter Wemm movq %r12,PCB_DR3(%r8) 236f001eabfSPeter Wemm movq %r11,PCB_DR6(%r8) 237f001eabfSPeter Wemm movq %rax,PCB_DR7(%r8) 238aaa95ccbSKonstantin Belousov andq $0x0000fc00, %rax /* disable all watchpoints */ 239db527225SPeter Wemm movq %rax,%dr7 240f001eabfSPeter Wemm jmp done_store_dr 241f001eabfSPeter Wemm 242f001eabfSPeter Wemmload_dr: 243f001eabfSPeter Wemm movq %dr7,%rax 244f001eabfSPeter Wemm movq PCB_DR0(%r8),%r15 245f001eabfSPeter Wemm movq PCB_DR1(%r8),%r14 246f001eabfSPeter Wemm movq PCB_DR2(%r8),%r13 247f001eabfSPeter Wemm movq PCB_DR3(%r8),%r12 248f001eabfSPeter Wemm movq PCB_DR6(%r8),%r11 249f001eabfSPeter Wemm movq PCB_DR7(%r8),%rcx 250f001eabfSPeter Wemm movq %r15,%dr0 251f001eabfSPeter Wemm movq %r14,%dr1 252f001eabfSPeter Wemm /* Preserve reserved bits in %dr7 */ 253f001eabfSPeter Wemm andq $0x0000fc00,%rax 254f001eabfSPeter Wemm andq $~0x0000fc00,%rcx 255f001eabfSPeter Wemm movq %r13,%dr2 256f001eabfSPeter Wemm movq %r12,%dr3 257f001eabfSPeter Wemm orq %rcx,%rax 258f001eabfSPeter Wemm movq %r11,%dr6 259f001eabfSPeter Wemm movq %rax,%dr7 260f001eabfSPeter Wemm jmp done_load_dr 2612c66cccaSKonstantin Belousov 2622c66cccaSKonstantin Belousovdo_tss: movq %rdx,PCPU(TSSP) 2632c66cccaSKonstantin Belousov movq %rdx,%rcx 2642c66cccaSKonstantin Belousov movq PCPU(TSS),%rax 265cfe92f33SDimitry Andric movw %cx,2(%rax) 2662c66cccaSKonstantin Belousov shrq $16,%rcx 2672c66cccaSKonstantin Belousov movb %cl,4(%rax) 2682c66cccaSKonstantin Belousov shrq $8,%rcx 2692c66cccaSKonstantin Belousov movb %cl,7(%rax) 2702c66cccaSKonstantin Belousov shrq $8,%rcx 2712c66cccaSKonstantin Belousov movl %ecx,8(%rax) 2722c66cccaSKonstantin Belousov movb $0x89,5(%rax) /* unset busy */ 2732c66cccaSKonstantin Belousov movl $TSSSEL,%eax 2742c66cccaSKonstantin Belousov ltr %ax 2752c66cccaSKonstantin Belousov jmp done_tss 2762c66cccaSKonstantin Belousov 2772c66cccaSKonstantin Belousovdo_ldt: movq PCPU(LDT),%rax 2782c66cccaSKonstantin Belousov movq P_MD+MD_LDT_SD(%rcx),%rdx 2792c66cccaSKonstantin Belousov movq %rdx,(%rax) 2802c66cccaSKonstantin Belousov movq P_MD+MD_LDT_SD+8(%rcx),%rdx 2812c66cccaSKonstantin Belousov movq %rdx,8(%rax) 2822c66cccaSKonstantin Belousov movl $LDTSEL,%eax 2832c66cccaSKonstantin Belousov jmp ld_ldt 284ea497502SJoseph KoshyEND(cpu_switch) 2850967373eSDavid Greenman 2860967373eSDavid Greenman/* 2872924d491SDavid Greenman * savectx(pcb) 2882924d491SDavid Greenman * Update pcb, saving current processor state. 2890967373eSDavid Greenman */ 2900967373eSDavid GreenmanENTRY(savectx) 291afa88623SPeter Wemm /* Save caller's return address. */ 292afa88623SPeter Wemm movq (%rsp),%rax 2933ab42a25SJung-uk Kim movq %rax,PCB_RIP(%rdi) 2942924d491SDavid Greenman 2953ab42a25SJung-uk Kim movq %rbx,PCB_RBX(%rdi) 2963ab42a25SJung-uk Kim movq %rsp,PCB_RSP(%rdi) 2973ab42a25SJung-uk Kim movq %rbp,PCB_RBP(%rdi) 2983ab42a25SJung-uk Kim movq %r12,PCB_R12(%rdi) 2993ab42a25SJung-uk Kim movq %r13,PCB_R13(%rdi) 3003ab42a25SJung-uk Kim movq %r14,PCB_R14(%rdi) 3013ab42a25SJung-uk Kim movq %r15,PCB_R15(%rdi) 302a2d2c836SJung-uk Kim 30305acaa9fSJung-uk Kim movq %cr0,%rax 30405acaa9fSJung-uk Kim movq %rax,PCB_CR0(%rdi) 305a2d2c836SJung-uk Kim movq %cr2,%rax 3063ab42a25SJung-uk Kim movq %rax,PCB_CR2(%rdi) 307afa88623SPeter Wemm movq %cr3,%rax 3083ab42a25SJung-uk Kim movq %rax,PCB_CR3(%rdi) 309a2d2c836SJung-uk Kim movq %cr4,%rax 3103ab42a25SJung-uk Kim movq %rax,PCB_CR4(%rdi) 311ac5f943cSPeter Wemm 312a2d2c836SJung-uk Kim movq %dr0,%rax 3133ab42a25SJung-uk Kim movq %rax,PCB_DR0(%rdi) 314a2d2c836SJung-uk Kim movq %dr1,%rax 3153ab42a25SJung-uk Kim movq %rax,PCB_DR1(%rdi) 316a2d2c836SJung-uk Kim movq %dr2,%rax 3173ab42a25SJung-uk Kim movq %rax,PCB_DR2(%rdi) 318a2d2c836SJung-uk Kim movq %dr3,%rax 3193ab42a25SJung-uk Kim movq %rax,PCB_DR3(%rdi) 320a2d2c836SJung-uk Kim movq %dr6,%rax 3213ab42a25SJung-uk Kim movq %rax,PCB_DR6(%rdi) 322a2d2c836SJung-uk Kim movq %dr7,%rax 3233ab42a25SJung-uk Kim movq %rax,PCB_DR7(%rdi) 324a2d2c836SJung-uk Kim 325a2d2c836SJung-uk Kim movl $MSR_FSBASE,%ecx 326a2d2c836SJung-uk Kim rdmsr 327305c5c0aSJung-uk Kim movl %eax,PCB_FSBASE(%rdi) 328305c5c0aSJung-uk Kim movl %edx,PCB_FSBASE+4(%rdi) 329a2d2c836SJung-uk Kim movl $MSR_GSBASE,%ecx 330a2d2c836SJung-uk Kim rdmsr 331305c5c0aSJung-uk Kim movl %eax,PCB_GSBASE(%rdi) 332305c5c0aSJung-uk Kim movl %edx,PCB_GSBASE+4(%rdi) 333a2d2c836SJung-uk Kim movl $MSR_KGSBASE,%ecx 334a2d2c836SJung-uk Kim rdmsr 335305c5c0aSJung-uk Kim movl %eax,PCB_KGSBASE(%rdi) 336305c5c0aSJung-uk Kim movl %edx,PCB_KGSBASE+4(%rdi) 337fb864578SMitsuru IWASAKI movl $MSR_EFER,%ecx 338fb864578SMitsuru IWASAKI rdmsr 339fb864578SMitsuru IWASAKI movl %eax,PCB_EFER(%rdi) 340fb864578SMitsuru IWASAKI movl %edx,PCB_EFER+4(%rdi) 341fb864578SMitsuru IWASAKI movl $MSR_STAR,%ecx 342fb864578SMitsuru IWASAKI rdmsr 343fb864578SMitsuru IWASAKI movl %eax,PCB_STAR(%rdi) 344fb864578SMitsuru IWASAKI movl %edx,PCB_STAR+4(%rdi) 345fb864578SMitsuru IWASAKI movl $MSR_LSTAR,%ecx 346fb864578SMitsuru IWASAKI rdmsr 347fb864578SMitsuru IWASAKI movl %eax,PCB_LSTAR(%rdi) 348fb864578SMitsuru IWASAKI movl %edx,PCB_LSTAR+4(%rdi) 349fb864578SMitsuru IWASAKI movl $MSR_CSTAR,%ecx 350fb864578SMitsuru IWASAKI rdmsr 351fb864578SMitsuru IWASAKI movl %eax,PCB_CSTAR(%rdi) 352fb864578SMitsuru IWASAKI movl %edx,PCB_CSTAR+4(%rdi) 353fb864578SMitsuru IWASAKI movl $MSR_SF_MASK,%ecx 354fb864578SMitsuru IWASAKI rdmsr 355fb864578SMitsuru IWASAKI movl %eax,PCB_SFMASK(%rdi) 356fb864578SMitsuru IWASAKI movl %edx,PCB_SFMASK+4(%rdi) 357a2d2c836SJung-uk Kim 3583ab42a25SJung-uk Kim sgdt PCB_GDT(%rdi) 3593ab42a25SJung-uk Kim sidt PCB_IDT(%rdi) 3603ab42a25SJung-uk Kim sldt PCB_LDT(%rdi) 3613ab42a25SJung-uk Kim str PCB_TR(%rdi) 362a2d2c836SJung-uk Kim 363a2d2c836SJung-uk Kim movl $1,%eax 3640967373eSDavid Greenman ret 365ea497502SJoseph KoshyEND(savectx) 3668c6f8f3dSKonstantin Belousov 3678c6f8f3dSKonstantin Belousov/* 3686ad79910SJung-uk Kim * resumectx(pcb) 369fb864578SMitsuru IWASAKI * Resuming processor state from pcb. 370fb864578SMitsuru IWASAKI */ 371fb864578SMitsuru IWASAKIENTRY(resumectx) 372fb864578SMitsuru IWASAKI /* Switch to KPML4phys. */ 3736ad79910SJung-uk Kim movq KPML4phys,%rax 3746ad79910SJung-uk Kim movq %rax,%cr3 375fb864578SMitsuru IWASAKI 376fb864578SMitsuru IWASAKI /* Force kernel segment registers. */ 377fb864578SMitsuru IWASAKI movl $KDSEL,%eax 378fb864578SMitsuru IWASAKI movw %ax,%ds 379fb864578SMitsuru IWASAKI movw %ax,%es 380fb864578SMitsuru IWASAKI movw %ax,%ss 381fb864578SMitsuru IWASAKI movl $KUF32SEL,%eax 382fb864578SMitsuru IWASAKI movw %ax,%fs 383fb864578SMitsuru IWASAKI movl $KUG32SEL,%eax 384fb864578SMitsuru IWASAKI movw %ax,%gs 385fb864578SMitsuru IWASAKI 386fb864578SMitsuru IWASAKI movl $MSR_FSBASE,%ecx 387fb864578SMitsuru IWASAKI movl PCB_FSBASE(%rdi),%eax 388fb864578SMitsuru IWASAKI movl 4 + PCB_FSBASE(%rdi),%edx 389fb864578SMitsuru IWASAKI wrmsr 390fb864578SMitsuru IWASAKI movl $MSR_GSBASE,%ecx 391fb864578SMitsuru IWASAKI movl PCB_GSBASE(%rdi),%eax 392fb864578SMitsuru IWASAKI movl 4 + PCB_GSBASE(%rdi),%edx 393fb864578SMitsuru IWASAKI wrmsr 394fb864578SMitsuru IWASAKI movl $MSR_KGSBASE,%ecx 395fb864578SMitsuru IWASAKI movl PCB_KGSBASE(%rdi),%eax 396fb864578SMitsuru IWASAKI movl 4 + PCB_KGSBASE(%rdi),%edx 397fb864578SMitsuru IWASAKI wrmsr 398fb864578SMitsuru IWASAKI 399bd101a66SKonstantin Belousov /* Restore EFER one more time. */ 400fb864578SMitsuru IWASAKI movl $MSR_EFER,%ecx 401fb864578SMitsuru IWASAKI movl PCB_EFER(%rdi),%eax 402fb864578SMitsuru IWASAKI wrmsr 403fb864578SMitsuru IWASAKI 404fb864578SMitsuru IWASAKI /* Restore fast syscall stuff. */ 405fb864578SMitsuru IWASAKI movl $MSR_STAR,%ecx 406fb864578SMitsuru IWASAKI movl PCB_STAR(%rdi),%eax 407fb864578SMitsuru IWASAKI movl 4 + PCB_STAR(%rdi),%edx 408fb864578SMitsuru IWASAKI wrmsr 409fb864578SMitsuru IWASAKI movl $MSR_LSTAR,%ecx 410fb864578SMitsuru IWASAKI movl PCB_LSTAR(%rdi),%eax 411fb864578SMitsuru IWASAKI movl 4 + PCB_LSTAR(%rdi),%edx 412fb864578SMitsuru IWASAKI wrmsr 413fb864578SMitsuru IWASAKI movl $MSR_CSTAR,%ecx 414fb864578SMitsuru IWASAKI movl PCB_CSTAR(%rdi),%eax 415fb864578SMitsuru IWASAKI movl 4 + PCB_CSTAR(%rdi),%edx 416fb864578SMitsuru IWASAKI wrmsr 417fb864578SMitsuru IWASAKI movl $MSR_SF_MASK,%ecx 418fb864578SMitsuru IWASAKI movl PCB_SFMASK(%rdi),%eax 419fb864578SMitsuru IWASAKI wrmsr 420fb864578SMitsuru IWASAKI 421b1d735baSJohn Baldwin /* Restore CR0, CR2, CR4 and CR3. */ 422fb864578SMitsuru IWASAKI movq PCB_CR0(%rdi),%rax 423fb864578SMitsuru IWASAKI movq %rax,%cr0 424fb864578SMitsuru IWASAKI movq PCB_CR2(%rdi),%rax 425fb864578SMitsuru IWASAKI movq %rax,%cr2 426fb864578SMitsuru IWASAKI movq PCB_CR4(%rdi),%rax 427fb864578SMitsuru IWASAKI movq %rax,%cr4 428fb864578SMitsuru IWASAKI movq PCB_CR3(%rdi),%rax 429fb864578SMitsuru IWASAKI movq %rax,%cr3 430fb864578SMitsuru IWASAKI 431fb864578SMitsuru IWASAKI /* Restore descriptor tables. */ 432fb864578SMitsuru IWASAKI lidt PCB_IDT(%rdi) 433fb864578SMitsuru IWASAKI lldt PCB_LDT(%rdi) 434fb864578SMitsuru IWASAKI 435fb864578SMitsuru IWASAKI#define SDT_SYSTSS 9 436fb864578SMitsuru IWASAKI#define SDT_SYSBSY 11 437fb864578SMitsuru IWASAKI 438fb864578SMitsuru IWASAKI /* Clear "task busy" bit and reload TR. */ 439fb864578SMitsuru IWASAKI movq PCPU(TSS),%rax 440fb864578SMitsuru IWASAKI andb $(~SDT_SYSBSY | SDT_SYSTSS),5(%rax) 441fb864578SMitsuru IWASAKI movw PCB_TR(%rdi),%ax 442fb864578SMitsuru IWASAKI ltr %ax 443fb864578SMitsuru IWASAKI 444fb864578SMitsuru IWASAKI#undef SDT_SYSTSS 445fb864578SMitsuru IWASAKI#undef SDT_SYSBSY 446fb864578SMitsuru IWASAKI 447fb864578SMitsuru IWASAKI /* Restore debug registers. */ 448fb864578SMitsuru IWASAKI movq PCB_DR0(%rdi),%rax 449fb864578SMitsuru IWASAKI movq %rax,%dr0 450fb864578SMitsuru IWASAKI movq PCB_DR1(%rdi),%rax 451fb864578SMitsuru IWASAKI movq %rax,%dr1 452fb864578SMitsuru IWASAKI movq PCB_DR2(%rdi),%rax 453fb864578SMitsuru IWASAKI movq %rax,%dr2 454fb864578SMitsuru IWASAKI movq PCB_DR3(%rdi),%rax 455fb864578SMitsuru IWASAKI movq %rax,%dr3 456fb864578SMitsuru IWASAKI movq PCB_DR6(%rdi),%rax 457fb864578SMitsuru IWASAKI movq %rax,%dr6 458fb864578SMitsuru IWASAKI movq PCB_DR7(%rdi),%rax 459fb864578SMitsuru IWASAKI movq %rax,%dr7 460fb864578SMitsuru IWASAKI 461fb864578SMitsuru IWASAKI /* Restore other callee saved registers. */ 462fb864578SMitsuru IWASAKI movq PCB_R15(%rdi),%r15 463fb864578SMitsuru IWASAKI movq PCB_R14(%rdi),%r14 464fb864578SMitsuru IWASAKI movq PCB_R13(%rdi),%r13 465fb864578SMitsuru IWASAKI movq PCB_R12(%rdi),%r12 466fb864578SMitsuru IWASAKI movq PCB_RBP(%rdi),%rbp 467fb864578SMitsuru IWASAKI movq PCB_RSP(%rdi),%rsp 468fb864578SMitsuru IWASAKI movq PCB_RBX(%rdi),%rbx 469fb864578SMitsuru IWASAKI 470fb864578SMitsuru IWASAKI /* Restore return address. */ 471fb864578SMitsuru IWASAKI movq PCB_RIP(%rdi),%rax 472fb864578SMitsuru IWASAKI movq %rax,(%rsp) 473fb864578SMitsuru IWASAKI 474fb864578SMitsuru IWASAKI xorl %eax,%eax 475fb864578SMitsuru IWASAKI ret 476fb864578SMitsuru IWASAKIEND(resumectx) 477