10967373eSDavid Greenman/*- 2fcfe57d6SPeter Wemm * Copyright (c) 2003 Peter Wemm. 30967373eSDavid Greenman * Copyright (c) 1990 The Regents of the University of California. 40967373eSDavid Greenman * All rights reserved. 50967373eSDavid Greenman * 60967373eSDavid Greenman * This code is derived from software contributed to Berkeley by 70967373eSDavid Greenman * William Jolitz. 80967373eSDavid Greenman * 90967373eSDavid Greenman * Redistribution and use in source and binary forms, with or without 100967373eSDavid Greenman * modification, are permitted provided that the following conditions 110967373eSDavid Greenman * are met: 120967373eSDavid Greenman * 1. Redistributions of source code must retain the above copyright 130967373eSDavid Greenman * notice, this list of conditions and the following disclaimer. 140967373eSDavid Greenman * 2. Redistributions in binary form must reproduce the above copyright 150967373eSDavid Greenman * notice, this list of conditions and the following disclaimer in the 160967373eSDavid Greenman * documentation and/or other materials provided with the distribution. 17fbbd9655SWarner Losh * 3. Neither the name of the University nor the names of its contributors 180967373eSDavid Greenman * may be used to endorse or promote products derived from this software 190967373eSDavid Greenman * without specific prior written permission. 200967373eSDavid Greenman * 210967373eSDavid Greenman * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 220967373eSDavid Greenman * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 230967373eSDavid Greenman * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 240967373eSDavid Greenman * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 250967373eSDavid Greenman * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 260967373eSDavid Greenman * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 270967373eSDavid Greenman * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 280967373eSDavid Greenman * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 290967373eSDavid Greenman * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 300967373eSDavid Greenman * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 310967373eSDavid Greenman * SUCH DAMAGE. 320967373eSDavid Greenman * 33c3aac50fSPeter Wemm * $FreeBSD$ 340967373eSDavid Greenman */ 350967373eSDavid Greenman 360513ce7fSBruce Evans#include <machine/asmacros.h> 37db527225SPeter Wemm#include <machine/specialreg.h> 38477a642cSPeter Wemm 390513ce7fSBruce Evans#include "assym.s" 4040380a6aSJeff Roberson#include "opt_sched.h" 410513ce7fSBruce Evans 420967373eSDavid Greenman/*****************************************************************************/ 430967373eSDavid Greenman/* Scheduling */ 440967373eSDavid Greenman/*****************************************************************************/ 450967373eSDavid Greenman 460967373eSDavid Greenman .text 470967373eSDavid Greenman 4851621230SPeter Wemm#ifdef SMP 4951621230SPeter Wemm#define LK lock ; 5051621230SPeter Wemm#else 5151621230SPeter Wemm#define LK 5251621230SPeter Wemm#endif 5351621230SPeter Wemm 5440380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 5540380a6aSJeff Roberson#define SETLK xchgq 5640380a6aSJeff Roberson#else 5740380a6aSJeff Roberson#define SETLK movq 5840380a6aSJeff Roberson#endif 5940380a6aSJeff Roberson 600967373eSDavid Greenman/* 610384fff8SJason Evans * cpu_throw() 62e602ba25SJulian Elischer * 632c87e001SPeter Wemm * This is the second half of cpu_switch(). It is used when the current 64e602ba25SJulian Elischer * thread is either a dummy or slated to die, and we no longer care 65cc66ebe2SPeter Wemm * about its state. This is only a slight optimization and is probably 66cc66ebe2SPeter Wemm * not worth it anymore. Note that we need to clear the pm_active bits so 67cc66ebe2SPeter Wemm * we do need the old proc if it still exists. 68afa88623SPeter Wemm * %rdi = oldtd 69afa88623SPeter Wemm * %rsi = newtd 700384fff8SJason Evans */ 710384fff8SJason EvansENTRY(cpu_throw) 72a546448bSKonstantin Belousov movq %rsi,%r12 73a546448bSKonstantin Belousov movq %rsi,%rdi 74a546448bSKonstantin Belousov call pmap_activate_sw 75a546448bSKonstantin Belousov jmp sw1 76ea497502SJoseph KoshyEND(cpu_throw) 770384fff8SJason Evans 780384fff8SJason Evans/* 795d68dad3SJeff Roberson * cpu_switch(old, new, mtx) 80e602ba25SJulian Elischer * 81e602ba25SJulian Elischer * Save the current thread state, then select the next thread to run 82e602ba25SJulian Elischer * and load its state. 83afa88623SPeter Wemm * %rdi = oldtd 84afa88623SPeter Wemm * %rsi = newtd 855d68dad3SJeff Roberson * %rdx = mtx 860967373eSDavid Greenman */ 8726f9a767SRodney W. GrimesENTRY(cpu_switch) 88cc66ebe2SPeter Wemm /* Switch to new thread. First, save context. */ 89bf1e8974SPeter Wemm movq TD_PCB(%rdi),%r8 900967373eSDavid Greenman 91afa88623SPeter Wemm movq (%rsp),%rax /* Hardware registers */ 92bf1e8974SPeter Wemm movq %r15,PCB_R15(%r8) 93f001eabfSPeter Wemm movq %r14,PCB_R14(%r8) 94f001eabfSPeter Wemm movq %r13,PCB_R13(%r8) 95f001eabfSPeter Wemm movq %r12,PCB_R12(%r8) 96f001eabfSPeter Wemm movq %rbp,PCB_RBP(%r8) 97f001eabfSPeter Wemm movq %rsp,PCB_RSP(%r8) 98f001eabfSPeter Wemm movq %rbx,PCB_RBX(%r8) 99f001eabfSPeter Wemm movq %rax,PCB_RIP(%r8) 100f98c3ea7SKonstantin Belousov 1013e902b3dSKonstantin Belousov testl $PCB_FULL_IRET,PCB_FLAGS(%r8) 1023e902b3dSKonstantin Belousov jnz 2f 1033e902b3dSKonstantin Belousov orl $PCB_FULL_IRET,PCB_FLAGS(%r8) 1043e902b3dSKonstantin Belousov testl $TDP_KTHREAD,TD_PFLAGS(%rdi) 1053e902b3dSKonstantin Belousov jnz 2f 1063e902b3dSKonstantin Belousov testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) 1073e902b3dSKonstantin Belousov jz 2f 1083e902b3dSKonstantin Belousov movl %fs,%eax 1093e902b3dSKonstantin Belousov cmpl $KUF32SEL,%eax 1103e902b3dSKonstantin Belousov jne 1f 1110d4e7ec5SRyan Libby rdfsbase %rax 1123e902b3dSKonstantin Belousov movq %rax,PCB_FSBASE(%r8) 1133e902b3dSKonstantin Belousov1: movl %gs,%eax 1143e902b3dSKonstantin Belousov cmpl $KUG32SEL,%eax 1153e902b3dSKonstantin Belousov jne 2f 1163e902b3dSKonstantin Belousov movq %rdx,%r12 1173e902b3dSKonstantin Belousov movl $MSR_KGSBASE,%ecx /* Read user gs base */ 1183e902b3dSKonstantin Belousov rdmsr 1193e902b3dSKonstantin Belousov shlq $32,%rdx 1203e902b3dSKonstantin Belousov orq %rdx,%rax 1213e902b3dSKonstantin Belousov movq %rax,PCB_GSBASE(%r8) 1223e902b3dSKonstantin Belousov movq %r12,%rdx 1233e902b3dSKonstantin Belousov 1243e902b3dSKonstantin Belousov2: 12550e3cec3SJung-uk Kim testl $PCB_DBREGS,PCB_FLAGS(%r8) 126f001eabfSPeter Wemm jnz store_dr /* static predict not taken */ 127f001eabfSPeter Wemmdone_store_dr: 128db527225SPeter Wemm 1290967373eSDavid Greenman /* have we used fp, and need a save? */ 130afa88623SPeter Wemm cmpq %rdi,PCPU(FPCURTHREAD) 1318c6f8f3dSKonstantin Belousov jne 3f 1326cf9a08dSKonstantin Belousov movq PCB_SAVEFPU(%r8),%r8 133db527225SPeter Wemm clts 1348c6f8f3dSKonstantin Belousov cmpl $0,use_xsave 1358c6f8f3dSKonstantin Belousov jne 1f 136db527225SPeter Wemm fxsave (%r8) 1378c6f8f3dSKonstantin Belousov jmp 2f 1388c6f8f3dSKonstantin Belousov1: movq %rdx,%rcx 1398c6f8f3dSKonstantin Belousov movl xsave_mask,%eax 1408c6f8f3dSKonstantin Belousov movl xsave_mask+4,%edx 141333d0c60SKonstantin Belousov .globl ctx_switch_xsave 142333d0c60SKonstantin Belousovctx_switch_xsave: 143333d0c60SKonstantin Belousov /* This is patched to xsaveopt if supported, see fpuinit_bsp1() */ 144f18d5bf4SKonstantin Belousov xsave (%r8) 1458c6f8f3dSKonstantin Belousov movq %rcx,%rdx 1468c6f8f3dSKonstantin Belousov2: smsw %ax 147db527225SPeter Wemm orb $CR0_TS,%al 148db527225SPeter Wemm lmsw %ax 14998df9e00SPeter Wemm xorl %eax,%eax 150db527225SPeter Wemm movq %rax,PCPU(FPCURTHREAD) 1518c6f8f3dSKonstantin Belousov3: 152cc66ebe2SPeter Wemm /* Save is done. Now fire up new thread. Leave old vmspace. */ 153a546448bSKonstantin Belousov movq %rsi,%r12 154a546448bSKonstantin Belousov movq %rdi,%r13 155a546448bSKonstantin Belousov movq %rdx,%r15 156a546448bSKonstantin Belousov movq %rsi,%rdi 157a546448bSKonstantin Belousov callq pmap_activate_sw 158a546448bSKonstantin Belousov SETLK %r15,TD_LOCK(%r13) /* Release the old thread */ 159cc66ebe2SPeter Wemmsw1: 160a546448bSKonstantin Belousov movq TD_PCB(%r12),%r8 16140380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 16240380a6aSJeff Roberson /* Wait for the new thread to become unblocked */ 16340380a6aSJeff Roberson movq $blocked_lock, %rdx 16440380a6aSJeff Roberson1: 165a546448bSKonstantin Belousov movq TD_LOCK(%r12),%rcx 16640380a6aSJeff Roberson cmpq %rcx, %rdx 16740380a6aSJeff Roberson pause 16840380a6aSJeff Roberson je 1b 16940380a6aSJeff Roberson#endif 170cc66ebe2SPeter Wemm /* 171cc66ebe2SPeter Wemm * At this point, we've switched address spaces and are ready 172cc66ebe2SPeter Wemm * to load up the rest of the next context. 173cc66ebe2SPeter Wemm */ 174bf1e8974SPeter Wemm 175843d5752SKonstantin Belousov /* Skip loading LDT and user fsbase/gsbase for kthreads */ 176a546448bSKonstantin Belousov testl $TDP_KTHREAD,TD_PFLAGS(%r12) 177f001eabfSPeter Wemm jnz do_kthread 178f001eabfSPeter Wemm 1792c66cccaSKonstantin Belousov /* 1802c66cccaSKonstantin Belousov * Load ldt register 1812c66cccaSKonstantin Belousov */ 182a546448bSKonstantin Belousov movq TD_PROC(%r12),%rcx 1832c66cccaSKonstantin Belousov cmpq $0, P_MD+MD_LDT(%rcx) 1842c66cccaSKonstantin Belousov jne do_ldt 1852c66cccaSKonstantin Belousov xorl %eax,%eax 1862c66cccaSKonstantin Belousovld_ldt: lldt %ax 1878f4a1f3aSKonstantin Belousov 1882c66cccaSKonstantin Belousov /* Restore fs base in GDT */ 189bf1e8974SPeter Wemm movl PCB_FSBASE(%r8),%eax 1902c66cccaSKonstantin Belousov movq PCPU(FS32P),%rdx 1912c66cccaSKonstantin Belousov movw %ax,2(%rdx) 1922c66cccaSKonstantin Belousov shrl $16,%eax 1932c66cccaSKonstantin Belousov movb %al,4(%rdx) 1942c66cccaSKonstantin Belousov shrl $8,%eax 1952c66cccaSKonstantin Belousov movb %al,7(%rdx) 1969869fa1dSJohn Baldwin 1972c66cccaSKonstantin Belousov /* Restore gs base in GDT */ 1982c66cccaSKonstantin Belousov movl PCB_GSBASE(%r8),%eax 1992c66cccaSKonstantin Belousov movq PCPU(GS32P),%rdx 2002c66cccaSKonstantin Belousov movw %ax,2(%rdx) 2012c66cccaSKonstantin Belousov shrl $16,%eax 2022c66cccaSKonstantin Belousov movb %al,4(%rdx) 2032c66cccaSKonstantin Belousov shrl $8,%eax 2042c66cccaSKonstantin Belousov movb %al,7(%rdx) 2052c66cccaSKonstantin Belousov 2062c66cccaSKonstantin Belousovdo_kthread: 2072c66cccaSKonstantin Belousov /* Do we need to reload tss ? */ 2080d2a2989SPeter Wemm movq PCPU(TSSP),%rax 2092c66cccaSKonstantin Belousov movq PCB_TSSP(%r8),%rdx 2102c66cccaSKonstantin Belousov testq %rdx,%rdx 2112c66cccaSKonstantin Belousov cmovzq PCPU(COMMONTSSP),%rdx 2122c66cccaSKonstantin Belousov cmpq %rax,%rdx 2132c66cccaSKonstantin Belousov jne do_tss 2142c66cccaSKonstantin Belousovdone_tss: 215f001eabfSPeter Wemm movq %r8,PCPU(RSP0) 2164e32b7b3SDavid Xu movq %r8,PCPU(CURPCB) 2172c66cccaSKonstantin Belousov /* Update the TSS_RSP0 pointer for the next interrupt */ 2182c66cccaSKonstantin Belousov movq %r8,COMMON_TSS_RSP0(%rdx) 219a546448bSKonstantin Belousov movq %r12,PCPU(CURTHREAD) /* into next thread */ 220f001eabfSPeter Wemm 221f001eabfSPeter Wemm /* Test if debug registers should be restored. */ 22250e3cec3SJung-uk Kim testl $PCB_DBREGS,PCB_FLAGS(%r8) 223f001eabfSPeter Wemm jnz load_dr /* static predict not taken */ 224f001eabfSPeter Wemmdone_load_dr: 2254e32b7b3SDavid Xu 226f001eabfSPeter Wemm /* Restore context. */ 227f001eabfSPeter Wemm movq PCB_R15(%r8),%r15 228f001eabfSPeter Wemm movq PCB_R14(%r8),%r14 229f001eabfSPeter Wemm movq PCB_R13(%r8),%r13 230f001eabfSPeter Wemm movq PCB_R12(%r8),%r12 231f001eabfSPeter Wemm movq PCB_RBP(%r8),%rbp 232f001eabfSPeter Wemm movq PCB_RSP(%r8),%rsp 233f001eabfSPeter Wemm movq PCB_RBX(%r8),%rbx 234f001eabfSPeter Wemm movq PCB_RIP(%r8),%rax 235f001eabfSPeter Wemm movq %rax,(%rsp) 236f001eabfSPeter Wemm ret 237f001eabfSPeter Wemm 238f001eabfSPeter Wemm /* 239f001eabfSPeter Wemm * We order these strangely for several reasons. 240f001eabfSPeter Wemm * 1: I wanted to use static branch prediction hints 241f001eabfSPeter Wemm * 2: Most athlon64/opteron cpus don't have them. They define 242f001eabfSPeter Wemm * a forward branch as 'predict not taken'. Intel cores have 243f001eabfSPeter Wemm * the 'rep' prefix to invert this. 244f001eabfSPeter Wemm * So, to make it work on both forms of cpu we do the detour. 245f001eabfSPeter Wemm * We use jumps rather than call in order to avoid the stack. 246f001eabfSPeter Wemm */ 247f001eabfSPeter Wemm 248f001eabfSPeter Wemmstore_dr: 249f001eabfSPeter Wemm movq %dr7,%rax /* yes, do the save */ 250f001eabfSPeter Wemm movq %dr0,%r15 251f001eabfSPeter Wemm movq %dr1,%r14 252f001eabfSPeter Wemm movq %dr2,%r13 253f001eabfSPeter Wemm movq %dr3,%r12 254f001eabfSPeter Wemm movq %dr6,%r11 255f001eabfSPeter Wemm movq %r15,PCB_DR0(%r8) 256f001eabfSPeter Wemm movq %r14,PCB_DR1(%r8) 257f001eabfSPeter Wemm movq %r13,PCB_DR2(%r8) 258f001eabfSPeter Wemm movq %r12,PCB_DR3(%r8) 259f001eabfSPeter Wemm movq %r11,PCB_DR6(%r8) 260f001eabfSPeter Wemm movq %rax,PCB_DR7(%r8) 261aaa95ccbSKonstantin Belousov andq $0x0000fc00, %rax /* disable all watchpoints */ 262db527225SPeter Wemm movq %rax,%dr7 263f001eabfSPeter Wemm jmp done_store_dr 264f001eabfSPeter Wemm 265f001eabfSPeter Wemmload_dr: 266f001eabfSPeter Wemm movq %dr7,%rax 267f001eabfSPeter Wemm movq PCB_DR0(%r8),%r15 268f001eabfSPeter Wemm movq PCB_DR1(%r8),%r14 269f001eabfSPeter Wemm movq PCB_DR2(%r8),%r13 270f001eabfSPeter Wemm movq PCB_DR3(%r8),%r12 271f001eabfSPeter Wemm movq PCB_DR6(%r8),%r11 272f001eabfSPeter Wemm movq PCB_DR7(%r8),%rcx 273f001eabfSPeter Wemm movq %r15,%dr0 274f001eabfSPeter Wemm movq %r14,%dr1 275f001eabfSPeter Wemm /* Preserve reserved bits in %dr7 */ 276f001eabfSPeter Wemm andq $0x0000fc00,%rax 277f001eabfSPeter Wemm andq $~0x0000fc00,%rcx 278f001eabfSPeter Wemm movq %r13,%dr2 279f001eabfSPeter Wemm movq %r12,%dr3 280f001eabfSPeter Wemm orq %rcx,%rax 281f001eabfSPeter Wemm movq %r11,%dr6 282f001eabfSPeter Wemm movq %rax,%dr7 283f001eabfSPeter Wemm jmp done_load_dr 2842c66cccaSKonstantin Belousov 2852c66cccaSKonstantin Belousovdo_tss: movq %rdx,PCPU(TSSP) 2862c66cccaSKonstantin Belousov movq %rdx,%rcx 2872c66cccaSKonstantin Belousov movq PCPU(TSS),%rax 288cfe92f33SDimitry Andric movw %cx,2(%rax) 2892c66cccaSKonstantin Belousov shrq $16,%rcx 2902c66cccaSKonstantin Belousov movb %cl,4(%rax) 2912c66cccaSKonstantin Belousov shrq $8,%rcx 2922c66cccaSKonstantin Belousov movb %cl,7(%rax) 2932c66cccaSKonstantin Belousov shrq $8,%rcx 2942c66cccaSKonstantin Belousov movl %ecx,8(%rax) 2952c66cccaSKonstantin Belousov movb $0x89,5(%rax) /* unset busy */ 2962c66cccaSKonstantin Belousov movl $TSSSEL,%eax 2972c66cccaSKonstantin Belousov ltr %ax 2982c66cccaSKonstantin Belousov jmp done_tss 2992c66cccaSKonstantin Belousov 3002c66cccaSKonstantin Belousovdo_ldt: movq PCPU(LDT),%rax 3012c66cccaSKonstantin Belousov movq P_MD+MD_LDT_SD(%rcx),%rdx 3022c66cccaSKonstantin Belousov movq %rdx,(%rax) 3032c66cccaSKonstantin Belousov movq P_MD+MD_LDT_SD+8(%rcx),%rdx 3042c66cccaSKonstantin Belousov movq %rdx,8(%rax) 3052c66cccaSKonstantin Belousov movl $LDTSEL,%eax 3062c66cccaSKonstantin Belousov jmp ld_ldt 307ea497502SJoseph KoshyEND(cpu_switch) 3080967373eSDavid Greenman 3090967373eSDavid Greenman/* 3102924d491SDavid Greenman * savectx(pcb) 3112924d491SDavid Greenman * Update pcb, saving current processor state. 3120967373eSDavid Greenman */ 3130967373eSDavid GreenmanENTRY(savectx) 314afa88623SPeter Wemm /* Save caller's return address. */ 315afa88623SPeter Wemm movq (%rsp),%rax 3163ab42a25SJung-uk Kim movq %rax,PCB_RIP(%rdi) 3172924d491SDavid Greenman 3183ab42a25SJung-uk Kim movq %rbx,PCB_RBX(%rdi) 3193ab42a25SJung-uk Kim movq %rsp,PCB_RSP(%rdi) 3203ab42a25SJung-uk Kim movq %rbp,PCB_RBP(%rdi) 3213ab42a25SJung-uk Kim movq %r12,PCB_R12(%rdi) 3223ab42a25SJung-uk Kim movq %r13,PCB_R13(%rdi) 3233ab42a25SJung-uk Kim movq %r14,PCB_R14(%rdi) 3243ab42a25SJung-uk Kim movq %r15,PCB_R15(%rdi) 325a2d2c836SJung-uk Kim 32605acaa9fSJung-uk Kim movq %cr0,%rax 32705acaa9fSJung-uk Kim movq %rax,PCB_CR0(%rdi) 328a2d2c836SJung-uk Kim movq %cr2,%rax 3293ab42a25SJung-uk Kim movq %rax,PCB_CR2(%rdi) 330afa88623SPeter Wemm movq %cr3,%rax 3313ab42a25SJung-uk Kim movq %rax,PCB_CR3(%rdi) 332a2d2c836SJung-uk Kim movq %cr4,%rax 3333ab42a25SJung-uk Kim movq %rax,PCB_CR4(%rdi) 334ac5f943cSPeter Wemm 335a2d2c836SJung-uk Kim movq %dr0,%rax 3363ab42a25SJung-uk Kim movq %rax,PCB_DR0(%rdi) 337a2d2c836SJung-uk Kim movq %dr1,%rax 3383ab42a25SJung-uk Kim movq %rax,PCB_DR1(%rdi) 339a2d2c836SJung-uk Kim movq %dr2,%rax 3403ab42a25SJung-uk Kim movq %rax,PCB_DR2(%rdi) 341a2d2c836SJung-uk Kim movq %dr3,%rax 3423ab42a25SJung-uk Kim movq %rax,PCB_DR3(%rdi) 343a2d2c836SJung-uk Kim movq %dr6,%rax 3443ab42a25SJung-uk Kim movq %rax,PCB_DR6(%rdi) 345a2d2c836SJung-uk Kim movq %dr7,%rax 3463ab42a25SJung-uk Kim movq %rax,PCB_DR7(%rdi) 347a2d2c836SJung-uk Kim 348a2d2c836SJung-uk Kim movl $MSR_FSBASE,%ecx 349a2d2c836SJung-uk Kim rdmsr 350305c5c0aSJung-uk Kim movl %eax,PCB_FSBASE(%rdi) 351305c5c0aSJung-uk Kim movl %edx,PCB_FSBASE+4(%rdi) 352a2d2c836SJung-uk Kim movl $MSR_GSBASE,%ecx 353a2d2c836SJung-uk Kim rdmsr 354305c5c0aSJung-uk Kim movl %eax,PCB_GSBASE(%rdi) 355305c5c0aSJung-uk Kim movl %edx,PCB_GSBASE+4(%rdi) 356a2d2c836SJung-uk Kim movl $MSR_KGSBASE,%ecx 357a2d2c836SJung-uk Kim rdmsr 358305c5c0aSJung-uk Kim movl %eax,PCB_KGSBASE(%rdi) 359305c5c0aSJung-uk Kim movl %edx,PCB_KGSBASE+4(%rdi) 360fb864578SMitsuru IWASAKI movl $MSR_EFER,%ecx 361fb864578SMitsuru IWASAKI rdmsr 362fb864578SMitsuru IWASAKI movl %eax,PCB_EFER(%rdi) 363fb864578SMitsuru IWASAKI movl %edx,PCB_EFER+4(%rdi) 364fb864578SMitsuru IWASAKI movl $MSR_STAR,%ecx 365fb864578SMitsuru IWASAKI rdmsr 366fb864578SMitsuru IWASAKI movl %eax,PCB_STAR(%rdi) 367fb864578SMitsuru IWASAKI movl %edx,PCB_STAR+4(%rdi) 368fb864578SMitsuru IWASAKI movl $MSR_LSTAR,%ecx 369fb864578SMitsuru IWASAKI rdmsr 370fb864578SMitsuru IWASAKI movl %eax,PCB_LSTAR(%rdi) 371fb864578SMitsuru IWASAKI movl %edx,PCB_LSTAR+4(%rdi) 372fb864578SMitsuru IWASAKI movl $MSR_CSTAR,%ecx 373fb864578SMitsuru IWASAKI rdmsr 374fb864578SMitsuru IWASAKI movl %eax,PCB_CSTAR(%rdi) 375fb864578SMitsuru IWASAKI movl %edx,PCB_CSTAR+4(%rdi) 376fb864578SMitsuru IWASAKI movl $MSR_SF_MASK,%ecx 377fb864578SMitsuru IWASAKI rdmsr 378fb864578SMitsuru IWASAKI movl %eax,PCB_SFMASK(%rdi) 379fb864578SMitsuru IWASAKI movl %edx,PCB_SFMASK+4(%rdi) 380a2d2c836SJung-uk Kim 3813ab42a25SJung-uk Kim sgdt PCB_GDT(%rdi) 3823ab42a25SJung-uk Kim sidt PCB_IDT(%rdi) 3833ab42a25SJung-uk Kim sldt PCB_LDT(%rdi) 3843ab42a25SJung-uk Kim str PCB_TR(%rdi) 385a2d2c836SJung-uk Kim 386a2d2c836SJung-uk Kim movl $1,%eax 3870967373eSDavid Greenman ret 388ea497502SJoseph KoshyEND(savectx) 3898c6f8f3dSKonstantin Belousov 3908c6f8f3dSKonstantin Belousov/* 3916ad79910SJung-uk Kim * resumectx(pcb) 392fb864578SMitsuru IWASAKI * Resuming processor state from pcb. 393fb864578SMitsuru IWASAKI */ 394fb864578SMitsuru IWASAKIENTRY(resumectx) 395fb864578SMitsuru IWASAKI /* Switch to KPML4phys. */ 3966ad79910SJung-uk Kim movq KPML4phys,%rax 3976ad79910SJung-uk Kim movq %rax,%cr3 398fb864578SMitsuru IWASAKI 399fb864578SMitsuru IWASAKI /* Force kernel segment registers. */ 400fb864578SMitsuru IWASAKI movl $KDSEL,%eax 401fb864578SMitsuru IWASAKI movw %ax,%ds 402fb864578SMitsuru IWASAKI movw %ax,%es 403fb864578SMitsuru IWASAKI movw %ax,%ss 404fb864578SMitsuru IWASAKI movl $KUF32SEL,%eax 405fb864578SMitsuru IWASAKI movw %ax,%fs 406fb864578SMitsuru IWASAKI movl $KUG32SEL,%eax 407fb864578SMitsuru IWASAKI movw %ax,%gs 408fb864578SMitsuru IWASAKI 409fb864578SMitsuru IWASAKI movl $MSR_FSBASE,%ecx 410fb864578SMitsuru IWASAKI movl PCB_FSBASE(%rdi),%eax 411fb864578SMitsuru IWASAKI movl 4 + PCB_FSBASE(%rdi),%edx 412fb864578SMitsuru IWASAKI wrmsr 413fb864578SMitsuru IWASAKI movl $MSR_GSBASE,%ecx 414fb864578SMitsuru IWASAKI movl PCB_GSBASE(%rdi),%eax 415fb864578SMitsuru IWASAKI movl 4 + PCB_GSBASE(%rdi),%edx 416fb864578SMitsuru IWASAKI wrmsr 417fb864578SMitsuru IWASAKI movl $MSR_KGSBASE,%ecx 418fb864578SMitsuru IWASAKI movl PCB_KGSBASE(%rdi),%eax 419fb864578SMitsuru IWASAKI movl 4 + PCB_KGSBASE(%rdi),%edx 420fb864578SMitsuru IWASAKI wrmsr 421fb864578SMitsuru IWASAKI 422bd101a66SKonstantin Belousov /* Restore EFER one more time. */ 423fb864578SMitsuru IWASAKI movl $MSR_EFER,%ecx 424fb864578SMitsuru IWASAKI movl PCB_EFER(%rdi),%eax 425fb864578SMitsuru IWASAKI wrmsr 426fb864578SMitsuru IWASAKI 427fb864578SMitsuru IWASAKI /* Restore fast syscall stuff. */ 428fb864578SMitsuru IWASAKI movl $MSR_STAR,%ecx 429fb864578SMitsuru IWASAKI movl PCB_STAR(%rdi),%eax 430fb864578SMitsuru IWASAKI movl 4 + PCB_STAR(%rdi),%edx 431fb864578SMitsuru IWASAKI wrmsr 432fb864578SMitsuru IWASAKI movl $MSR_LSTAR,%ecx 433fb864578SMitsuru IWASAKI movl PCB_LSTAR(%rdi),%eax 434fb864578SMitsuru IWASAKI movl 4 + PCB_LSTAR(%rdi),%edx 435fb864578SMitsuru IWASAKI wrmsr 436fb864578SMitsuru IWASAKI movl $MSR_CSTAR,%ecx 437fb864578SMitsuru IWASAKI movl PCB_CSTAR(%rdi),%eax 438fb864578SMitsuru IWASAKI movl 4 + PCB_CSTAR(%rdi),%edx 439fb864578SMitsuru IWASAKI wrmsr 440fb864578SMitsuru IWASAKI movl $MSR_SF_MASK,%ecx 441fb864578SMitsuru IWASAKI movl PCB_SFMASK(%rdi),%eax 442fb864578SMitsuru IWASAKI wrmsr 443fb864578SMitsuru IWASAKI 444b1d735baSJohn Baldwin /* Restore CR0, CR2, CR4 and CR3. */ 445fb864578SMitsuru IWASAKI movq PCB_CR0(%rdi),%rax 446fb864578SMitsuru IWASAKI movq %rax,%cr0 447fb864578SMitsuru IWASAKI movq PCB_CR2(%rdi),%rax 448fb864578SMitsuru IWASAKI movq %rax,%cr2 449fb864578SMitsuru IWASAKI movq PCB_CR4(%rdi),%rax 450fb864578SMitsuru IWASAKI movq %rax,%cr4 451fb864578SMitsuru IWASAKI movq PCB_CR3(%rdi),%rax 452fb864578SMitsuru IWASAKI movq %rax,%cr3 453fb864578SMitsuru IWASAKI 454fb864578SMitsuru IWASAKI /* Restore descriptor tables. */ 455fb864578SMitsuru IWASAKI lidt PCB_IDT(%rdi) 456fb864578SMitsuru IWASAKI lldt PCB_LDT(%rdi) 457fb864578SMitsuru IWASAKI 458fb864578SMitsuru IWASAKI#define SDT_SYSTSS 9 459fb864578SMitsuru IWASAKI#define SDT_SYSBSY 11 460fb864578SMitsuru IWASAKI 461fb864578SMitsuru IWASAKI /* Clear "task busy" bit and reload TR. */ 462fb864578SMitsuru IWASAKI movq PCPU(TSS),%rax 463fb864578SMitsuru IWASAKI andb $(~SDT_SYSBSY | SDT_SYSTSS),5(%rax) 464fb864578SMitsuru IWASAKI movw PCB_TR(%rdi),%ax 465fb864578SMitsuru IWASAKI ltr %ax 466fb864578SMitsuru IWASAKI 467fb864578SMitsuru IWASAKI#undef SDT_SYSTSS 468fb864578SMitsuru IWASAKI#undef SDT_SYSBSY 469fb864578SMitsuru IWASAKI 470fb864578SMitsuru IWASAKI /* Restore debug registers. */ 471fb864578SMitsuru IWASAKI movq PCB_DR0(%rdi),%rax 472fb864578SMitsuru IWASAKI movq %rax,%dr0 473fb864578SMitsuru IWASAKI movq PCB_DR1(%rdi),%rax 474fb864578SMitsuru IWASAKI movq %rax,%dr1 475fb864578SMitsuru IWASAKI movq PCB_DR2(%rdi),%rax 476fb864578SMitsuru IWASAKI movq %rax,%dr2 477fb864578SMitsuru IWASAKI movq PCB_DR3(%rdi),%rax 478fb864578SMitsuru IWASAKI movq %rax,%dr3 479fb864578SMitsuru IWASAKI movq PCB_DR6(%rdi),%rax 480fb864578SMitsuru IWASAKI movq %rax,%dr6 481fb864578SMitsuru IWASAKI movq PCB_DR7(%rdi),%rax 482fb864578SMitsuru IWASAKI movq %rax,%dr7 483fb864578SMitsuru IWASAKI 484fb864578SMitsuru IWASAKI /* Restore other callee saved registers. */ 485fb864578SMitsuru IWASAKI movq PCB_R15(%rdi),%r15 486fb864578SMitsuru IWASAKI movq PCB_R14(%rdi),%r14 487fb864578SMitsuru IWASAKI movq PCB_R13(%rdi),%r13 488fb864578SMitsuru IWASAKI movq PCB_R12(%rdi),%r12 489fb864578SMitsuru IWASAKI movq PCB_RBP(%rdi),%rbp 490fb864578SMitsuru IWASAKI movq PCB_RSP(%rdi),%rsp 491fb864578SMitsuru IWASAKI movq PCB_RBX(%rdi),%rbx 492fb864578SMitsuru IWASAKI 493fb864578SMitsuru IWASAKI /* Restore return address. */ 494fb864578SMitsuru IWASAKI movq PCB_RIP(%rdi),%rax 495fb864578SMitsuru IWASAKI movq %rax,(%rsp) 496fb864578SMitsuru IWASAKI 497fb864578SMitsuru IWASAKI xorl %eax,%eax 498fb864578SMitsuru IWASAKI ret 499fb864578SMitsuru IWASAKIEND(resumectx) 500