10967373eSDavid Greenman/*- 2fcfe57d6SPeter Wemm * Copyright (c) 2003 Peter Wemm. 30967373eSDavid Greenman * Copyright (c) 1990 The Regents of the University of California. 40967373eSDavid Greenman * All rights reserved. 50967373eSDavid Greenman * 60967373eSDavid Greenman * This code is derived from software contributed to Berkeley by 70967373eSDavid Greenman * William Jolitz. 80967373eSDavid Greenman * 90967373eSDavid Greenman * Redistribution and use in source and binary forms, with or without 100967373eSDavid Greenman * modification, are permitted provided that the following conditions 110967373eSDavid Greenman * are met: 120967373eSDavid Greenman * 1. Redistributions of source code must retain the above copyright 130967373eSDavid Greenman * notice, this list of conditions and the following disclaimer. 140967373eSDavid Greenman * 2. Redistributions in binary form must reproduce the above copyright 150967373eSDavid Greenman * notice, this list of conditions and the following disclaimer in the 160967373eSDavid Greenman * documentation and/or other materials provided with the distribution. 170967373eSDavid Greenman * 4. Neither the name of the University nor the names of its contributors 180967373eSDavid Greenman * may be used to endorse or promote products derived from this software 190967373eSDavid Greenman * without specific prior written permission. 200967373eSDavid Greenman * 210967373eSDavid Greenman * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 220967373eSDavid Greenman * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 230967373eSDavid Greenman * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 240967373eSDavid Greenman * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 250967373eSDavid Greenman * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 260967373eSDavid Greenman * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 270967373eSDavid Greenman * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 280967373eSDavid Greenman * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 290967373eSDavid Greenman * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 300967373eSDavid Greenman * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 310967373eSDavid Greenman * SUCH DAMAGE. 320967373eSDavid Greenman * 33c3aac50fSPeter Wemm * $FreeBSD$ 340967373eSDavid Greenman */ 350967373eSDavid Greenman 360513ce7fSBruce Evans#include <machine/asmacros.h> 37db527225SPeter Wemm#include <machine/specialreg.h> 38477a642cSPeter Wemm 390513ce7fSBruce Evans#include "assym.s" 4040380a6aSJeff Roberson#include "opt_sched.h" 410513ce7fSBruce Evans 420967373eSDavid Greenman/*****************************************************************************/ 430967373eSDavid Greenman/* Scheduling */ 440967373eSDavid Greenman/*****************************************************************************/ 450967373eSDavid Greenman 460967373eSDavid Greenman .text 470967373eSDavid Greenman 4851621230SPeter Wemm#ifdef SMP 4951621230SPeter Wemm#define LK lock ; 5051621230SPeter Wemm#else 5151621230SPeter Wemm#define LK 5251621230SPeter Wemm#endif 5351621230SPeter Wemm 5440380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 5540380a6aSJeff Roberson#define SETLK xchgq 5640380a6aSJeff Roberson#else 5740380a6aSJeff Roberson#define SETLK movq 5840380a6aSJeff Roberson#endif 5940380a6aSJeff Roberson 600967373eSDavid Greenman/* 610384fff8SJason Evans * cpu_throw() 62e602ba25SJulian Elischer * 632c87e001SPeter Wemm * This is the second half of cpu_switch(). It is used when the current 64e602ba25SJulian Elischer * thread is either a dummy or slated to die, and we no longer care 65cc66ebe2SPeter Wemm * about its state. This is only a slight optimization and is probably 66cc66ebe2SPeter Wemm * not worth it anymore. Note that we need to clear the pm_active bits so 67cc66ebe2SPeter Wemm * we do need the old proc if it still exists. 68afa88623SPeter Wemm * %rdi = oldtd 69afa88623SPeter Wemm * %rsi = newtd 700384fff8SJason Evans */ 710384fff8SJason EvansENTRY(cpu_throw) 72afa88623SPeter Wemm movl PCPU(CPUID),%eax 7377cb6e6fSAlan Cox testq %rdi,%rdi 7477cb6e6fSAlan Cox jz 1f 75cc66ebe2SPeter Wemm /* release bit from old pm_active */ 7677cb6e6fSAlan Cox movq PCPU(CURPMAP),%rdx 7777cb6e6fSAlan Cox LK btrl %eax,PM_ACTIVE(%rdx) /* clear old */ 7877cb6e6fSAlan Cox1: 7949a30208SJohn Baldwin movq TD_PCB(%rsi),%r8 /* newtd->td_pcb */ 8037eed841SKonstantin Belousov movq PCB_CR3(%r8),%rcx /* new address space */ 815d68dad3SJeff Roberson jmp swact 82ea497502SJoseph KoshyEND(cpu_throw) 830384fff8SJason Evans 840384fff8SJason Evans/* 855d68dad3SJeff Roberson * cpu_switch(old, new, mtx) 86e602ba25SJulian Elischer * 87e602ba25SJulian Elischer * Save the current thread state, then select the next thread to run 88e602ba25SJulian Elischer * and load its state. 89afa88623SPeter Wemm * %rdi = oldtd 90afa88623SPeter Wemm * %rsi = newtd 915d68dad3SJeff Roberson * %rdx = mtx 920967373eSDavid Greenman */ 9326f9a767SRodney W. GrimesENTRY(cpu_switch) 94cc66ebe2SPeter Wemm /* Switch to new thread. First, save context. */ 95bf1e8974SPeter Wemm movq TD_PCB(%rdi),%r8 9650e3cec3SJung-uk Kim orl $PCB_FULL_IRET,PCB_FLAGS(%r8) 970967373eSDavid Greenman 98afa88623SPeter Wemm movq (%rsp),%rax /* Hardware registers */ 99bf1e8974SPeter Wemm movq %r15,PCB_R15(%r8) 100f001eabfSPeter Wemm movq %r14,PCB_R14(%r8) 101f001eabfSPeter Wemm movq %r13,PCB_R13(%r8) 102f001eabfSPeter Wemm movq %r12,PCB_R12(%r8) 103f001eabfSPeter Wemm movq %rbp,PCB_RBP(%r8) 104f001eabfSPeter Wemm movq %rsp,PCB_RSP(%r8) 105f001eabfSPeter Wemm movq %rbx,PCB_RBX(%r8) 106f001eabfSPeter Wemm movq %rax,PCB_RIP(%r8) 107f98c3ea7SKonstantin Belousov 10850e3cec3SJung-uk Kim testl $PCB_DBREGS,PCB_FLAGS(%r8) 109f001eabfSPeter Wemm jnz store_dr /* static predict not taken */ 110f001eabfSPeter Wemmdone_store_dr: 111db527225SPeter Wemm 1120967373eSDavid Greenman /* have we used fp, and need a save? */ 113afa88623SPeter Wemm cmpq %rdi,PCPU(FPCURTHREAD) 1148c6f8f3dSKonstantin Belousov jne 3f 1156cf9a08dSKonstantin Belousov movq PCB_SAVEFPU(%r8),%r8 116db527225SPeter Wemm clts 1178c6f8f3dSKonstantin Belousov cmpl $0,use_xsave 1188c6f8f3dSKonstantin Belousov jne 1f 119db527225SPeter Wemm fxsave (%r8) 1208c6f8f3dSKonstantin Belousov jmp 2f 1218c6f8f3dSKonstantin Belousov1: movq %rdx,%rcx 1228c6f8f3dSKonstantin Belousov movl xsave_mask,%eax 1238c6f8f3dSKonstantin Belousov movl xsave_mask+4,%edx 124333d0c60SKonstantin Belousov .globl ctx_switch_xsave 125333d0c60SKonstantin Belousovctx_switch_xsave: 126333d0c60SKonstantin Belousov /* This is patched to xsaveopt if supported, see fpuinit_bsp1() */ 127f18d5bf4SKonstantin Belousov xsave (%r8) 1288c6f8f3dSKonstantin Belousov movq %rcx,%rdx 1298c6f8f3dSKonstantin Belousov2: smsw %ax 130db527225SPeter Wemm orb $CR0_TS,%al 131db527225SPeter Wemm lmsw %ax 13298df9e00SPeter Wemm xorl %eax,%eax 133db527225SPeter Wemm movq %rax,PCPU(FPCURTHREAD) 1348c6f8f3dSKonstantin Belousov3: 1350967373eSDavid Greenman 136cc66ebe2SPeter Wemm /* Save is done. Now fire up new thread. Leave old vmspace. */ 137bf1e8974SPeter Wemm movq TD_PCB(%rsi),%r8 138b40ce416SJulian Elischer 1390967373eSDavid Greenman /* switch address space */ 1405d68dad3SJeff Roberson movq PCB_CR3(%r8),%rcx 141db527225SPeter Wemm movq %cr3,%rax 1425d68dad3SJeff Roberson cmpq %rcx,%rax /* Same address space? */ 1435d68dad3SJeff Roberson jne swinact 14440380a6aSJeff Roberson SETLK %rdx, TD_LOCK(%rdi) /* Release the old thread */ 1455d68dad3SJeff Roberson jmp sw1 1465d68dad3SJeff Robersonswinact: 147db527225SPeter Wemm movl PCPU(CPUID),%eax 148cc66ebe2SPeter Wemm /* Release bit from old pmap->pm_active */ 14937eed841SKonstantin Belousov movq PCPU(CURPMAP),%r12 15037eed841SKonstantin Belousov LK btrl %eax,PM_ACTIVE(%r12) /* clear old */ 15140380a6aSJeff Roberson SETLK %rdx,TD_LOCK(%rdi) /* Release the old thread */ 1525d68dad3SJeff Robersonswact: 153cc66ebe2SPeter Wemm /* Set bit in new pmap->pm_active */ 154afa88623SPeter Wemm movq TD_PROC(%rsi),%rdx /* newproc */ 155afa88623SPeter Wemm movq P_VMSPACE(%rdx), %rdx 15677cb6e6fSAlan Cox addq $VM_PMAP,%rdx 15737eed841SKonstantin Belousov cmpl $-1,PM_PCID(%rdx) 15837eed841SKonstantin Belousov je 1f 15937eed841SKonstantin Belousov LK btsl %eax,PM_SAVE(%rdx) 16037eed841SKonstantin Belousov jnc 1f 16137eed841SKonstantin Belousov btsq $63,%rcx /* CR3_PCID_SAVE */ 16237eed841SKonstantin Belousov incq PCPU(PM_SAVE_CNT) 16337eed841SKonstantin Belousov1: 16437eed841SKonstantin Belousov movq %rcx,%cr3 /* new address space */ 16577cb6e6fSAlan Cox LK btsl %eax,PM_ACTIVE(%rdx) /* set new */ 16677cb6e6fSAlan Cox movq %rdx,PCPU(CURPMAP) 167cc66ebe2SPeter Wemm 16837eed841SKonstantin Belousov /* 16937eed841SKonstantin Belousov * We might lose the race and other CPU might have changed 17037eed841SKonstantin Belousov * the pmap after we set our bit in pmap->pm_save. Recheck. 17137eed841SKonstantin Belousov * Reload %cr3 with CR3_PCID_SAVE bit cleared if pmap was 17237eed841SKonstantin Belousov * modified, causing TLB flush for this pcid. 17337eed841SKonstantin Belousov */ 17437eed841SKonstantin Belousov btrq $63,%rcx 17537eed841SKonstantin Belousov jnc 1f 17637eed841SKonstantin Belousov LK btsl %eax,PM_SAVE(%rdx) 17737eed841SKonstantin Belousov jc 1f 17837eed841SKonstantin Belousov decq PCPU(PM_SAVE_CNT) 17937eed841SKonstantin Belousov movq %rcx,%cr3 18037eed841SKonstantin Belousov1: 18137eed841SKonstantin Belousov 182cc66ebe2SPeter Wemmsw1: 18340380a6aSJeff Roberson#if defined(SCHED_ULE) && defined(SMP) 18440380a6aSJeff Roberson /* Wait for the new thread to become unblocked */ 18540380a6aSJeff Roberson movq $blocked_lock, %rdx 18640380a6aSJeff Roberson1: 18740380a6aSJeff Roberson movq TD_LOCK(%rsi),%rcx 18840380a6aSJeff Roberson cmpq %rcx, %rdx 18940380a6aSJeff Roberson pause 19040380a6aSJeff Roberson je 1b 19140380a6aSJeff Roberson#endif 192cc66ebe2SPeter Wemm /* 193cc66ebe2SPeter Wemm * At this point, we've switched address spaces and are ready 194cc66ebe2SPeter Wemm * to load up the rest of the next context. 195cc66ebe2SPeter Wemm */ 196bf1e8974SPeter Wemm 197f001eabfSPeter Wemm /* Skip loading user fsbase/gsbase for kthreads */ 198f001eabfSPeter Wemm testl $TDP_KTHREAD,TD_PFLAGS(%rsi) 199f001eabfSPeter Wemm jnz do_kthread 200f001eabfSPeter Wemm 2012c66cccaSKonstantin Belousov /* 2022c66cccaSKonstantin Belousov * Load ldt register 2032c66cccaSKonstantin Belousov */ 2042c66cccaSKonstantin Belousov movq TD_PROC(%rsi),%rcx 2052c66cccaSKonstantin Belousov cmpq $0, P_MD+MD_LDT(%rcx) 2062c66cccaSKonstantin Belousov jne do_ldt 2072c66cccaSKonstantin Belousov xorl %eax,%eax 2082c66cccaSKonstantin Belousovld_ldt: lldt %ax 2098f4a1f3aSKonstantin Belousov 2102c66cccaSKonstantin Belousov /* Restore fs base in GDT */ 211bf1e8974SPeter Wemm movl PCB_FSBASE(%r8),%eax 2122c66cccaSKonstantin Belousov movq PCPU(FS32P),%rdx 2132c66cccaSKonstantin Belousov movw %ax,2(%rdx) 2142c66cccaSKonstantin Belousov shrl $16,%eax 2152c66cccaSKonstantin Belousov movb %al,4(%rdx) 2162c66cccaSKonstantin Belousov shrl $8,%eax 2172c66cccaSKonstantin Belousov movb %al,7(%rdx) 2189869fa1dSJohn Baldwin 2192c66cccaSKonstantin Belousov /* Restore gs base in GDT */ 2202c66cccaSKonstantin Belousov movl PCB_GSBASE(%r8),%eax 2212c66cccaSKonstantin Belousov movq PCPU(GS32P),%rdx 2222c66cccaSKonstantin Belousov movw %ax,2(%rdx) 2232c66cccaSKonstantin Belousov shrl $16,%eax 2242c66cccaSKonstantin Belousov movb %al,4(%rdx) 2252c66cccaSKonstantin Belousov shrl $8,%eax 2262c66cccaSKonstantin Belousov movb %al,7(%rdx) 2272c66cccaSKonstantin Belousov 2282c66cccaSKonstantin Belousovdo_kthread: 2292c66cccaSKonstantin Belousov /* Do we need to reload tss ? */ 2300d2a2989SPeter Wemm movq PCPU(TSSP),%rax 2312c66cccaSKonstantin Belousov movq PCB_TSSP(%r8),%rdx 2322c66cccaSKonstantin Belousov testq %rdx,%rdx 2332c66cccaSKonstantin Belousov cmovzq PCPU(COMMONTSSP),%rdx 2342c66cccaSKonstantin Belousov cmpq %rax,%rdx 2352c66cccaSKonstantin Belousov jne do_tss 2362c66cccaSKonstantin Belousovdone_tss: 237f001eabfSPeter Wemm movq %r8,PCPU(RSP0) 2384e32b7b3SDavid Xu movq %r8,PCPU(CURPCB) 2392c66cccaSKonstantin Belousov /* Update the TSS_RSP0 pointer for the next interrupt */ 2402c66cccaSKonstantin Belousov movq %r8,COMMON_TSS_RSP0(%rdx) 2414e32b7b3SDavid Xu movq %rsi,PCPU(CURTHREAD) /* into next thread */ 242f001eabfSPeter Wemm 243f001eabfSPeter Wemm /* Test if debug registers should be restored. */ 24450e3cec3SJung-uk Kim testl $PCB_DBREGS,PCB_FLAGS(%r8) 245f001eabfSPeter Wemm jnz load_dr /* static predict not taken */ 246f001eabfSPeter Wemmdone_load_dr: 2474e32b7b3SDavid Xu 248f001eabfSPeter Wemm /* Restore context. */ 249f001eabfSPeter Wemm movq PCB_R15(%r8),%r15 250f001eabfSPeter Wemm movq PCB_R14(%r8),%r14 251f001eabfSPeter Wemm movq PCB_R13(%r8),%r13 252f001eabfSPeter Wemm movq PCB_R12(%r8),%r12 253f001eabfSPeter Wemm movq PCB_RBP(%r8),%rbp 254f001eabfSPeter Wemm movq PCB_RSP(%r8),%rsp 255f001eabfSPeter Wemm movq PCB_RBX(%r8),%rbx 256f001eabfSPeter Wemm movq PCB_RIP(%r8),%rax 257f001eabfSPeter Wemm movq %rax,(%rsp) 258f001eabfSPeter Wemm ret 259f001eabfSPeter Wemm 260f001eabfSPeter Wemm /* 261f001eabfSPeter Wemm * We order these strangely for several reasons. 262f001eabfSPeter Wemm * 1: I wanted to use static branch prediction hints 263f001eabfSPeter Wemm * 2: Most athlon64/opteron cpus don't have them. They define 264f001eabfSPeter Wemm * a forward branch as 'predict not taken'. Intel cores have 265f001eabfSPeter Wemm * the 'rep' prefix to invert this. 266f001eabfSPeter Wemm * So, to make it work on both forms of cpu we do the detour. 267f001eabfSPeter Wemm * We use jumps rather than call in order to avoid the stack. 268f001eabfSPeter Wemm */ 269f001eabfSPeter Wemm 270f001eabfSPeter Wemmstore_dr: 271f001eabfSPeter Wemm movq %dr7,%rax /* yes, do the save */ 272f001eabfSPeter Wemm movq %dr0,%r15 273f001eabfSPeter Wemm movq %dr1,%r14 274f001eabfSPeter Wemm movq %dr2,%r13 275f001eabfSPeter Wemm movq %dr3,%r12 276f001eabfSPeter Wemm movq %dr6,%r11 277f001eabfSPeter Wemm movq %r15,PCB_DR0(%r8) 278f001eabfSPeter Wemm movq %r14,PCB_DR1(%r8) 279f001eabfSPeter Wemm movq %r13,PCB_DR2(%r8) 280f001eabfSPeter Wemm movq %r12,PCB_DR3(%r8) 281f001eabfSPeter Wemm movq %r11,PCB_DR6(%r8) 282f001eabfSPeter Wemm movq %rax,PCB_DR7(%r8) 283aaa95ccbSKonstantin Belousov andq $0x0000fc00, %rax /* disable all watchpoints */ 284db527225SPeter Wemm movq %rax,%dr7 285f001eabfSPeter Wemm jmp done_store_dr 286f001eabfSPeter Wemm 287f001eabfSPeter Wemmload_dr: 288f001eabfSPeter Wemm movq %dr7,%rax 289f001eabfSPeter Wemm movq PCB_DR0(%r8),%r15 290f001eabfSPeter Wemm movq PCB_DR1(%r8),%r14 291f001eabfSPeter Wemm movq PCB_DR2(%r8),%r13 292f001eabfSPeter Wemm movq PCB_DR3(%r8),%r12 293f001eabfSPeter Wemm movq PCB_DR6(%r8),%r11 294f001eabfSPeter Wemm movq PCB_DR7(%r8),%rcx 295f001eabfSPeter Wemm movq %r15,%dr0 296f001eabfSPeter Wemm movq %r14,%dr1 297f001eabfSPeter Wemm /* Preserve reserved bits in %dr7 */ 298f001eabfSPeter Wemm andq $0x0000fc00,%rax 299f001eabfSPeter Wemm andq $~0x0000fc00,%rcx 300f001eabfSPeter Wemm movq %r13,%dr2 301f001eabfSPeter Wemm movq %r12,%dr3 302f001eabfSPeter Wemm orq %rcx,%rax 303f001eabfSPeter Wemm movq %r11,%dr6 304f001eabfSPeter Wemm movq %rax,%dr7 305f001eabfSPeter Wemm jmp done_load_dr 3062c66cccaSKonstantin Belousov 3072c66cccaSKonstantin Belousovdo_tss: movq %rdx,PCPU(TSSP) 3082c66cccaSKonstantin Belousov movq %rdx,%rcx 3092c66cccaSKonstantin Belousov movq PCPU(TSS),%rax 310cfe92f33SDimitry Andric movw %cx,2(%rax) 3112c66cccaSKonstantin Belousov shrq $16,%rcx 3122c66cccaSKonstantin Belousov movb %cl,4(%rax) 3132c66cccaSKonstantin Belousov shrq $8,%rcx 3142c66cccaSKonstantin Belousov movb %cl,7(%rax) 3152c66cccaSKonstantin Belousov shrq $8,%rcx 3162c66cccaSKonstantin Belousov movl %ecx,8(%rax) 3172c66cccaSKonstantin Belousov movb $0x89,5(%rax) /* unset busy */ 3182c66cccaSKonstantin Belousov movl $TSSSEL,%eax 3192c66cccaSKonstantin Belousov ltr %ax 3202c66cccaSKonstantin Belousov jmp done_tss 3212c66cccaSKonstantin Belousov 3222c66cccaSKonstantin Belousovdo_ldt: movq PCPU(LDT),%rax 3232c66cccaSKonstantin Belousov movq P_MD+MD_LDT_SD(%rcx),%rdx 3242c66cccaSKonstantin Belousov movq %rdx,(%rax) 3252c66cccaSKonstantin Belousov movq P_MD+MD_LDT_SD+8(%rcx),%rdx 3262c66cccaSKonstantin Belousov movq %rdx,8(%rax) 3272c66cccaSKonstantin Belousov movl $LDTSEL,%eax 3282c66cccaSKonstantin Belousov jmp ld_ldt 329ea497502SJoseph KoshyEND(cpu_switch) 3300967373eSDavid Greenman 3310967373eSDavid Greenman/* 3322924d491SDavid Greenman * savectx(pcb) 3332924d491SDavid Greenman * Update pcb, saving current processor state. 3340967373eSDavid Greenman */ 3350967373eSDavid GreenmanENTRY(savectx) 336afa88623SPeter Wemm /* Save caller's return address. */ 337afa88623SPeter Wemm movq (%rsp),%rax 3383ab42a25SJung-uk Kim movq %rax,PCB_RIP(%rdi) 3392924d491SDavid Greenman 3403ab42a25SJung-uk Kim movq %rbx,PCB_RBX(%rdi) 3413ab42a25SJung-uk Kim movq %rsp,PCB_RSP(%rdi) 3423ab42a25SJung-uk Kim movq %rbp,PCB_RBP(%rdi) 3433ab42a25SJung-uk Kim movq %r12,PCB_R12(%rdi) 3443ab42a25SJung-uk Kim movq %r13,PCB_R13(%rdi) 3453ab42a25SJung-uk Kim movq %r14,PCB_R14(%rdi) 3463ab42a25SJung-uk Kim movq %r15,PCB_R15(%rdi) 347a2d2c836SJung-uk Kim 34805acaa9fSJung-uk Kim movq %cr0,%rax 34905acaa9fSJung-uk Kim movq %rax,PCB_CR0(%rdi) 350a2d2c836SJung-uk Kim movq %cr2,%rax 3513ab42a25SJung-uk Kim movq %rax,PCB_CR2(%rdi) 352afa88623SPeter Wemm movq %cr3,%rax 3533ab42a25SJung-uk Kim movq %rax,PCB_CR3(%rdi) 354a2d2c836SJung-uk Kim movq %cr4,%rax 3553ab42a25SJung-uk Kim movq %rax,PCB_CR4(%rdi) 356ac5f943cSPeter Wemm 357a2d2c836SJung-uk Kim movq %dr0,%rax 3583ab42a25SJung-uk Kim movq %rax,PCB_DR0(%rdi) 359a2d2c836SJung-uk Kim movq %dr1,%rax 3603ab42a25SJung-uk Kim movq %rax,PCB_DR1(%rdi) 361a2d2c836SJung-uk Kim movq %dr2,%rax 3623ab42a25SJung-uk Kim movq %rax,PCB_DR2(%rdi) 363a2d2c836SJung-uk Kim movq %dr3,%rax 3643ab42a25SJung-uk Kim movq %rax,PCB_DR3(%rdi) 365a2d2c836SJung-uk Kim movq %dr6,%rax 3663ab42a25SJung-uk Kim movq %rax,PCB_DR6(%rdi) 367a2d2c836SJung-uk Kim movq %dr7,%rax 3683ab42a25SJung-uk Kim movq %rax,PCB_DR7(%rdi) 369a2d2c836SJung-uk Kim 370a2d2c836SJung-uk Kim movl $MSR_FSBASE,%ecx 371a2d2c836SJung-uk Kim rdmsr 372305c5c0aSJung-uk Kim movl %eax,PCB_FSBASE(%rdi) 373305c5c0aSJung-uk Kim movl %edx,PCB_FSBASE+4(%rdi) 374a2d2c836SJung-uk Kim movl $MSR_GSBASE,%ecx 375a2d2c836SJung-uk Kim rdmsr 376305c5c0aSJung-uk Kim movl %eax,PCB_GSBASE(%rdi) 377305c5c0aSJung-uk Kim movl %edx,PCB_GSBASE+4(%rdi) 378a2d2c836SJung-uk Kim movl $MSR_KGSBASE,%ecx 379a2d2c836SJung-uk Kim rdmsr 380305c5c0aSJung-uk Kim movl %eax,PCB_KGSBASE(%rdi) 381305c5c0aSJung-uk Kim movl %edx,PCB_KGSBASE+4(%rdi) 382fb864578SMitsuru IWASAKI movl $MSR_EFER,%ecx 383fb864578SMitsuru IWASAKI rdmsr 384fb864578SMitsuru IWASAKI movl %eax,PCB_EFER(%rdi) 385fb864578SMitsuru IWASAKI movl %edx,PCB_EFER+4(%rdi) 386fb864578SMitsuru IWASAKI movl $MSR_STAR,%ecx 387fb864578SMitsuru IWASAKI rdmsr 388fb864578SMitsuru IWASAKI movl %eax,PCB_STAR(%rdi) 389fb864578SMitsuru IWASAKI movl %edx,PCB_STAR+4(%rdi) 390fb864578SMitsuru IWASAKI movl $MSR_LSTAR,%ecx 391fb864578SMitsuru IWASAKI rdmsr 392fb864578SMitsuru IWASAKI movl %eax,PCB_LSTAR(%rdi) 393fb864578SMitsuru IWASAKI movl %edx,PCB_LSTAR+4(%rdi) 394fb864578SMitsuru IWASAKI movl $MSR_CSTAR,%ecx 395fb864578SMitsuru IWASAKI rdmsr 396fb864578SMitsuru IWASAKI movl %eax,PCB_CSTAR(%rdi) 397fb864578SMitsuru IWASAKI movl %edx,PCB_CSTAR+4(%rdi) 398fb864578SMitsuru IWASAKI movl $MSR_SF_MASK,%ecx 399fb864578SMitsuru IWASAKI rdmsr 400fb864578SMitsuru IWASAKI movl %eax,PCB_SFMASK(%rdi) 401fb864578SMitsuru IWASAKI movl %edx,PCB_SFMASK+4(%rdi) 402fb864578SMitsuru IWASAKI movl xsave_mask,%eax 403fb864578SMitsuru IWASAKI movl %eax,PCB_XSMASK(%rdi) 404fb864578SMitsuru IWASAKI movl xsave_mask+4,%eax 405fb864578SMitsuru IWASAKI movl %eax,PCB_XSMASK+4(%rdi) 406a2d2c836SJung-uk Kim 4073ab42a25SJung-uk Kim sgdt PCB_GDT(%rdi) 4083ab42a25SJung-uk Kim sidt PCB_IDT(%rdi) 4093ab42a25SJung-uk Kim sldt PCB_LDT(%rdi) 4103ab42a25SJung-uk Kim str PCB_TR(%rdi) 411a2d2c836SJung-uk Kim 412a2d2c836SJung-uk Kim movl $1,%eax 4130967373eSDavid Greenman ret 414ea497502SJoseph KoshyEND(savectx) 4158c6f8f3dSKonstantin Belousov 4168c6f8f3dSKonstantin Belousov/* 4176ad79910SJung-uk Kim * resumectx(pcb) 418fb864578SMitsuru IWASAKI * Resuming processor state from pcb. 419fb864578SMitsuru IWASAKI */ 420fb864578SMitsuru IWASAKIENTRY(resumectx) 421fb864578SMitsuru IWASAKI /* Switch to KPML4phys. */ 4226ad79910SJung-uk Kim movq KPML4phys,%rax 4236ad79910SJung-uk Kim movq %rax,%cr3 424fb864578SMitsuru IWASAKI 425fb864578SMitsuru IWASAKI /* Force kernel segment registers. */ 426fb864578SMitsuru IWASAKI movl $KDSEL,%eax 427fb864578SMitsuru IWASAKI movw %ax,%ds 428fb864578SMitsuru IWASAKI movw %ax,%es 429fb864578SMitsuru IWASAKI movw %ax,%ss 430fb864578SMitsuru IWASAKI movl $KUF32SEL,%eax 431fb864578SMitsuru IWASAKI movw %ax,%fs 432fb864578SMitsuru IWASAKI movl $KUG32SEL,%eax 433fb864578SMitsuru IWASAKI movw %ax,%gs 434fb864578SMitsuru IWASAKI 435fb864578SMitsuru IWASAKI movl $MSR_FSBASE,%ecx 436fb864578SMitsuru IWASAKI movl PCB_FSBASE(%rdi),%eax 437fb864578SMitsuru IWASAKI movl 4 + PCB_FSBASE(%rdi),%edx 438fb864578SMitsuru IWASAKI wrmsr 439fb864578SMitsuru IWASAKI movl $MSR_GSBASE,%ecx 440fb864578SMitsuru IWASAKI movl PCB_GSBASE(%rdi),%eax 441fb864578SMitsuru IWASAKI movl 4 + PCB_GSBASE(%rdi),%edx 442fb864578SMitsuru IWASAKI wrmsr 443fb864578SMitsuru IWASAKI movl $MSR_KGSBASE,%ecx 444fb864578SMitsuru IWASAKI movl PCB_KGSBASE(%rdi),%eax 445fb864578SMitsuru IWASAKI movl 4 + PCB_KGSBASE(%rdi),%edx 446fb864578SMitsuru IWASAKI wrmsr 447fb864578SMitsuru IWASAKI 448fb864578SMitsuru IWASAKI /* Restore EFER. */ 449fb864578SMitsuru IWASAKI movl $MSR_EFER,%ecx 450fb864578SMitsuru IWASAKI movl PCB_EFER(%rdi),%eax 451fb864578SMitsuru IWASAKI wrmsr 452fb864578SMitsuru IWASAKI 453fb864578SMitsuru IWASAKI /* Restore fast syscall stuff. */ 454fb864578SMitsuru IWASAKI movl $MSR_STAR,%ecx 455fb864578SMitsuru IWASAKI movl PCB_STAR(%rdi),%eax 456fb864578SMitsuru IWASAKI movl 4 + PCB_STAR(%rdi),%edx 457fb864578SMitsuru IWASAKI wrmsr 458fb864578SMitsuru IWASAKI movl $MSR_LSTAR,%ecx 459fb864578SMitsuru IWASAKI movl PCB_LSTAR(%rdi),%eax 460fb864578SMitsuru IWASAKI movl 4 + PCB_LSTAR(%rdi),%edx 461fb864578SMitsuru IWASAKI wrmsr 462fb864578SMitsuru IWASAKI movl $MSR_CSTAR,%ecx 463fb864578SMitsuru IWASAKI movl PCB_CSTAR(%rdi),%eax 464fb864578SMitsuru IWASAKI movl 4 + PCB_CSTAR(%rdi),%edx 465fb864578SMitsuru IWASAKI wrmsr 466fb864578SMitsuru IWASAKI movl $MSR_SF_MASK,%ecx 467fb864578SMitsuru IWASAKI movl PCB_SFMASK(%rdi),%eax 468fb864578SMitsuru IWASAKI wrmsr 469fb864578SMitsuru IWASAKI 470fb864578SMitsuru IWASAKI /* Restore CR0 except for FPU mode. */ 471fb864578SMitsuru IWASAKI movq PCB_CR0(%rdi),%rax 472fb864578SMitsuru IWASAKI andq $~(CR0_EM | CR0_TS),%rax 473fb864578SMitsuru IWASAKI movq %rax,%cr0 474fb864578SMitsuru IWASAKI 475fb864578SMitsuru IWASAKI /* Restore CR2, CR4 and CR3. */ 476fb864578SMitsuru IWASAKI movq PCB_CR2(%rdi),%rax 477fb864578SMitsuru IWASAKI movq %rax,%cr2 478fb864578SMitsuru IWASAKI movq PCB_CR4(%rdi),%rax 479fb864578SMitsuru IWASAKI movq %rax,%cr4 480fb864578SMitsuru IWASAKI movq PCB_CR3(%rdi),%rax 481fb864578SMitsuru IWASAKI movq %rax,%cr3 482fb864578SMitsuru IWASAKI 483fb864578SMitsuru IWASAKI /* Restore descriptor tables. */ 484fb864578SMitsuru IWASAKI lidt PCB_IDT(%rdi) 485fb864578SMitsuru IWASAKI lldt PCB_LDT(%rdi) 486fb864578SMitsuru IWASAKI 487fb864578SMitsuru IWASAKI#define SDT_SYSTSS 9 488fb864578SMitsuru IWASAKI#define SDT_SYSBSY 11 489fb864578SMitsuru IWASAKI 490fb864578SMitsuru IWASAKI /* Clear "task busy" bit and reload TR. */ 491fb864578SMitsuru IWASAKI movq PCPU(TSS),%rax 492fb864578SMitsuru IWASAKI andb $(~SDT_SYSBSY | SDT_SYSTSS),5(%rax) 493fb864578SMitsuru IWASAKI movw PCB_TR(%rdi),%ax 494fb864578SMitsuru IWASAKI ltr %ax 495fb864578SMitsuru IWASAKI 496fb864578SMitsuru IWASAKI#undef SDT_SYSTSS 497fb864578SMitsuru IWASAKI#undef SDT_SYSBSY 498fb864578SMitsuru IWASAKI 499fb864578SMitsuru IWASAKI /* Restore debug registers. */ 500fb864578SMitsuru IWASAKI movq PCB_DR0(%rdi),%rax 501fb864578SMitsuru IWASAKI movq %rax,%dr0 502fb864578SMitsuru IWASAKI movq PCB_DR1(%rdi),%rax 503fb864578SMitsuru IWASAKI movq %rax,%dr1 504fb864578SMitsuru IWASAKI movq PCB_DR2(%rdi),%rax 505fb864578SMitsuru IWASAKI movq %rax,%dr2 506fb864578SMitsuru IWASAKI movq PCB_DR3(%rdi),%rax 507fb864578SMitsuru IWASAKI movq %rax,%dr3 508fb864578SMitsuru IWASAKI movq PCB_DR6(%rdi),%rax 509fb864578SMitsuru IWASAKI movq %rax,%dr6 510fb864578SMitsuru IWASAKI movq PCB_DR7(%rdi),%rax 511fb864578SMitsuru IWASAKI movq %rax,%dr7 512fb864578SMitsuru IWASAKI 513fb864578SMitsuru IWASAKI /* Restore FPU state. */ 514fb864578SMitsuru IWASAKI fninit 515fb864578SMitsuru IWASAKI movq PCB_FPUSUSPEND(%rdi),%rbx 516fb864578SMitsuru IWASAKI movq PCB_XSMASK(%rdi),%rax 517fb864578SMitsuru IWASAKI testq %rax,%rax 518fb864578SMitsuru IWASAKI jz 1f 519fb864578SMitsuru IWASAKI movq %rax,%rdx 520fb864578SMitsuru IWASAKI shrq $32,%rdx 521fb864578SMitsuru IWASAKI movl $XCR0,%ecx 522f18d5bf4SKonstantin Belousov xsetbv 523f18d5bf4SKonstantin Belousov xrstor (%rbx) 524fb864578SMitsuru IWASAKI jmp 2f 525fb864578SMitsuru IWASAKI1: 526fb864578SMitsuru IWASAKI fxrstor (%rbx) 527fb864578SMitsuru IWASAKI2: 528fb864578SMitsuru IWASAKI 529fb864578SMitsuru IWASAKI /* Reload CR0. */ 530fb864578SMitsuru IWASAKI movq PCB_CR0(%rdi),%rax 531fb864578SMitsuru IWASAKI movq %rax,%cr0 532fb864578SMitsuru IWASAKI 533fb864578SMitsuru IWASAKI /* Restore other callee saved registers. */ 534fb864578SMitsuru IWASAKI movq PCB_R15(%rdi),%r15 535fb864578SMitsuru IWASAKI movq PCB_R14(%rdi),%r14 536fb864578SMitsuru IWASAKI movq PCB_R13(%rdi),%r13 537fb864578SMitsuru IWASAKI movq PCB_R12(%rdi),%r12 538fb864578SMitsuru IWASAKI movq PCB_RBP(%rdi),%rbp 539fb864578SMitsuru IWASAKI movq PCB_RSP(%rdi),%rsp 540fb864578SMitsuru IWASAKI movq PCB_RBX(%rdi),%rbx 541fb864578SMitsuru IWASAKI 542fb864578SMitsuru IWASAKI /* Restore return address. */ 543fb864578SMitsuru IWASAKI movq PCB_RIP(%rdi),%rax 544fb864578SMitsuru IWASAKI movq %rax,(%rsp) 545fb864578SMitsuru IWASAKI 546fb864578SMitsuru IWASAKI xorl %eax,%eax 547fb864578SMitsuru IWASAKI ret 548fb864578SMitsuru IWASAKIEND(resumectx) 549fb864578SMitsuru IWASAKI 550fb864578SMitsuru IWASAKI/* 55105acaa9fSJung-uk Kim * Wrapper around fpusave to care about CR0_TS. 5528c6f8f3dSKonstantin Belousov */ 5538c6f8f3dSKonstantin BelousovENTRY(ctx_fpusave) 55405acaa9fSJung-uk Kim movq %cr0,%rax 5558c6f8f3dSKonstantin Belousov clts 5568c6f8f3dSKonstantin Belousov call fpusave 55705acaa9fSJung-uk Kim movq %rax,%cr0 5588c6f8f3dSKonstantin Belousov ret 5598c6f8f3dSKonstantin BelousovEND(ctx_fpusave) 560