1*012ea299Sniklas/* $OpenBSD: locore.S,v 1.11 2004/06/13 21:49:12 niklas Exp $ */ 2f5df1827Smickey/* $NetBSD: locore.S,v 1.2 2003/04/26 19:34:45 fvdl Exp $ */ 3f5df1827Smickey 4f5df1827Smickey/* 5f5df1827Smickey * Copyright-o-rama! 6f5df1827Smickey */ 7f5df1827Smickey 8f5df1827Smickey/* 9f5df1827Smickey * Copyright (c) 2001 Wasabi Systems, Inc. 10f5df1827Smickey * All rights reserved. 11f5df1827Smickey * 12f5df1827Smickey * Written by Frank van der Linden for Wasabi Systems, Inc. 13f5df1827Smickey * 14f5df1827Smickey * Redistribution and use in source and binary forms, with or without 15f5df1827Smickey * modification, are permitted provided that the following conditions 16f5df1827Smickey * are met: 17f5df1827Smickey * 1. Redistributions of source code must retain the above copyright 18f5df1827Smickey * notice, this list of conditions and the following disclaimer. 19f5df1827Smickey * 2. Redistributions in binary form must reproduce the above copyright 20f5df1827Smickey * notice, this list of conditions and the following disclaimer in the 21f5df1827Smickey * documentation and/or other materials provided with the distribution. 22f5df1827Smickey * 3. All advertising materials mentioning features or use of this software 23f5df1827Smickey * must display the following acknowledgement: 24f5df1827Smickey * This product includes software developed for the NetBSD Project by 25f5df1827Smickey * Wasabi Systems, Inc. 26f5df1827Smickey * 4. The name of Wasabi Systems, Inc. may not be used to endorse 27f5df1827Smickey * or promote products derived from this software without specific prior 28f5df1827Smickey * written permission. 29f5df1827Smickey * 30f5df1827Smickey * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 31f5df1827Smickey * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32f5df1827Smickey * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33f5df1827Smickey * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 34f5df1827Smickey * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35f5df1827Smickey * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36f5df1827Smickey * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37f5df1827Smickey * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38f5df1827Smickey * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39f5df1827Smickey * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40f5df1827Smickey * POSSIBILITY OF SUCH DAMAGE. 41f5df1827Smickey */ 42f5df1827Smickey 43f5df1827Smickey 44f5df1827Smickey/*- 45f5df1827Smickey * Copyright (c) 1998, 2000 The NetBSD Foundation, Inc. 46f5df1827Smickey * All rights reserved. 47f5df1827Smickey * 48f5df1827Smickey * This code is derived from software contributed to The NetBSD Foundation 49f5df1827Smickey * by Charles M. Hannum. 50f5df1827Smickey * 51f5df1827Smickey * Redistribution and use in source and binary forms, with or without 52f5df1827Smickey * modification, are permitted provided that the following conditions 53f5df1827Smickey * are met: 54f5df1827Smickey * 1. Redistributions of source code must retain the above copyright 55f5df1827Smickey * notice, this list of conditions and the following disclaimer. 56f5df1827Smickey * 2. Redistributions in binary form must reproduce the above copyright 57f5df1827Smickey * notice, this list of conditions and the following disclaimer in the 58f5df1827Smickey * documentation and/or other materials provided with the distribution. 59f5df1827Smickey * 3. All advertising materials mentioning features or use of this software 60f5df1827Smickey * must display the following acknowledgement: 61f5df1827Smickey * This product includes software developed by the NetBSD 62f5df1827Smickey * Foundation, Inc. and its contributors. 63f5df1827Smickey * 4. Neither the name of The NetBSD Foundation nor the names of its 64f5df1827Smickey * contributors may be used to endorse or promote products derived 65f5df1827Smickey * from this software without specific prior written permission. 66f5df1827Smickey * 67f5df1827Smickey * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 68f5df1827Smickey * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 69f5df1827Smickey * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 70f5df1827Smickey * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 71f5df1827Smickey * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 72f5df1827Smickey * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 73f5df1827Smickey * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 74f5df1827Smickey * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 75f5df1827Smickey * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 76f5df1827Smickey * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 77f5df1827Smickey * POSSIBILITY OF SUCH DAMAGE. 78f5df1827Smickey */ 79f5df1827Smickey 80f5df1827Smickey/*- 81f5df1827Smickey * Copyright (c) 1990 The Regents of the University of California. 82f5df1827Smickey * All rights reserved. 83f5df1827Smickey * 84f5df1827Smickey * This code is derived from software contributed to Berkeley by 85f5df1827Smickey * William Jolitz. 86f5df1827Smickey * 87f5df1827Smickey * Redistribution and use in source and binary forms, with or without 88f5df1827Smickey * modification, are permitted provided that the following conditions 89f5df1827Smickey * are met: 90f5df1827Smickey * 1. Redistributions of source code must retain the above copyright 91f5df1827Smickey * notice, this list of conditions and the following disclaimer. 92f5df1827Smickey * 2. Redistributions in binary form must reproduce the above copyright 93f5df1827Smickey * notice, this list of conditions and the following disclaimer in the 94f5df1827Smickey * documentation and/or other materials provided with the distribution. 95f5df1827Smickey * 3. All advertising materials mentioning features or use of this software 96f5df1827Smickey * must display the following acknowledgement: 97f5df1827Smickey * This product includes software developed by the University of 98f5df1827Smickey * California, Berkeley and its contributors. 99f5df1827Smickey * 4. Neither the name of the University nor the names of its contributors 100f5df1827Smickey * may be used to endorse or promote products derived from this software 101f5df1827Smickey * without specific prior written permission. 102f5df1827Smickey * 103f5df1827Smickey * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 104f5df1827Smickey * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 105f5df1827Smickey * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 106f5df1827Smickey * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 107f5df1827Smickey * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 108f5df1827Smickey * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 109f5df1827Smickey * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 110f5df1827Smickey * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 111f5df1827Smickey * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 112f5df1827Smickey * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 113f5df1827Smickey * SUCH DAMAGE. 114f5df1827Smickey * 115f5df1827Smickey * @(#)locore.s 7.3 (Berkeley) 5/13/91 116f5df1827Smickey */ 117f5df1827Smickey 118f5df1827Smickey#include "assym.h" 119f5df1827Smickey#include "lapic.h" 120f5df1827Smickey#include "ioapic.h" 121f5df1827Smickey#include "ksyms.h" 122f5df1827Smickey 123f5df1827Smickey#include <sys/errno.h> 124f5df1827Smickey#include <sys/syscall.h> 125f5df1827Smickey 126f5df1827Smickey#include <machine/param.h> 127f5df1827Smickey#include <machine/segments.h> 128f5df1827Smickey#include <machine/specialreg.h> 129f5df1827Smickey#include <machine/trap.h> 130f5df1827Smickey#include <machine/frameasm.h> 131f5df1827Smickey 132f5df1827Smickey#if NLAPIC > 0 133f5df1827Smickey#include <machine/i82489reg.h> 134f5df1827Smickey#endif 135f5df1827Smickey 136f5df1827Smickey/* 137f5df1827Smickey * override user-land alignment before including asm.h 138f5df1827Smickey */ 139f5df1827Smickey#define ALIGN_DATA .align 8 140f5df1827Smickey#define ALIGN_TEXT .align 16,0x90 141f5df1827Smickey#define _ALIGN_TEXT ALIGN_TEXT 142f5df1827Smickey 143f5df1827Smickey#include <machine/asm.h> 144f5df1827Smickey 145f5df1827Smickey#if defined(MULTIPROCESSOR) 146f5df1827Smickey 147f5df1827Smickey#define SET_CURPROC(proc,cpu) \ 148f5df1827Smickey movq CPUVAR(SELF),cpu ; \ 149f5df1827Smickey movq proc,CPUVAR(CURPROC) ; \ 150f5df1827Smickey movq cpu,P_CPU(proc) 151f5df1827Smickey 152f5df1827Smickey#else 153f5df1827Smickey 154f5df1827Smickey#define SET_CURPROC(proc,tcpu) movq proc,CPUVAR(CURPROC) 155f5df1827Smickey#define GET_CURPROC(reg) movq CPUVAR(CURPROC),reg 156f5df1827Smickey 157f5df1827Smickey#endif 158f5df1827Smickey 159f5df1827Smickey#define GET_CURPCB(reg) movq CPUVAR(CURPCB),reg 160f5df1827Smickey#define SET_CURPCB(reg) movq reg,CPUVAR(CURPCB) 161f5df1827Smickey 162f5df1827Smickey 163f5df1827Smickey/* XXX temporary kluge; these should not be here */ 164f5df1827Smickey/* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */ 165f5df1827Smickey#include <dev/isa/isareg.h> 166f5df1827Smickey 167f5df1827Smickey 168f5df1827Smickey/* 169f5df1827Smickey * Initialization 170f5df1827Smickey */ 171f5df1827Smickey .data 172f5df1827Smickey 173f5df1827Smickey#if NLAPIC > 0 174f5df1827Smickey .align NBPG 175f5df1827Smickey .globl _C_LABEL(local_apic), _C_LABEL(lapic_id), _C_LABEL(lapic_tpr) 176f5df1827Smickey_C_LABEL(local_apic): 177f5df1827Smickey .space LAPIC_ID 178f5df1827Smickey_C_LABEL(lapic_id): 179f5df1827Smickey .long 0x00000000 180f5df1827Smickey .space LAPIC_TPRI-(LAPIC_ID+4) 181f5df1827Smickey_C_LABEL(lapic_tpr): 182f5df1827Smickey .space LAPIC_PPRI-LAPIC_TPRI 183f5df1827Smickey_C_LABEL(lapic_ppr): 184f5df1827Smickey .space LAPIC_ISR-LAPIC_PPRI 185f5df1827Smickey_C_LABEL(lapic_isr): 186f5df1827Smickey .space NBPG-LAPIC_ISR 187f5df1827Smickey#endif 188f5df1827Smickey 189f5df1827Smickey .globl _C_LABEL(cpu_id),_C_LABEL(cpu_vendor), _C_LABEL(cpu_brand_id) 190f5df1827Smickey .globl _C_LABEL(cpuid_level),_C_LABEL(cpu_feature) 191a47f7207Smickey .globl _C_LABEL(esym),_C_LABEL(boothowto),_C_LABEL(bootdev) 1926483bf47Sderaadt .globl _C_LABEL(bootinfo), _C_LABEL(bootinfo_size), _C_LABEL(atdevbase) 193f5df1827Smickey .globl _C_LABEL(proc0paddr),_C_LABEL(PTDpaddr) 194f5df1827Smickey .globl _C_LABEL(biosbasemem),_C_LABEL(biosextmem) 1956483bf47Sderaadt .globl _C_LABEL(bootapiver) 196f5df1827Smickey .globl _C_LABEL(gdtstore) 197f5df1827Smickey_C_LABEL(cpu): .long 0 # are we 386, 386sx, or 486, 198f5df1827Smickey # or Pentium, or.. 199f5df1827Smickey_C_LABEL(cpu_id): .long 0 # saved from `cpuid' instruction 200f5df1827Smickey_C_LABEL(cpu_feature): .long 0 # feature flags from 'cpuid' 201f5df1827Smickey # instruction 202f5df1827Smickey_C_LABEL(cpuid_level): .long -1 # max. level accepted by 'cpuid' 203f5df1827Smickey # instruction 204f5df1827Smickey_C_LABEL(cpu_vendor): .space 16 # vendor string returned by `cpuid' 205f5df1827Smickey # instruction 206f5df1827Smickey_C_LABEL(cpu_brand_id): .long 0 # brand ID from 'cpuid' instruction 207f5df1827Smickey_C_LABEL(esym): .quad 0 # ptr to end of syms 208f5df1827Smickey_C_LABEL(atdevbase): .quad 0 # location of start of iomem in virtual 209e9dacf7aStom_C_LABEL(bootapiver): .long 0 # /boot API version 210f5df1827Smickey_C_LABEL(proc0paddr): .quad 0 211f5df1827Smickey_C_LABEL(PTDpaddr): .quad 0 # paddr of PTD, for libkvm 212f5df1827Smickey#ifndef REALBASEMEM 213f5df1827Smickey_C_LABEL(biosbasemem): .long 0 # base memory reported by BIOS 214f5df1827Smickey#else 215f5df1827Smickey_C_LABEL(biosbasemem): .long REALBASEMEM 216f5df1827Smickey#endif 217f5df1827Smickey#ifndef REALEXTMEM 218f5df1827Smickey_C_LABEL(biosextmem): .long 0 # extended memory reported by BIOS 219f5df1827Smickey#else 220f5df1827Smickey_C_LABEL(biosextmem): .long REALEXTMEM 221f5df1827Smickey#endif 222f5df1827Smickey 223f5df1827Smickey#define _RELOC(x) ((x) - KERNBASE) 224f5df1827Smickey#define RELOC(x) _RELOC(_C_LABEL(x)) 225f5df1827Smickey 226f5df1827Smickey .globl gdt64 227f5df1827Smickey 228f5df1827Smickeygdt64: 229f5df1827Smickey .word gdt64_end-gdt64_start 230f5df1827Smickey .quad _RELOC(gdt64_start) 231f5df1827Smickey.align 64 232f5df1827Smickey 233f5df1827Smickeygdt64_start: 234f5df1827Smickey .quad 0x0000000000000000 /* always empty */ 235f5df1827Smickey .quad 0x00af9a000000ffff /* kernel CS */ 236f5df1827Smickey .quad 0x00cf92000000ffff /* kernel DS */ 237f5df1827Smickeygdt64_end: 238f5df1827Smickey 239f5df1827Smickeyfarjmp64: 240f5df1827Smickey .long longmode-KERNBASE 241f5df1827Smickey .word GSEL(GCODE_SEL, SEL_KPL) 242f5df1827Smickey 243f5df1827Smickey .space 512 244f5df1827Smickeytmpstk: 245f5df1827Smickey 246f5df1827Smickey .globl _C_LABEL(cpu_private) 247f5df1827Smickey .comm _C_LABEL(cpu_private),NBPG,NBPG 248f5df1827Smickey 249f5df1827Smickey/* 250f5df1827Smickey * Some hackage to deal with 64bit symbols in 32 bit mode. 251e9dacf7aStom * This may not be needed if things are cleaned up a little. 252f5df1827Smickey */ 253f5df1827Smickey 254f5df1827Smickey 255f5df1827Smickey .text 256f5df1827Smickey .globl _C_LABEL(kernel_text) 257f5df1827Smickey .set _C_LABEL(kernel_text),KERNTEXTOFF 258f5df1827Smickey 259f5df1827Smickey .code32 260f5df1827Smickey 261f5df1827Smickey .globl start 262f5df1827Smickeystart: movw $0x1234,0x472 # warm boot 263f5df1827Smickey 264f5df1827Smickey /* 265f5df1827Smickey * Load parameters from stack 266930756e7Stom * (howto, bootdev, bootapiver, esym, extmem, cnvmem, ac, av) 267f5df1827Smickey */ 268f5df1827Smickey movl 4(%esp),%eax 269930756e7Stom movl %eax, RELOC(boothowto) 270a47f7207Smickey movl 8(%esp),%eax 271930756e7Stom movl %eax, RELOC(bootdev) 272f5df1827Smickey 273f5df1827Smickey movl 16(%esp), %eax 274f5df1827Smickey testl %eax,%eax 275f5df1827Smickey jz 1f 276f5df1827Smickey addl $KERNBASE_LO,%eax 277a47f7207Smickey movl $RELOC(esym),%ebp 278f5df1827Smickey movl %eax,(%ebp) 279f5df1827Smickey movl $KERNBASE_HI,4(%ebp) 280a47f7207Smickey1: 281f5df1827Smickey movl 20(%esp), %eax 282a47f7207Smickey movl %eax, RELOC(biosextmem) 283f5df1827Smickey movl 24(%esp), %eax 284a47f7207Smickey movl %eax, RELOC(biosbasemem) 285f5df1827Smickey 286e9dacf7aStom movl 12(%esp), %eax 287e9dacf7aStom movl %eax, RELOC(bootapiver) 2886483bf47Sderaadt 2896483bf47Sderaadt /* 2906483bf47Sderaadt * Copy the boot arguments to bootinfo[] in machdep.c. 2916483bf47Sderaadt * 2926483bf47Sderaadt * We are passed the size of bootinfo[] in bootinfo_size, and 2936483bf47Sderaadt * we report how much data /boot passed us back in the same variable. 2946483bf47Sderaadt * 2956483bf47Sderaadt * machdep.c can then take action if bootinfo_size >= bootinfo[] 2966483bf47Sderaadt * (which would meant that we may have been passed too much data). 2976483bf47Sderaadt */ 298e9dacf7aStom movl 28(%esp), %eax 2996483bf47Sderaadt movl %eax, %ecx 3006483bf47Sderaadt cmpl RELOC(bootinfo_size), %ecx /* Too much? */ 3016483bf47Sderaadt jnc bi_size_ok 3026483bf47Sderaadt movl RELOC(bootinfo_size), %ecx /* Only copy this much */ 3036483bf47Sderaadtbi_size_ok: 3046483bf47Sderaadt movl %eax, RELOC(bootinfo_size) /* Report full amount */ 3056483bf47Sderaadt 3066483bf47Sderaadt movl $RELOC(bootinfo), %edi /* Destination */ 3076483bf47Sderaadt movl 32(%esp), %esi /* Source */ 3086483bf47Sderaadt rep movsb /* Copy this many bytes */ 309e9dacf7aStom 310f5df1827Smickey /* First, reset the PSL. */ 311f5df1827Smickey pushl $PSL_MBO 312f5df1827Smickey popfl 313f5df1827Smickey 314f5df1827Smickey xorl %eax,%eax 315f5df1827Smickey cpuid 316f5df1827Smickey movl %eax,RELOC(cpuid_level) 317f5df1827Smickey movl $RELOC(cpu_vendor),%ebp 318f5df1827Smickey movl %ebx,(%ebp) 319f5df1827Smickey movl %edx,4(%ebp) 320f5df1827Smickey movl %ecx,8(%ebp) 321f5df1827Smickey movl $0, 12(%ebp) 322f5df1827Smickey 323f5df1827Smickey movl $1,%eax 324f5df1827Smickey cpuid 325f5df1827Smickey movl %eax,RELOC(cpu_id) 326f5df1827Smickey movl %edx,RELOC(cpu_feature) 327f5df1827Smickey 3281fef3300Smickey movl $0x80000001, %eax 3291fef3300Smickey cpuid 3301fef3300Smickey andl $CPUID_NXE, %edx /* other bits may clash */ 3311fef3300Smickey orl %edx, RELOC(cpu_feature) 3321fef3300Smickey 333f5df1827Smickey /* Brand ID is bits 0-7 of %ebx */ 334f5df1827Smickey andl $255,%ebx 335f5df1827Smickey movl %ebx,RELOC(cpu_brand_id) 336f5df1827Smickey 337f5df1827Smickey /* 338f5df1827Smickey * Finished with old stack; load new %esp now instead of later so we 339f5df1827Smickey * can trace this code without having to worry about the trace trap 340f5df1827Smickey * clobbering the memory test or the zeroing of the bss+bootstrap page 341f5df1827Smickey * tables. 342f5df1827Smickey * 343f5df1827Smickey * The boot program should check: 344f5df1827Smickey * text+data <= &stack_variable - more_space_for_stack 345f5df1827Smickey * text+data+bss+pad+space_for_page_tables <= end_of_memory 346f5df1827Smickey * Oops, the gdt is in the carcass of the boot program so clearing 347f5df1827Smickey * the rest of memory is still not possible. 348f5df1827Smickey */ 349f5df1827Smickey movl $RELOC(tmpstk),%esp 350f5df1827Smickey 351f5df1827Smickey/* 352f5df1827Smickey * Virtual address space of kernel: 353f5df1827Smickey * 354f5df1827Smickey * text | data | bss | [syms] | page dir | proc0 kstack | L1 ptp | L2 ptp | L3 355f5df1827Smickey * 0 1 2 3 356f5df1827Smickey */ 357f5df1827Smickey 358f5df1827Smickey#if L2_SLOT_KERNBASE > 0 359f5df1827Smickey#define TABLE_L2_ENTRIES (2 * (NKL2_KIMG_ENTRIES + 1)) 360f5df1827Smickey#else 361f5df1827Smickey#define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1) 362f5df1827Smickey#endif 363f5df1827Smickey 364f5df1827Smickey#if L3_SLOT_KERNBASE > 0 365f5df1827Smickey#define TABLE_L3_ENTRIES (2 * NKL3_KIMG_ENTRIES) 366f5df1827Smickey#else 367f5df1827Smickey#define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES 368f5df1827Smickey#endif 369f5df1827Smickey 370f5df1827Smickey 371f5df1827Smickey#define PROC0_PML4_OFF 0 372f5df1827Smickey#define PROC0_STK_OFF (PROC0_PML4_OFF + NBPG) 373f5df1827Smickey#define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * NBPG) 374f5df1827Smickey#define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * NBPG) 375f5df1827Smickey#define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * NBPG) 376f5df1827Smickey#define TABLESIZE \ 377f5df1827Smickey ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \ 378f5df1827Smickey * NBPG) 379f5df1827Smickey 380f5df1827Smickey#define fillkpt \ 381f5df1827Smickey1: movl %eax,(%ebx) ; /* store phys addr */ \ 382f5df1827Smickey movl $0,4(%ebx) ; /* upper 32 bits 0 */ \ 383f5df1827Smickey addl $8,%ebx ; /* next pte/pde */ \ 384f5df1827Smickey addl $NBPG,%eax ; /* next phys page */ \ 385f5df1827Smickey loop 1b ; \ 386f5df1827Smickey 387f5df1827Smickey 388f5df1827Smickey /* Find end of kernel image. */ 389f5df1827Smickey movl $RELOC(end),%edi 390f5df1827Smickey#if (NKSYMS || defined(DDB) || defined(LKM)) && !defined(SYMTAB_SPACE) 391f5df1827Smickey /* Save the symbols (if loaded). */ 392f5df1827Smickey movl RELOC(esym),%eax 393f5df1827Smickey testl %eax,%eax 394f5df1827Smickey jz 1f 395f5df1827Smickey subl $KERNBASE_LO,%eax /* XXX */ 396f5df1827Smickey movl %eax,%edi 397f5df1827Smickey1: 398f5df1827Smickey#endif 399f5df1827Smickey /* Clear tables */ 400f5df1827Smickey movl %edi,%esi 401f5df1827Smickey addl $PGOFSET,%esi 402f5df1827Smickey andl $~PGOFSET,%esi 403f5df1827Smickey 404f5df1827Smickey movl %esi,%edi 405f5df1827Smickey xorl %eax,%eax 406f5df1827Smickey cld 407f5df1827Smickey movl $TABLESIZE,%ecx 408f5df1827Smickey shrl $2,%ecx 409f5df1827Smickey rep 410f5df1827Smickey stosl 411f5df1827Smickey 412f5df1827Smickey leal (PROC0_PTP1_OFF)(%esi), %ebx 413f5df1827Smickey 414f5df1827Smickey /* 415f5df1827Smickey * Compute etext - KERNBASE. This can't be > 4G, or we can't deal 416f5df1827Smickey * with it anyway, since we can't load it in 32 bit mode. So use 417f5df1827Smickey * the bottom 32 bits. 418f5df1827Smickey */ 419f5df1827Smickey movl $RELOC(etext),%edx 420f5df1827Smickey addl $PGOFSET,%edx 421f5df1827Smickey andl $~PGOFSET,%edx 422f5df1827Smickey 423f5df1827Smickey /* 424f5df1827Smickey * Skip the first MB. 425f5df1827Smickey */ 426f5df1827Smickey movl $(KERNTEXTOFF_LO - KERNBASE_LO),%eax 427f5df1827Smickey movl %eax,%ecx 428f5df1827Smickey shrl $(PGSHIFT-3),%ecx /* ((n >> PGSHIFT) << 3) for # pdes */ 429f5df1827Smickey addl %ecx,%ebx 430f5df1827Smickey 431f5df1827Smickey /* Map kernel text read-only */ 432f5df1827Smickey movl %edx,%ecx 433f5df1827Smickey subl %eax,%ecx 434f5df1827Smickey shrl $PGSHIFT,%ecx 435f5df1827Smickey orl $(PG_V|PG_KR),%eax 436f5df1827Smickey fillkpt 437f5df1827Smickey 438f5df1827Smickey /* Map the data, BSS, and bootstrap tables read-write. */ 439f5df1827Smickey leal (PG_V|PG_KW)(%edx),%eax 440f5df1827Smickey movl $TABLESIZE,%ecx 441f5df1827Smickey addl %esi,%ecx /* %ecx = &end[TABLESIZE] */ 442f5df1827Smickey subl %edx,%ecx /* %ecx = %ecx - etext */ 443f5df1827Smickey shrl $PGSHIFT,%ecx 444f5df1827Smickey fillkpt 445f5df1827Smickey 446f5df1827Smickey /* Map ISA I/O mem (later atdevbase) */ 447f5df1827Smickey movl $(IOM_BEGIN|PG_V|PG_KW/*|PG_N*/),%eax 448f5df1827Smickey movl $(IOM_SIZE>>PGSHIFT),%ecx 449f5df1827Smickey fillkpt 450f5df1827Smickey 451f5df1827Smickey /* Set up level 2 pages */ 452f5df1827Smickey leal (PROC0_PTP2_OFF)(%esi),%ebx 453f5df1827Smickey leal (PROC0_PTP1_OFF)(%esi),%eax 454f5df1827Smickey orl $(PG_V|PG_KW), %eax 455f5df1827Smickey movl $(NKL2_KIMG_ENTRIES+1),%ecx 456f5df1827Smickey fillkpt 457f5df1827Smickey 458f5df1827Smickey#if L2_SLOT_KERNBASE > 0 459f5df1827Smickey /* If needed, set up level 2 entries for actual kernel mapping */ 460f5df1827Smickey leal (PROC0_PTP2_OFF+ L2_SLOT_KERNBASE*8)(%esi),%ebx 461f5df1827Smickey leal (PROC0_PTP1_OFF)(%esi),%eax 462f5df1827Smickey orl $(PG_V|PG_KW), %eax 463f5df1827Smickey movl $(NKL2_KIMG_ENTRIES+1),%ecx 464f5df1827Smickey fillkpt 465f5df1827Smickey#endif 466f5df1827Smickey 467f5df1827Smickey /* Set up level 3 pages */ 468f5df1827Smickey leal (PROC0_PTP3_OFF)(%esi),%ebx 469f5df1827Smickey leal (PROC0_PTP2_OFF)(%esi),%eax 470f5df1827Smickey orl $(PG_V|PG_KW), %eax 471f5df1827Smickey movl $NKL3_KIMG_ENTRIES,%ecx 472f5df1827Smickey fillkpt 473f5df1827Smickey 474f5df1827Smickey#if L3_SLOT_KERNBASE > 0 475f5df1827Smickey /* If needed, set up level 3 entries for actual kernel mapping */ 476f5df1827Smickey leal (PROC0_PTP3_OFF+ L3_SLOT_KERNBASE*8)(%esi),%ebx 477f5df1827Smickey leal (PROC0_PTP2_OFF)(%esi),%eax 478f5df1827Smickey orl $(PG_V|PG_KW), %eax 479f5df1827Smickey movl $NKL3_KIMG_ENTRIES,%ecx 480f5df1827Smickey fillkpt 481f5df1827Smickey#endif 482f5df1827Smickey 483f5df1827Smickey /* Set up top level entries for identity mapping */ 484f5df1827Smickey leal (PROC0_PML4_OFF)(%esi),%ebx 485f5df1827Smickey leal (PROC0_PTP3_OFF)(%esi),%eax 486f5df1827Smickey orl $(PG_V|PG_KW), %eax 487f5df1827Smickey movl $NKL4_KIMG_ENTRIES,%ecx 488f5df1827Smickey fillkpt 489f5df1827Smickey 490f5df1827Smickey /* Set up top level entries for actual kernel mapping */ 491f5df1827Smickey leal (PROC0_PML4_OFF + L4_SLOT_KERNBASE*8)(%esi),%ebx 492f5df1827Smickey leal (PROC0_PTP3_OFF)(%esi),%eax 493f5df1827Smickey orl $(PG_V|PG_KW), %eax 494f5df1827Smickey movl $NKL4_KIMG_ENTRIES,%ecx 495f5df1827Smickey fillkpt 496f5df1827Smickey 497f5df1827Smickey /* Install recursive top level PDE */ 498f5df1827Smickey leal (PROC0_PML4_OFF + PDIR_SLOT_PTE*8)(%esi),%ebx 499f5df1827Smickey leal (PROC0_PML4_OFF)(%esi),%eax 500f5df1827Smickey orl $(PG_V|PG_KW),%eax 501f5df1827Smickey movl %eax,(%ebx) 502f5df1827Smickey movl $0, 4(%ebx) 503f5df1827Smickey 504f5df1827Smickey 505f5df1827Smickey /* Save phys. addr of PTD, for libkvm. */ 506f5df1827Smickey movl $RELOC(PTDpaddr),%ebp 507f5df1827Smickey movl %esi,(%ebp) 508f5df1827Smickey movl $0,4(%ebp) 509f5df1827Smickey 510f5df1827Smickey /* 511f5df1827Smickey * Startup checklist: 512f5df1827Smickey * 1. Enable PAE (and SSE while here). 513f5df1827Smickey */ 514f5df1827Smickey movl %cr4,%eax 515f5df1827Smickey orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax 516f5df1827Smickey movl %eax,%cr4 517f5df1827Smickey 518f5df1827Smickey /* 519f5df1827Smickey * 2. Set Long Mode Enable in EFER. Also enable the 520f5df1827Smickey * syscall extensions. 521f5df1827Smickey */ 522f5df1827Smickey movl $MSR_EFER,%ecx 523f5df1827Smickey rdmsr 524f5df1827Smickey xorl %eax,%eax /* XXX */ 525f5df1827Smickey orl $(EFER_LME|EFER_SCE),%eax 526f5df1827Smickey wrmsr 527f5df1827Smickey 528f5df1827Smickey /* 529f5df1827Smickey * 3. Load %cr3 with pointer to PML4. 530f5df1827Smickey */ 531f5df1827Smickey movl %esi,%eax 532f5df1827Smickey movl %eax,%cr3 533f5df1827Smickey 534f5df1827Smickey /* 535f5df1827Smickey * 4. Enable paging and the rest of it. 536f5df1827Smickey */ 537f5df1827Smickey movl %cr0,%eax 538f5df1827Smickey orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP),%eax 539f5df1827Smickey movl %eax,%cr0 540f5df1827Smickey jmp compat 541f5df1827Smickeycompat: 542f5df1827Smickey 543f5df1827Smickey /* 544f5df1827Smickey * 5. 545f5df1827Smickey * Not quite done yet, we're now in a compatibility segment, 546f5df1827Smickey * in legacy mode. We must jump to a long mode segment. 547f5df1827Smickey * Need to set up a temporary GDT with a long mode segment 548f5df1827Smickey * in it to do that. 549f5df1827Smickey */ 550f5df1827Smickey 551f5df1827Smickey movl $RELOC(gdt64),%eax 552f5df1827Smickey lgdt (%eax) 553f5df1827Smickey movl $RELOC(farjmp64),%eax 554f5df1827Smickey ljmp *(%eax) 555f5df1827Smickey 556f5df1827Smickey .code64 557f5df1827Smickeylongmode: 558f5df1827Smickey /* 559f5df1827Smickey * 6. 560f5df1827Smickey * Finally, we're in long mode. However, we're still 561f5df1827Smickey * in the identity mapped area (could not jump out 562f5df1827Smickey * of that earlier because it would have been a > 32bit 563f5df1827Smickey * jump). We can do that now, so here we go. 564f5df1827Smickey */ 565f5df1827Smickey movabsq $longmode_hi,%rax 566f5df1827Smickey jmp *%rax 567f5df1827Smickeylongmode_hi: 568f5df1827Smickey /* 569f5df1827Smickey * We have arrived. 570f5df1827Smickey * There's no need anymore for the identity mapping in low 571f5df1827Smickey * memory, remove it. 572f5df1827Smickey */ 573f5df1827Smickey movq $KERNBASE,%r8 574f5df1827Smickey 575f5df1827Smickey#if L2_SLOT_KERNBASE > 0 576f5df1827Smickey movq $(NKL2_KIMG_ENTRIES+1),%rcx 577f5df1827Smickey leaq (PROC0_PTP2_OFF)(%rsi),%rbx 578f5df1827Smickey addq %r8, %rbx 579f5df1827Smickey1: movq $0,(%rbx) 580f5df1827Smickey addq $8,%rbx 581f5df1827Smickey loop 1b 582f5df1827Smickey#endif 583f5df1827Smickey 584f5df1827Smickey#if L3_SLOT_KERNBASE > 0 585f5df1827Smickey movq $NKL3_KIMG_ENTRIES,%rcx 586f5df1827Smickey leaq (PROC0_PTP3_OFF)(%rsi),%rbx 587f5df1827Smickey addq %r8, %rbx 588f5df1827Smickey1: movq $0,(%rbx) 589f5df1827Smickey addq $8,%rbx 590f5df1827Smickey loop 1b 591f5df1827Smickey#endif 592f5df1827Smickey 593f5df1827Smickey movq $NKL4_KIMG_ENTRIES,%rcx 594f5df1827Smickey leaq (PROC0_PML4_OFF)(%rsi),%rbx # old, phys address of PML4 595f5df1827Smickey addq %r8, %rbx # new, virtual adress of PML4 596f5df1827Smickey1: movq $0,(%rbx) 597f5df1827Smickey addq $8,%rbx 598f5df1827Smickey loop 1b 599f5df1827Smickey 600f5df1827Smickey /* Relocate atdevbase. */ 601f5df1827Smickey movq $(TABLESIZE+KERNBASE),%rdx 602f5df1827Smickey addq %rsi,%rdx 603f5df1827Smickey movq %rdx,_C_LABEL(atdevbase)(%rip) 604f5df1827Smickey 605f5df1827Smickey /* Set up bootstrap stack. */ 606f5df1827Smickey leaq (PROC0_STK_OFF)(%rsi),%rax 607f5df1827Smickey addq %r8,%rax 608f5df1827Smickey movq %rax,_C_LABEL(proc0paddr)(%rip) 609f5df1827Smickey leaq (USPACE-FRAMESIZE)(%rax),%rsp 610f5df1827Smickey movq %rsi,PCB_CR3(%rax) # pcb->pcb_cr3 611f5df1827Smickey xorq %rbp,%rbp # mark end of frames 612f5df1827Smickey 613f5df1827Smickey xorw %ax,%ax 614f5df1827Smickey movw %ax,%gs 615f5df1827Smickey movw %ax,%fs 616f5df1827Smickey 617f5df1827Smickey /* XXX merge these */ 618f5df1827Smickey leaq TABLESIZE(%rsi),%rdi 619f5df1827Smickey call _C_LABEL(init_x86_64) 620f5df1827Smickey 621f5df1827Smickey call _C_LABEL(main) 622f5df1827Smickey 623f5df1827Smickey/*****************************************************************************/ 624f5df1827Smickey 625f5df1827Smickey/* 626f5df1827Smickey * Signal trampoline; copied to top of user stack. 627f5df1827Smickey */ 628f5df1827SmickeyNENTRY(sigcode) 629f5df1827Smickey call *%rax 630f5df1827Smickey 631f5df1827Smickey movq %rsp,%rdi 632f5df1827Smickey pushq %rdi /* fake return address */ 633f5df1827Smickey movq $SYS_sigreturn,%rax 634cb09579aSderaadt int $0x80 635f5df1827Smickey movq $SYS_exit,%rax 636f5df1827Smickey syscall 637f5df1827Smickey .globl _C_LABEL(esigcode) 638f5df1827Smickey_C_LABEL(esigcode): 639f5df1827Smickey 640f5df1827Smickey/* 641f5df1827Smickey * void lgdt(struct region_descriptor *rdp); 642f5df1827Smickey * Change the global descriptor table. 643f5df1827Smickey */ 644f5df1827SmickeyNENTRY(lgdt) 645f5df1827Smickey /* Reload the descriptor table. */ 646f5df1827Smickey movq %rdi,%rax 647f5df1827Smickey lgdt (%rax) 648f5df1827Smickey /* Flush the prefetch q. */ 649f5df1827Smickey jmp 1f 650f5df1827Smickey nop 651f5df1827Smickey1: /* Reload "stale" selectors. */ 652f5df1827Smickey movl $GSEL(GDATA_SEL, SEL_KPL),%eax 653f5df1827Smickey movl %eax,%ds 654f5df1827Smickey movl %eax,%es 655f5df1827Smickey movl %eax,%ss 656f5df1827Smickey /* Reload code selector by doing intersegment return. */ 657f5df1827Smickey popq %rax 658f5df1827Smickey pushq $GSEL(GCODE_SEL, SEL_KPL) 659f5df1827Smickey pushq %rax 660f5df1827Smickey lretq 661f5df1827Smickey 662f5df1827SmickeyENTRY(setjmp) 663f5df1827Smickey /* 664f5df1827Smickey * Only save registers that must be preserved across function 665f5df1827Smickey * calls according to the ABI (%rbx, %rsp, %rbp, %r12-%r15) 666f5df1827Smickey * and %rip. 667f5df1827Smickey */ 668f5df1827Smickey movq %rdi,%rax 669f5df1827Smickey movq %rbx,(%rax) 670f5df1827Smickey movq %rsp,8(%rax) 671f5df1827Smickey movq %rbp,16(%rax) 672f5df1827Smickey movq %r12,24(%rax) 673f5df1827Smickey movq %r13,32(%rax) 674f5df1827Smickey movq %r14,40(%rax) 675f5df1827Smickey movq %r15,48(%rax) 676f5df1827Smickey movq (%rsp),%rdx 677f5df1827Smickey movq %rdx,56(%rax) 678f5df1827Smickey xorl %eax,%eax 679f5df1827Smickey ret 680f5df1827Smickey 681f5df1827SmickeyENTRY(longjmp) 682f5df1827Smickey movq %rdi,%rax 683f5df1827Smickey movq (%rax),%rbx 684f5df1827Smickey movq 8(%rax),%rsp 685f5df1827Smickey movq 16(%rax),%rbp 686f5df1827Smickey movq 24(%rax),%r12 687f5df1827Smickey movq 32(%rax),%r13 688f5df1827Smickey movq 40(%rax),%r14 689f5df1827Smickey movq 48(%rax),%r15 690f5df1827Smickey movq 56(%rax),%rdx 691f5df1827Smickey movq %rdx,(%rsp) 692f5df1827Smickey xorl %eax,%eax 693f5df1827Smickey incl %eax 694f5df1827Smickey ret 695f5df1827Smickey 696f5df1827Smickey/*****************************************************************************/ 697f5df1827Smickey 698f5df1827Smickey/* 699f5df1827Smickey * The following primitives manipulate the run queues. 700f5df1827Smickey * _whichqs tells which of the 32 queues _qs 701f5df1827Smickey * have processes in them. Setrq puts processes into queues, Remrq 702f5df1827Smickey * removes them from queues. The running process is on no queue, 703f5df1827Smickey * other processes are on a queue related to p->p_pri, divided by 4 704f5df1827Smickey * actually to shrink the 0-127 range of priorities into the 32 available 705f5df1827Smickey * queues. 706f5df1827Smickey */ 707f5df1827Smickey .globl _C_LABEL(whichqs),_C_LABEL(qs) 708f5df1827Smickey .globl _C_LABEL(uvmexp),_C_LABEL(panic) 709f5df1827Smickey 710f5df1827Smickey#if NAPM > 0 711f5df1827Smickey .globl _C_LABEL(apm_cpu_idle),_C_LABEL(apm_cpu_busy) 712f5df1827Smickey#endif 713f5df1827Smickey 714f5df1827Smickey#ifdef DIAGNOSTIC 715f5df1827SmickeyNENTRY(switch_error1) 716f5df1827Smickey movabsq $1f,%rdi 717f5df1827Smickey call _C_LABEL(panic) 718f5df1827Smickey /* NOTREACHED */ 719f5df1827Smickey1: .asciz "cpu_switch 1" 720f5df1827SmickeyNENTRY(switch_error2) 721f5df1827Smickey movabsq $1f,%rdi 722f5df1827Smickey call _C_LABEL(panic) 723f5df1827Smickey /* NOTREACHED */ 724f5df1827Smickey1: .asciz "cpu_switch 2" 725f5df1827SmickeyNENTRY(switch_error3) 726f5df1827Smickey movabsq $1f,%rdi 727f5df1827Smickey call _C_LABEL(panic) 728f5df1827Smickey /* NOTREACHED */ 729f5df1827Smickey1: .asciz "cpu_switch 3" 730f5df1827Smickey#endif /* DIAGNOSTIC */ 731f5df1827Smickey 732f5df1827Smickey/* 733f5df1827Smickey * When no processes are on the runq, cpu_switch() branches to here to wait for 734f5df1827Smickey * something to come ready. 735f5df1827Smickey */ 736f5df1827SmickeyENTRY(idle) 737f5df1827Smickey#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 738f5df1827Smickey call _C_LABEL(sched_unlock_idle) 739f5df1827Smickey#endif 740f5df1827Smickey jmp idle_start 741f5df1827Smickeyidle_zero: 742f5df1827Smickey sti 743f5df1827Smickey call _C_LABEL(uvm_pageidlezero) 744f5df1827Smickey jmp idle_start 745f5df1827Smickeyidle_loop: 746f5df1827Smickey /* Try to zero some pages. */ 747f5df1827Smickey movl _C_LABEL(uvm)+UVM_PAGE_IDLE_ZERO(%rip),%ecx 748f5df1827Smickey testl %ecx,%ecx 749f5df1827Smickey jnz idle_zero 750f5df1827Smickey sti 751f5df1827Smickey hlt 752f5df1827SmickeyNENTRY(mpidle) 753f5df1827Smickeyidle_start: 754f5df1827Smickey cli 755f5df1827Smickey movl _C_LABEL(whichqs)(%rip),%ecx 756f5df1827Smickey testl %ecx, %ecx 757f5df1827Smickey jz idle_loop 758f5df1827Smickey#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 759f5df1827Smickey call _C_LABEL(sched_lock_idle) 760f5df1827Smickey#endif 761f5df1827Smickey jmp sw1 762f5df1827Smickey 763f5df1827Smickey/* 764f5df1827Smickey * void cpu_switch(struct proc *) 765f5df1827Smickey * Find a runnable process and switch to it. Wait if necessary. If the new 766f5df1827Smickey * proc is the same as the old one, we short-circuit the context save and 767f5df1827Smickey * restore. 768f5df1827Smickey */ 769f5df1827SmickeyENTRY(cpu_switch) 770f5df1827Smickey pushq %rbx 771f5df1827Smickey pushq %rbp 772f5df1827Smickey pushq %r12 773f5df1827Smickey pushq %r13 774f5df1827Smickey pushq %r14 775f5df1827Smickey pushq %r15 776f5df1827Smickey movl CPUVAR(ILEVEL), %ebx 777f5df1827Smickey pushq %rbx 778f5df1827Smickey 779f5df1827Smickey movq %rdi,%r13 780f5df1827Smickey 781f5df1827Smickey /* 782f5df1827Smickey * Clear curproc so that we don't accumulate system time while idle. 783f5df1827Smickey * This also insures that schedcpu() will move the old proc to 784f5df1827Smickey * the correct queue if it happens to get called from the spllower() 785f5df1827Smickey * below and changes the priority. (See corresponding comment in 786f5df1827Smickey * userret()). 787f5df1827Smickey */ 788f5df1827Smickey movq $0, CPUVAR(CURPROC) 789f5df1827Smickey 790f5df1827Smickey movl $IPL_NONE, %edi 791f5df1827Smickey call _C_LABEL(Xspllower) 792f5df1827Smickey 793f5df1827Smickey /* 794f5df1827Smickey * First phase: find new proc. 795f5df1827Smickey * 796f5df1827Smickey * Registers: 797f5df1827Smickey * %rax - queue head, scratch, then zero 798f5df1827Smickey * %r8 - queue number 799f5df1827Smickey * %ecx - cached value of whichqs 800f5df1827Smickey * %rdx - next process in queue 801f5df1827Smickey * %r13 - old proc 802f5df1827Smickey * %r12 - new proc 803f5df1827Smickey */ 804f5df1827Smickey 805f5df1827Smickeyswitch_search: 806f5df1827Smickey /* Look for new proc. */ 807f5df1827Smickey cli # splhigh doesn't do a cli 808f5df1827Smickey movl _C_LABEL(whichqs)(%rip),%ecx 809f5df1827Smickey 810f5df1827Smickeysw1: bsfl %ecx,%r8d # find a full q 811f5df1827Smickey jz _C_LABEL(idle) 812f5df1827Smickey 813f5df1827Smickeyswitch_dequeue: 814f5df1827Smickey movq %r8,%r9 815f5df1827Smickey 816f5df1827Smickey shlq $4, %r9 817f5df1827Smickey leaq _C_LABEL(qs)(%rip),%rax 818f5df1827Smickey addq %r9,%rax 819f5df1827Smickey /* movq (%rax),%rax */ 820f5df1827Smickey 821f5df1827Smickey movq P_FORW(%rax),%r12 # unlink from front of process q 822f5df1827Smickey#ifdef DIAGNOSTIC 823f5df1827Smickey cmpq %r12,%rax # linked to self (i.e. nothing queued)? 824f5df1827Smickey je _C_LABEL(switch_error1) # not possible 825f5df1827Smickey#endif /* DIAGNOSTIC */ 826f5df1827Smickey movq P_FORW(%r12),%rdx 827f5df1827Smickey movq %rdx,P_FORW(%rax) 828f5df1827Smickey movq %rax,P_BACK(%rdx) 829f5df1827Smickey 830f5df1827Smickey cmpq %rdx,%rax # q empty? 831f5df1827Smickey jne 3f 832f5df1827Smickey 833f5df1827Smickey btrl %r8d,%ecx # yes, clear to indicate empty 834f5df1827Smickey movl %ecx,_C_LABEL(whichqs)(%rip) # update q status 835f5df1827Smickey 836f5df1827Smickey3: /* We just did it. */ 837f5df1827Smickey xorq %rax,%rax 838f5df1827Smickey movl %eax,CPUVAR(RESCHED) 839f5df1827Smickeyswitch_resume: 840f5df1827Smickey#ifdef DIAGNOSTIC 841f5df1827Smickey cmpq %rax,P_WCHAN(%r12) 842f5df1827Smickey jne _C_LABEL(switch_error2) 843f5df1827Smickey cmpb $SRUN,P_STAT(%r12) 844f5df1827Smickey jne _C_LABEL(switch_error3) 845f5df1827Smickey#endif 846f5df1827Smickey 847f5df1827Smickey /* Isolate proc. XXX Is this necessary? */ 848f5df1827Smickey movq %rax,P_BACK(%r12) 849f5df1827Smickey 850f5df1827Smickey /* Record new proc. */ 851*012ea299Sniklas movb $SONPROC,P_STAT(%r12) # p->p_stat = SONPROC 852f5df1827Smickey SET_CURPROC(%r12,%rcx) 853f5df1827Smickey 854f5df1827Smickey sti 855f5df1827Smickey 856f5df1827Smickey /* Skip context switch if same proc. */ 857f5df1827Smickey movl $1,%eax 858f5df1827Smickey cmpq %r12,%r13 859f5df1827Smickey je switch_return 860f5df1827Smickey 861f5df1827Smickey /* If old proc exited, don't bother. */ 862f5df1827Smickey testq %r13,%r13 863f5df1827Smickey jz switch_exited 864f5df1827Smickey 865f5df1827Smickey /* 866f5df1827Smickey * Second phase: save old context. 867f5df1827Smickey * 868f5df1827Smickey * Registers: 869f5df1827Smickey * %rax, %rcx - scratch 870f5df1827Smickey * %r13 - old proc, then old pcb 871f5df1827Smickey * %r12 - new proc 872f5df1827Smickey */ 873f5df1827Smickey 874f5df1827Smickey movq P_ADDR(%r13),%r13 875f5df1827Smickey 876f5df1827Smickey /* Save stack pointers. */ 877f5df1827Smickey movq %rsp,PCB_RSP(%r13) 878f5df1827Smickey movq %rbp,PCB_RBP(%r13) 879f5df1827Smickey 880f5df1827Smickeyswitch_exited: 881f5df1827Smickey /* 882f5df1827Smickey * Third phase: restore saved context. 883f5df1827Smickey * 884f5df1827Smickey * Registers: 885f5df1827Smickey * %rax, %rcx, %rdx - scratch 886f5df1827Smickey * %r13 - new pcb 887f5df1827Smickey * %r12 - new process 888f5df1827Smickey */ 889f5df1827Smickey 890f5df1827Smickey /* No interrupts while loading new state. */ 891f5df1827Smickey cli 892f5df1827Smickey movq P_ADDR(%r12),%r13 893f5df1827Smickey 894f5df1827Smickey /* Restore stack pointers. */ 895f5df1827Smickey movq PCB_RSP(%r13),%rsp 896f5df1827Smickey movq PCB_RBP(%r13),%rbp 897f5df1827Smickey 898f5df1827Smickey#if 0 899f5df1827Smickey /* Don't bother with the rest if switching to a system process. */ 900f5df1827Smickey testl $P_SYSTEM,P_FLAG(%r12) 901f5df1827Smickey jnz switch_restored 902f5df1827Smickey#endif 903f5df1827Smickey 904f5df1827Smickey /* Load TSS info. */ 905f5df1827Smickey#ifdef MULTIPROCESSOR 906f5df1827Smickey movq CPUVAR(GDT),%rax 907f5df1827Smickey#else 908f5df1827Smickey movq _C_LABEL(gdtstore)(%rip),%rax 909f5df1827Smickey#endif 910f5df1827Smickey movl P_MD_TSS_SEL(%r12),%edx 911f5df1827Smickey 912f5df1827Smickey /* Switch address space. */ 913f5df1827Smickey movq PCB_CR3(%r13),%rcx 914f5df1827Smickey movq %rcx,%cr3 915f5df1827Smickey 916f5df1827Smickey /* Switch TSS. Reset "task busy" flag before */ 917f5df1827Smickey andl $~0x0200,4(%rax,%rdx, 1) 918f5df1827Smickey ltr %dx 919f5df1827Smickey 920f5df1827Smickey movq PCB_LDT_SEL(%r13),%rcx 921f5df1827Smickey lldt %cx 922f5df1827Smickey#if 0 923f5df1827Smickeyswitch_restored: 924f5df1827Smickey#endif 925f5df1827Smickey /* Restore cr0 (including FPU state). */ 926f5df1827Smickey movl PCB_CR0(%r13),%ecx 927f5df1827Smickey#ifdef MULTIPROCESSOR 928f5df1827Smickey movq PCB_FPCPU(%r13),%r8 929f5df1827Smickey cmpq CPUVAR(SELF),%r8 930f5df1827Smickey jz 1f 931f5df1827Smickey orl $CR0_TS,%ecx 932f5df1827Smickey1: 933f5df1827Smickey#endif 934f5df1827Smickey movq %rcx,%cr0 935f5df1827Smickey 936f5df1827Smickey SET_CURPCB(%r13) 937f5df1827Smickey 938f5df1827Smickey /* Interrupts are okay again. */ 939f5df1827Smickey sti 940f5df1827Smickey 941f5df1827Smickeyswitch_return: 942f5df1827Smickey#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 943f5df1827Smickey call _C_LABEL(sched_unlock_idle) 944f5df1827Smickey#endif 945f5df1827Smickey /* 946f5df1827Smickey * Restore old cpl from stack. Note that this is always an increase, 947f5df1827Smickey * due to the spl0() on entry. 948f5df1827Smickey */ 949f5df1827Smickey popq %rbx 950f5df1827Smickey movl %ebx, CPUVAR(ILEVEL) 951f5df1827Smickey 952f5df1827Smickey popq %r15 953f5df1827Smickey popq %r14 954f5df1827Smickey popq %r13 955f5df1827Smickey popq %r12 956f5df1827Smickey popq %rbp 957f5df1827Smickey popq %rbx 958f5df1827Smickey ret 959f5df1827Smickey 960f5df1827SmickeyENTRY(cpu_switchto) 961f5df1827Smickey pushq %rbx 962f5df1827Smickey pushq %rbp 963f5df1827Smickey pushq %r12 964f5df1827Smickey pushq %r13 965f5df1827Smickey pushq %r14 966f5df1827Smickey pushq %r15 967f5df1827Smickey 968f5df1827Smickey movq %rdi,%r13 969f5df1827Smickey movq %rsi,%r12 970f5df1827Smickey 971f5df1827Smickey movq $0,CPUVAR(CURPROC) 972f5df1827Smickey 973f5df1827Smickey xorq %rax,%rax 974f5df1827Smickey jmp switch_resume 975f5df1827Smickey 976f5df1827Smickey 977f5df1827Smickey/* 978f5df1827Smickey * void switch_exit(struct proc *l, void (*exit)(struct proc *)); 979f5df1827Smickey * Switch to proc0's saved context and deallocate the address space and kernel 980f5df1827Smickey * stack for p. Then jump into cpu_switch(), as if we were in proc0 all along. 981f5df1827Smickey */ 982f5df1827Smickey .globl _C_LABEL(proc0),_C_LABEL(uvmspace_free),_C_LABEL(kernel_map) 983f5df1827Smickey .globl _C_LABEL(uvm_km_free),_C_LABEL(tss_free) 984f5df1827SmickeyENTRY(switch_exit) 985f5df1827Smickey#ifdef MULTIPROCESSOR 986f5df1827Smickey movq CPUVAR(IDLE_PCB),%r8 987f5df1827Smickey movl CPUVAR(IDLE_TSS_SEL),%edx 988f5df1827Smickey#else 989f5df1827Smickey leaq _C_LABEL(proc0)(%rip),%r9 990f5df1827Smickey movq P_ADDR(%r9),%r8 991f5df1827Smickey movl P_MD_TSS_SEL(%r9),%edx 992f5df1827Smickey#endif 993f5df1827Smickey 994f5df1827Smickey /* In case we fault... */ 995f5df1827Smickey movq $0,CPUVAR(CURPROC) 996f5df1827Smickey 997f5df1827Smickey cli 998f5df1827Smickey 999f5df1827Smickey /* Restore stack pointers. */ 1000f5df1827Smickey movq PCB_RSP(%r8),%rsp 1001f5df1827Smickey movq PCB_RBP(%r8),%rbp 1002f5df1827Smickey 1003f5df1827Smickey /* Load TSS info. */ 1004f5df1827Smickey#ifdef MULTIPROCESSOR 1005f5df1827Smickey movq CPUVAR(GDT),%rax 1006f5df1827Smickey#else 1007f5df1827Smickey movq _C_LABEL(gdtstore)(%rip),%rax 1008f5df1827Smickey#endif 1009f5df1827Smickey 1010f5df1827Smickey /* Switch address space. */ 1011f5df1827Smickey movq PCB_CR3(%r8),%rcx 1012f5df1827Smickey movq %rcx,%cr3 1013f5df1827Smickey 1014f5df1827Smickey /* Switch TSS. */ 1015f5df1827Smickey andl $~0x0200,4-SEL_KPL(%rax,%rdx,1) 1016f5df1827Smickey ltr %dx 1017f5df1827Smickey 1018f5df1827Smickey /* We're always in the kernel, so we don't need the LDT. */ 1019f5df1827Smickey 1020f5df1827Smickey /* Restore cr0 (including FPU state). */ 1021f5df1827Smickey movl PCB_CR0(%r8),%ecx 1022f5df1827Smickey movq %rcx,%cr0 1023f5df1827Smickey 1024f5df1827Smickey /* Record new pcb. */ 1025f5df1827Smickey SET_CURPCB(%r8) 1026f5df1827Smickey 1027f5df1827Smickey /* Interrupts are okay again. */ 1028f5df1827Smickey sti 1029f5df1827Smickey 1030f5df1827Smickey /* 1031f5df1827Smickey * Schedule the dead process's vmspace and stack to be freed. 1032f5df1827Smickey * {lpw_}exit2(l). Function still in %rsi (2nd arg), proc in 1033f5df1827Smickey * %rdi (first arg). 1034f5df1827Smickey */ 1035f5df1827Smickey 1036f5df1827Smickey call *%rsi 1037f5df1827Smickey 1038f5df1827Smickey /* Jump into cpu_switch() with the right state. */ 1039f5df1827Smickey movq %r9, %r13 1040f5df1827Smickey movq $0, CPUVAR(CURPROC) 1041f5df1827Smickey jmp switch_search 1042f5df1827Smickey 1043f5df1827Smickey/* 1044f5df1827Smickey * savectx(struct pcb *pcb); 1045f5df1827Smickey * Update pcb, saving current processor state. 1046f5df1827Smickey */ 1047f5df1827SmickeyENTRY(savectx) 1048f5df1827Smickey /* Save stack pointers. */ 1049f5df1827Smickey movq %rsp,PCB_RSP(%rdi) 1050f5df1827Smickey movq %rbp,PCB_RBP(%rdi) 1051f5df1827Smickey 1052f5df1827Smickey ret 1053f5df1827Smickey 1054f5df1827SmickeyIDTVEC(syscall32) 1055f5df1827Smickey sysret /* go away please */ 1056f5df1827Smickey 1057f5df1827Smickey/* 1058f5df1827Smickey * syscall insn entry. This currently isn't much faster, but 1059f5df1827Smickey * it can be made faster in the future. 1060f5df1827Smickey */ 1061f5df1827SmickeyIDTVEC(syscall) 1062f5df1827Smickey swapgs 1063f5df1827Smickey movq %r15,CPUVAR(SCRATCH) 1064f5df1827Smickey movq CPUVAR(CURPCB),%r15 1065f5df1827Smickey movq PCB_RSP0(%r15),%r15 1066f5df1827Smickey xchgq %r15,%rsp 1067f5df1827Smickey sti 1068f5df1827Smickey 1069f5df1827Smickey /* 1070f5df1827Smickey * XXX don't need this whole frame, split of the 1071f5df1827Smickey * syscall frame and trapframe is needed. 1072f5df1827Smickey * First, leave some room for the trapno, error, 1073f5df1827Smickey * ss:rsp, etc, so that all GP registers can be 1074f5df1827Smickey * saved. Then, fill in the rest. 1075f5df1827Smickey */ 1076f5df1827Smickey pushq $(LSEL(LUDATA_SEL, SEL_UPL)) 1077f5df1827Smickey pushq %r15 1078f5df1827Smickey subq $(TF_RSP-TF_TRAPNO),%rsp 1079f5df1827Smickey movq CPUVAR(SCRATCH),%r15 1080f5df1827Smickey subq $32,%rsp 1081f5df1827Smickey INTR_SAVE_GPRS 1082f5df1827Smickey movw %fs,TF_FS(%rsp) 1083f5df1827Smickey movw %gs,TF_GS(%rsp) 1084f5df1827Smickey movw %es,TF_ES(%rsp) 1085f5df1827Smickey movw $(LSEL(LUDATA_SEL, SEL_UPL)),TF_DS(%rsp) 1086f5df1827Smickey movq %r11, TF_RFLAGS(%rsp) /* old rflags from syscall insn */ 1087f5df1827Smickey movq $(LSEL(LUCODE_SEL, SEL_UPL)), TF_CS(%rsp) 1088f5df1827Smickey movq %rcx,TF_RIP(%rsp) 1089f5df1827Smickey movq $2,TF_ERR(%rsp) 1090f5df1827Smickey movq $T_ASTFLT, TF_TRAPNO(%rsp) 1091f5df1827Smickey 1092f5df1827Smickey movq CPUVAR(CURPROC),%r14 1093f5df1827Smickey movq %rsp,P_MD_REGS(%r14) # save pointer to frame 1094f5df1827Smickey andl $~MDP_IRET,P_MD_FLAGS(%r14) 10954e1a77ceSsturm call _C_LABEL(syscall) 1096f5df1827Smickey1: /* Check for ASTs on exit to user mode. */ 1097f5df1827Smickey cli 1098f5df1827Smickey CHECK_ASTPENDING(%r11) 1099f5df1827Smickey je 2f 1100f5df1827Smickey /* Always returning to user mode here. */ 1101f5df1827Smickey CLEAR_ASTPENDING(%r11) 1102f5df1827Smickey sti 1103f5df1827Smickey /* Pushed T_ASTFLT into tf_trapno on entry. */ 1104f5df1827Smickey call _C_LABEL(trap) 1105f5df1827Smickey jmp 1b 1106f5df1827Smickey2: 1107f5df1827Smickey sti 1108f5df1827Smickey testl $MDP_IRET, P_MD_FLAGS(%r14) 1109f5df1827Smickey jne iret_return; 1110f5df1827Smickeysyscall_return: 1111f5df1827Smickey#ifdef DIAGNOSTIC 1112f5df1827Smickey movl CPUVAR(ILEVEL), %r8d 1113f5df1827Smickey testl %r8d, %r8d 1114f5df1827Smickey jne 3f 1115f5df1827Smickey#endif 1116f5df1827Smickey /* 1117f5df1827Smickey * XXX interrupts off longer than they should be here. 1118f5df1827Smickey */ 1119f5df1827Smickey cli 1120f5df1827Smickey swapgs 1121f5df1827Smickey movw TF_GS(%rsp),%gs 1122f5df1827Smickey movw TF_FS(%rsp),%fs 1123f5df1827Smickey movw TF_ES(%rsp),%es 1124f5df1827Smickey INTR_RESTORE_GPRS 1125f5df1827Smickey addq $48,%rsp 1126f5df1827Smickey popq %rcx /* return rip */ 1127f5df1827Smickey addq $8,%rsp 1128f5df1827Smickey popq %r11 /* flags as set by sysret insn */ 1129f5df1827Smickey movq (%rsp),%rsp 1130f5df1827Smickey sysretq 1131f5df1827Smickey 1132f5df1827Smickey#ifdef DIAGNOSTIC 1133f5df1827Smickey3: movabsq $4f, %rdi 1134f5df1827Smickey movl TF_RAX(%rsp),%esi 1135f5df1827Smickey movl TF_RDI(%rsp),%edx 1136f5df1827Smickey movl %ebx,%ecx 1137f5df1827Smickey xorq %rax,%rax 1138f5df1827Smickey call _C_LABEL(printf) 1139f5df1827Smickey#ifdef DDB 1140f5df1827Smickey int $3 1141f5df1827Smickey#endif /* DDB */ 1142f5df1827Smickey movl $IPL_NONE,CPUVAR(ILEVEL) 1143f5df1827Smickey jmp 1b 1144f5df1827Smickey4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL %d %d EXIT %x %x\n" 1145f5df1827Smickey#endif 1146f5df1827Smickey 1147f5df1827Smickey 1148f5df1827SmickeyNENTRY(proc_trampoline) 1149f5df1827Smickey#ifdef MULTIPROCESSOR 1150f5df1827Smickey call _C_LABEL(proc_trampoline_mp) 1151f5df1827Smickey#endif 1152f5df1827Smickey movl $IPL_NONE,CPUVAR(ILEVEL) 1153f5df1827Smickey movq %r13,%rdi 1154f5df1827Smickey call *%r12 1155f5df1827Smickey INTRFASTEXIT 1156f5df1827Smickey /* NOTREACHED */ 1157f5df1827Smickey 1158f5df1827SmickeyNENTRY(child_trampoline) 1159f5df1827Smickey#ifdef MULTIPROCESSOR 1160f5df1827Smickey call _C_LABEL(proc_trampoline_mp) 1161f5df1827Smickey#endif 1162f5df1827Smickey movl $IPL_NONE,CPUVAR(ILEVEL) 1163f5df1827Smickey movq %r13,%rdi 1164f5df1827Smickey call *%r12 1165f5df1827Smickey jmp syscall_return 1166f5df1827Smickey 1167f5df1827Smickey .globl _C_LABEL(osyscall_return) 1168f5df1827Smickey 1169f5df1827Smickey/* 1170f5df1827Smickey * Old call gate entry for syscall. XXXfvdl: only needed if we're 1171f5df1827Smickey * going to support running old NetBSD or ibcs2 binaries, etc, 1172f5df1827Smickey * on NetBSD/amd64. 1173f5df1827Smickey */ 1174f5df1827SmickeyIDTVEC(oosyscall) 1175f5df1827Smickey /* Set rflags in trap frame. */ 1176f5df1827Smickey pushfq 1177f5df1827Smickey popq 8(%rsp) 1178f5df1827Smickey pushq $7 # size of instruction for restart 1179f5df1827Smickey jmp osyscall1 1180f5df1827Smickey 1181f5df1827Smickey/* 1182f5df1827Smickey * Trap gate entry for int $80 syscall, also used by sigreturn. 1183f5df1827Smickey */ 1184f5df1827SmickeyIDTVEC(osyscall) 1185f5df1827Smickey pushq $2 # size of instruction for restart 1186f5df1827Smickeyosyscall1: 1187f5df1827Smickey pushq $T_ASTFLT # trap # for doing ASTs 1188f5df1827Smickey INTRENTRY 1189f5df1827Smickey sti 1190f5df1827Smickey movq CPUVAR(CURPROC),%rdx 1191f5df1827Smickey movq %rsp,P_MD_REGS(%rdx) # save pointer to frame 11924e1a77ceSsturm call _C_LABEL(syscall) 1193f5df1827Smickey_C_LABEL(osyscall_return): 1194f5df1827Smickey2: /* Check for ASTs on exit to user mode. */ 1195f5df1827Smickey cli 1196f5df1827Smickey CHECK_ASTPENDING(%r11) 1197f5df1827Smickey je 1f 1198f5df1827Smickey /* Always returning to user mode here. */ 1199f5df1827Smickey CLEAR_ASTPENDING(%r11) 1200f5df1827Smickey sti 1201f5df1827Smickey /* Pushed T_ASTFLT into tf_trapno on entry. */ 1202f5df1827Smickey call _C_LABEL(trap) 1203f5df1827Smickey jmp 2b 1204f5df1827Smickeyiret_return: 1205f5df1827Smickey#ifndef DIAGNOSTIC 1206f5df1827Smickey1: INTRFASTEXIT 1207f5df1827Smickey#else /* DIAGNOSTIC */ 1208f5df1827Smickey1: cmpl $IPL_NONE,CPUVAR(ILEVEL) 1209f5df1827Smickey jne 3f 1210f5df1827Smickey INTRFASTEXIT 1211f5df1827Smickey3: sti 1212f5df1827Smickey movabsq $4f, %rdi 1213f5df1827Smickey xorq %rax,%rax 1214f5df1827Smickey call _C_LABEL(printf) 1215f5df1827Smickey#ifdef DDB 1216f5df1827Smickey int $3 1217f5df1827Smickey#endif /* DDB */ 1218f5df1827Smickey movl $IPL_NONE,CPUVAR(ILEVEL) 1219f5df1827Smickey jmp 2b 1220f5df1827Smickey4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL EXIT\n" 1221f5df1827Smickey#endif /* DIAGNOSTIC */ 1222