xref: /openbsd/sys/arch/i386/i386/locore.s (revision ad7ea506) 
1/*	$OpenBSD: locore.s,v 1.206 2024/10/21 07:21:18 jsg Exp $	*/
2/*	$NetBSD: locore.s,v 1.145 1996/05/03 19:41:19 christos Exp $	*/
3
4/*-
5 * Copyright (c) 1993, 1994, 1995 Charles M. Hannum.  All rights reserved.
6 * Copyright (c) 1990 The Regents of the University of California.
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * William Jolitz.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 *    notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 *    notice, this list of conditions and the following disclaimer in the
19 *    documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 *    may be used to endorse or promote products derived from this software
22 *    without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 *	@(#)locore.s	7.3 (Berkeley) 5/13/91
37 */
38
39#include "npx.h"
40#include "assym.h"
41#include "lapic.h"
42
43#include <sys/errno.h>
44#include <sys/syscall.h>
45
46#include <machine/codepatch.h>
47#include <machine/param.h>
48#include <machine/pte.h>
49#include <machine/segments.h>
50#include <machine/specialreg.h>
51#include <machine/trap.h>
52
53#if NLAPIC > 0
54#include <machine/i82489reg.h>
55#endif
56
57/*
58 * As stac/clac SMAP instructions are 3 bytes, we want the fastest
59 * 3 byte nop sequence possible here.  This will be replaced by
60 * stac/clac instructions if SMAP is detected after booting.
61 *
62 * Intel documents multi-byte NOP sequences as being available
63 * on all family 0x6 and 0xf processors (ie 686+)
64 * So use 3 of the single byte nops for compatibility
65 */
66#define SMAP_NOP	.byte 0x90, 0x90, 0x90
67#define SMAP_STAC	CODEPATCH_START			;\
68			SMAP_NOP			;\
69			CODEPATCH_END(CPTAG_STAC)
70#define SMAP_CLAC	CODEPATCH_START			;\
71			SMAP_NOP			;\
72			CODEPATCH_END(CPTAG_CLAC)
73
74/*
75 * override user-land alignment before including asm.h
76 */
77
78#define	ALIGN_DATA	.align  4,0xcc
79#define	ALIGN_TEXT	.align  4,0x90	/* 4-byte boundaries, NOP-filled */
80#define _ALIGN_TEXT	ALIGN_TEXT
81#include <machine/asm.h>
82
83#define CPL lapic_tpr
84
85#define	GET_CURPCB(reg)					\
86	movl	CPUVAR(CURPCB), reg
87
88#define	CHECK_ASTPENDING(treg)				\
89	movl 	CPUVAR(CURPROC),treg		;	\
90	cmpl	$0, treg			;	\
91	je	1f				;	\
92	cmpl	$0,P_MD_ASTPENDING(treg)	;	\
93	1:
94
95#define	CLEAR_ASTPENDING(cpreg)				\
96	movl	$0,P_MD_ASTPENDING(cpreg)
97
98/*
99 * These are used on interrupt or trap entry or exit.
100 */
101#define INTR_COPY_FROM_TRAMP_STACK	\
102	movl	TRF_SS(%ebp),%eax	; \
103	movl	%eax,IRF_SS(%esp)	; \
104	movl	TRF_ESP(%ebp),%eax	; \
105	movl	%eax,IRF_ESP(%esp)	; \
106	movl	TRF_EFLAGS(%ebp),%eax	; \
107	movl	%eax,IRF_EFLAGS(%esp)	; \
108	movl	TRF_CS(%ebp),%eax	; \
109	movl	%eax,IRF_CS(%esp)	; \
110	movl	TRF_EIP(%ebp),%eax	; \
111	movl	%eax,IRF_EIP(%esp)	; \
112	movl	TRF_ERR(%ebp),%eax	; \
113	movl	%eax,IRF_ERR(%esp)	; \
114	movl	TRF_TRAPNO(%ebp),%eax	; \
115	movl	%eax,IRF_TRAPNO(%esp)
116
117#define INTR_ENABLE_U_PLUS_K	\
118	movl	$GSEL(GCPU_SEL, SEL_KPL),%eax	; \
119	movw	%ax,%fs			; \
120	movl	CPUVAR(KERN_CR3),%eax	; \
121	testl	%eax,%eax		; \
122	jz	100f			; \
123	movl	%eax,%cr3		; \
124	100:
125
126#define INTRENTRY_LABEL(label)	X##label##_untramp
127#define	INTRENTRY(label) \
128	/* we have an iretframe */	; \
129	testb	$SEL_RPL,IRF_CS(%esp)	; \
130	/* from kernel, stay on kernel stack, use iretframe */	; \
131	je	INTRENTRY_LABEL(label)	; \
132	/* entering from user space, map kernel */	; \
133	pushl	%ebp			; \
134	pushl	%eax			; \
135	pushl	%fs			; \
136	INTR_ENABLE_U_PLUS_K		; \
137	jmp	99f			; \
138	.text				; \
139	.global INTRENTRY_LABEL(label) ; \
140INTRENTRY_LABEL(label):	/* from kernel */	; \
141	jmp	98f			; \
142	/* from user space, build trampframe */	; \
14399:	movl	CPUVAR(KERN_ESP),%eax	; \
144	pushl	%eax			; \
145	pushl	$0xdeadbeef		; \
146	movl	%esp,%ebp		; \
147	movl	%eax,%esp		; \
148	subl	$SIZEOF_IRETFRAME,%esp	; \
149	/* we have a trampframe, copy to iretframe on kernel stack */	; \
150	INTR_COPY_FROM_TRAMP_STACK	; \
151	movl	TRF_FS(%ebp),%eax	; \
152	movw	%ax,%fs			; \
153	movl	TRF_EAX(%ebp),%eax	; \
154	movl	TRF_EBP(%ebp),%ebp	; \
15598:	INTR_SAVE_ALL
156
157#define INTR_SAVE_ALL \
158	cld				; \
159	SMAP_CLAC			; \
160	/* we have an iretframe, build trapframe */	; \
161	subl	$44,%esp		; \
162	movl	%eax,TF_EAX(%esp)	; \
163	/* the hardware puts err next to %eip, we move it elsewhere and */ ; \
164	/* later put %ebp in this slot to make it look like a call frame */ ; \
165	movl	(TF_EIP - 4)(%esp),%eax ; \
166	movl	%eax,TF_ERR(%esp)	; \
167	movl	%ecx,TF_ECX(%esp)	; \
168	movl	%edx,TF_EDX(%esp)	; \
169	movl	%ebx,TF_EBX(%esp)	; \
170	movl	%ebp,TF_EBP(%esp)	; \
171	leal	TF_EBP(%esp),%ebp	; \
172	movl	%esi,TF_ESI(%esp)	; \
173	movl	%edi,TF_EDI(%esp)	; \
174	movw	%ds,TF_DS(%esp)		; \
175	movw	%es,TF_ES(%esp)		; \
176	movw	%gs,TF_GS(%esp)		; \
177	movl	$GSEL(GDATA_SEL, SEL_KPL),%eax	; \
178	movw	%ax,%ds			; \
179	movw	%ax,%es			; \
180	xorl	%eax,%eax	; /* $GSEL(GNULL_SEL, SEL_KPL) == 0 */ \
181	movw	%ax,%gs			; \
182	movw	%fs,TF_FS(%esp)		; \
183	movl	$GSEL(GCPU_SEL, SEL_KPL),%eax	; \
184	movw	%ax,%fs
185
186#define	INTR_RESTORE_ALL \
187	popl	%fs		; \
188	popl	%gs		; \
189	popl	%es		; \
190	popl	%ds		; \
191	popl	%edi		; \
192	popl	%esi		; \
193	addl	$4,%esp	/*err*/ ; \
194	popl	%ebx		; \
195	popl	%edx		; \
196	popl	%ecx		; \
197	popl	%eax		; \
198	movl	4(%esp),%ebp
199
200#define	INTRFASTEXIT \
201	jmp	intr_fast_exit
202
203#define	INTR_FAKE_TRAP_PUSH_RPB	0xbadabada
204#define	INTR_FAKE_TRAP_POP_RBP	0xbcbcbcbc
205
206/*
207 * PTmap is recursive pagemap at top of virtual address space.
208 * Within PTmap, the page directory can be found (third indirection).
209 */
210	.globl	PTmap, PTD
211	.set	PTmap, (PDSLOT_PTE << PDSHIFT)
212	.set	PTD, (PTmap + PDSLOT_PTE * NBPG)
213
214/*
215 * Initialization
216 */
217	.data
218
219	.globl	cpu_id, cpu_vendor
220	.globl	cpu_brandstr
221	.globl	cpuid_level
222	.globl	cpu_miscinfo
223	.globl	cpu_feature, cpu_ecxfeature
224	.globl	ecpu_feature, ecpu_eaxfeature
225	.globl	ecpu_ecxfeature
226	.globl	cpu_cache_eax, cpu_cache_ebx
227	.globl	cpu_cache_ecx, cpu_cache_edx
228	.globl	cpu_perf_eax
229	.globl	cpu_perf_ebx
230	.globl	cpu_perf_edx
231	.globl	cpu_apmi_edx
232	.globl	cold, cnvmem, extmem
233	.globl	cpu_pae
234	.globl	esym
235	.globl	ssym
236	.globl	nkptp_max
237	.globl	boothowto, bootdev, atdevbase
238	.globl	proc0paddr, PTDpaddr, PTDsize
239	.globl	gdt
240	.globl	bootapiver, bootargc, bootargv
241	.globl	lapic_tpr
242	.globl	pg_g_kern
243	.globl	cpu_meltdown
244
245#if NLAPIC > 0
246	.align NBPG
247	.globl local_apic, lapic_id
248local_apic:
249	.space	LAPIC_ID
250lapic_id:
251	.long	0x00000000
252	.space	LAPIC_TPRI-(LAPIC_ID+4)
253lapic_tpr:
254	.space	LAPIC_PPRI-LAPIC_TPRI
255lapic_ppr:
256	.space	LAPIC_ISR-LAPIC_PPRI
257lapic_isr:
258	.space	NBPG-LAPIC_ISR
259#else
260lapic_tpr:
261	.long	0
262#endif
263
264cpu_id:			.long	0	# saved from 'cpuid' instruction
265cpu_pae:		.long	0	# are we using PAE paging mode?
266cpu_miscinfo:		.long	0	# misc info (apic/brand id) from 'cpuid'
267cpu_feature:		.long	0	# feature flags from 'cpuid' instruction
268ecpu_feature:		.long	0	# extended feature flags from 'cpuid'
269cpu_ecxfeature:		.long	0	# ecx feature flags from 'cpuid'
270ecpu_eaxfeature:	.long	0	# extended eax feature flags
271ecpu_ecxfeature:	.long	0	# extended ecx feature flags
272cpuid_level:		.long	-1	# max. lvl accepted by 'cpuid' insn
273cpu_cache_eax:		.long	0
274cpu_cache_ebx:		.long	0
275cpu_cache_ecx:		.long	0
276cpu_cache_edx:		.long	0
277cpu_perf_eax:		.long	0	# arch. perf. mon. flags from 'cpuid'
278cpu_perf_ebx:		.long	0	# arch. perf. mon. flags from 'cpuid'
279cpu_perf_edx:		.long	0	# arch. perf. mon. flags from 'cpuid'
280cpu_apmi_edx:		.long	0	# adv. power management info. 'cpuid'
281cpu_vendor:		.space	16	# vendor string returned by 'cpuid' instruction
282cpu_brandstr:		.space	48	# brand string returned by 'cpuid'
283cold:			.long	1	# cold till we are not
284ssym:			.long	0	# ptr to start of syms
285esym:			.long	0	# ptr to end of syms
286cnvmem:			.long	0	# conventional memory size
287extmem:			.long	0	# extended memory size
288atdevbase:		.long	0	# location of start of iomem in virtual
289bootapiver:		.long	0	# /boot API version
290bootargc:		.long	0	# /boot argc
291bootargv:		.long	0	# /boot argv
292bootdev:		.long	0	# device we booted from
293proc0paddr:		.long	0
294PTDpaddr:		.long	0	# paddr of PTD, for libkvm
295PTDsize:		.long	NBPG	# size of PTD, for libkvm
296pg_g_kern:		.long	0	# 0x100 if global pages should be used
297					# in kernel mappings, 0 otherwise (for
298					# insecure CPUs)
299cpu_meltdown: .long		0	# 1 if this CPU has Meltdown
300
301	.text
302
303NENTRY(proc_trampoline)
304	call	proc_trampoline_mi
305	pushl	%ebx
306	call	*%esi
307	addl	$4,%esp
308#ifdef DIAGNOSTIC
309	movl	$0xfe,%esi
310#endif
311	jmp	.Lsyscall_check_asts
312
313	/* This must come before any use of the CODEPATCH macros */
314       .section .codepatch,"a"
315       .align  8
316       .globl codepatch_begin
317codepatch_begin:
318       .previous
319
320       .section .codepatchend,"a"
321       .globl codepatch_end
322codepatch_end:
323       .previous
324
325/*****************************************************************************/
326
327/*
328 * Signal trampoline; copied to top of user stack.
329 */
330	.section .rodata
331	.globl	sigcode
332sigcode:
333	call	*SIGF_HANDLER(%esp)
334	leal	SIGF_SC(%esp),%eax	# scp (the call may have clobbered the
335					# copy at SIGF_SCP(%esp))
336	pushl	%eax
337	pushl	%eax			# junk to fake return address
338	movl	$SYS_sigreturn,%eax
339	.globl	sigcodecall
340sigcodecall:
341	int	$0x80			# enter kernel with args on stack
342	.globl	sigcoderet
343sigcoderet:
344	.globl	esigcode
345esigcode:
346	/* FALLTHROUGH */
347	.globl	sigfill
348sigfill:
349	int3
350esigfill:
351
352	.data
353	.globl	sigfillsiz
354sigfillsiz:
355	.long	esigfill - sigfill
356
357	.text
358
359/*****************************************************************************/
360
361/*
362 * The following primitives are used to fill and copy regions of memory.
363 */
364
365/* Frame pointer reserve on stack. */
366#ifdef DDB
367#define FPADD 4
368#else
369#define FPADD 0
370#endif
371
372/*
373 * kcopy(caddr_t from, caddr_t to, size_t len);
374 * Copy len bytes, abort on fault.
375 */
376ENTRY(kcopy)
377#ifdef DDB
378	pushl	%ebp
379	movl	%esp,%ebp
380#endif
381	pushl	%esi
382	pushl	%edi
383	GET_CURPCB(%eax)		# load curpcb into eax and set on-fault
384	pushl	PCB_ONFAULT(%eax)
385	movl	$copy_fault, PCB_ONFAULT(%eax)
386
387	movl	16+FPADD(%esp),%esi
388	movl	20+FPADD(%esp),%edi
389	movl	24+FPADD(%esp),%ecx
390	movl	%edi,%eax
391	subl	%esi,%eax
392	cmpl	%ecx,%eax		# overlapping?
393	jb	1f
394	shrl	$2,%ecx			# nope, copy forward by 32-bit words
395	rep
396	movsl
397	movl	24+FPADD(%esp),%ecx
398	andl	$3,%ecx			# any bytes left?
399	rep
400	movsb
401
402	GET_CURPCB(%edx)		# XXX save curpcb?
403	popl	PCB_ONFAULT(%edx)
404	popl	%edi
405	popl	%esi
406	xorl	%eax,%eax
407#ifdef DDB
408	leave
409#endif
410	ret
411
412	.align  4,0xcc
4131:	addl	%ecx,%edi		# copy backward
414	addl	%ecx,%esi
415	std
416	andl	$3,%ecx			# any fractional bytes?
417	decl	%edi
418	decl	%esi
419	rep
420	movsb
421	movl	24+FPADD(%esp),%ecx	# copy remainder by 32-bit words
422	shrl	$2,%ecx
423	subl	$3,%esi
424	subl	$3,%edi
425	rep
426	movsl
427	cld
428
429	GET_CURPCB(%edx)
430	popl	PCB_ONFAULT(%edx)
431	popl	%edi
432	popl	%esi
433	xorl	%eax,%eax
434#ifdef DDB
435	leave
436#endif
437	ret
438
439/*****************************************************************************/
440
441/*
442 * The following primitives are used to copy data in and out of the user's
443 * address space.
444 */
445
446/*
447 * copyout(caddr_t from, caddr_t to, size_t len);
448 * Copy len bytes into the user's address space.
449 */
450ENTRY(copyout)
451#ifdef DDB
452	pushl	%ebp
453	movl	%esp,%ebp
454#endif
455	pushl	%esi
456	pushl	%edi
457	pushl	$0
458
459	movl	16+FPADD(%esp),%esi
460	movl	20+FPADD(%esp),%edi
461	movl	24+FPADD(%esp),%eax
462
463	/*
464	 * We check that the end of the destination buffer is not past the end
465	 * of the user's address space.  If it's not, then we only need to
466	 * check that each page is writable.  The 486 will do this for us; the
467	 * 386 will not.  (We assume that pages in user space that are not
468	 * writable by the user are not writable by the kernel either.)
469	 */
470	movl	%edi,%edx
471	addl	%eax,%edx
472	jc	copy_fault
473	cmpl	$VM_MAXUSER_ADDRESS,%edx
474	ja	copy_fault
475
476	GET_CURPCB(%edx)
477	movl	$copy_fault,PCB_ONFAULT(%edx)
478	SMAP_STAC
479
480	/* bcopy(%esi, %edi, %eax); */
481	movl	%eax,%ecx
482	shrl	$2,%ecx
483	rep
484	movsl
485	movl	%eax,%ecx
486	andl	$3,%ecx
487	rep
488	movsb
489
490	SMAP_CLAC
491	popl	PCB_ONFAULT(%edx)
492	popl	%edi
493	popl	%esi
494	xorl	%eax,%eax
495#ifdef DDB
496	leave
497#endif
498	ret
499
500/*
501 * _copyin(caddr_t from, caddr_t to, size_t len);
502 * Copy len bytes from the user's address space.
503 */
504ENTRY(_copyin)
505#ifdef DDB
506	pushl	%ebp
507	movl	%esp,%ebp
508#endif
509	pushl	%esi
510	pushl	%edi
511	GET_CURPCB(%eax)
512	pushl	$0
513	movl	$copy_fault,PCB_ONFAULT(%eax)
514	SMAP_STAC
515
516	movl	16+FPADD(%esp),%esi
517	movl	20+FPADD(%esp),%edi
518	movl	24+FPADD(%esp),%eax
519
520	/*
521	 * We check that the end of the destination buffer is not past the end
522	 * of the user's address space.  If it's not, then we only need to
523	 * check that each page is readable, and the CPU will do that for us.
524	 */
525	movl	%esi,%edx
526	addl	%eax,%edx
527	jc	copy_fault
528	cmpl	$VM_MAXUSER_ADDRESS,%edx
529	ja	copy_fault
530
531	/* bcopy(%esi, %edi, %eax); */
532	movl	%eax,%ecx
533	shrl	$2,%ecx
534	rep
535	movsl
536	movb	%al,%cl
537	andb	$3,%cl
538	rep
539	movsb
540
541	SMAP_CLAC
542	GET_CURPCB(%edx)
543	popl	PCB_ONFAULT(%edx)
544	popl	%edi
545	popl	%esi
546	xorl	%eax,%eax
547#ifdef DDB
548	leave
549#endif
550	ret
551
552ENTRY(copy_fault)
553	cld
554	SMAP_CLAC
555	GET_CURPCB(%edx)
556	popl	PCB_ONFAULT(%edx)
557	popl	%edi
558	popl	%esi
559	movl	$EFAULT,%eax
560#ifdef DDB
561	leave
562#endif
563	ret
564
565/*
566 * copyoutstr(caddr_t from, caddr_t to, size_t maxlen, size_t *lencopied);
567 * Copy a NUL-terminated string, at most maxlen characters long, into the
568 * user's address space.  Return the number of characters copied (including the
569 * NUL) in *lencopied.  If the string is too long, return ENAMETOOLONG; else
570 * return 0 or EFAULT.
571 */
572ENTRY(copyoutstr)
573#ifdef DDB
574	pushl	%ebp
575	movl	%esp,%ebp
576#endif
577	pushl	%esi
578	pushl	%edi
579
580	movl	12+FPADD(%esp),%esi		# esi = from
581	movl	16+FPADD(%esp),%edi		# edi = to
582	movl	20+FPADD(%esp),%edx		# edx = maxlen
583
5845:	GET_CURPCB(%eax)
585	movl	$copystr_fault,PCB_ONFAULT(%eax)
586	SMAP_STAC
587	/*
588	 * Get min(%edx, VM_MAXUSER_ADDRESS-%edi).
589	 */
590	movl	$VM_MAXUSER_ADDRESS,%eax
591	subl	%edi,%eax
592	jbe	copystr_fault			# die if CF == 1 || ZF == 1
593						# i.e. make sure that %edi
594						# is below VM_MAXUSER_ADDRESS
595
596	cmpl	%edx,%eax
597	jae	1f
598	movl	%eax,%edx
599	movl	%eax,20+FPADD(%esp)
600
6011:	incl	%edx
602
6031:	decl	%edx
604	jz	2f
605	lodsb
606	stosb
607	testb	%al,%al
608	jnz	1b
609
610	/* Success -- 0 byte reached. */
611	decl	%edx
612	xorl	%eax,%eax
613	jmp	copystr_return
614
6152:	/* edx is zero -- return EFAULT or ENAMETOOLONG. */
616	cmpl	$VM_MAXUSER_ADDRESS,%edi
617	jae	copystr_fault
618	movl	$ENAMETOOLONG,%eax
619	jmp	copystr_return
620
621/*
622 * _copyinstr(caddr_t from, caddr_t to, size_t maxlen, size_t *lencopied);
623 * Copy a NUL-terminated string, at most maxlen characters long, from the
624 * user's address space.  Return the number of characters copied (including the
625 * NUL) in *lencopied.  If the string is too long, return ENAMETOOLONG; else
626 * return 0 or EFAULT.
627 */
628ENTRY(_copyinstr)
629#ifdef DDB
630	pushl	%ebp
631	movl	%esp,%ebp
632#endif
633	pushl	%esi
634	pushl	%edi
635	GET_CURPCB(%ecx)
636	movl	$copystr_fault,PCB_ONFAULT(%ecx)
637	SMAP_STAC
638
639	movl	12+FPADD(%esp),%esi		# %esi = from
640	movl	16+FPADD(%esp),%edi		# %edi = to
641	movl	20+FPADD(%esp),%edx		# %edx = maxlen
642
643	/*
644	 * Get min(%edx, VM_MAXUSER_ADDRESS-%esi).
645	 */
646	movl	$VM_MAXUSER_ADDRESS,%eax
647	subl	%esi,%eax
648	jbe	copystr_fault			# Error if CF == 1 || ZF == 1
649						# i.e. make sure that %esi
650						# is below VM_MAXUSER_ADDRESS
651	cmpl	%edx,%eax
652	jae	1f
653	movl	%eax,%edx
654	movl	%eax,20+FPADD(%esp)
655
6561:	incl	%edx
657
6581:	decl	%edx
659	jz	2f
660	lodsb
661	stosb
662	testb	%al,%al
663	jnz	1b
664
665	/* Success -- 0 byte reached. */
666	decl	%edx
667	xorl	%eax,%eax
668	jmp	copystr_return
669
6702:	/* edx is zero -- return EFAULT or ENAMETOOLONG. */
671	cmpl	$VM_MAXUSER_ADDRESS,%esi
672	jae	copystr_fault
673	movl	$ENAMETOOLONG,%eax
674	jmp	copystr_return
675
676ENTRY(copystr_fault)
677	movl	$EFAULT,%eax
678
679copystr_return:
680	SMAP_CLAC
681	/* Set *lencopied and return %eax. */
682	GET_CURPCB(%ecx)
683	movl	$0,PCB_ONFAULT(%ecx)
684	movl	20+FPADD(%esp),%ecx
685	subl	%edx,%ecx
686	movl	24+FPADD(%esp),%edx
687	testl	%edx,%edx
688	jz	8f
689	movl	%ecx,(%edx)
690
6918:	popl	%edi
692	popl	%esi
693#ifdef DDB
694	leave
695#endif
696	ret
697
698/*****************************************************************************/
699
700/*
701 * The following is i386-specific nonsense.
702 */
703
704/*
705 * void lgdt(struct region_descriptor *rdp);
706 * Change the global descriptor table.
707 */
708NENTRY(lgdt)
709	/* Reload the descriptor table. */
710	movl	4(%esp),%eax
711	lgdt	(%eax)
712	/* Flush the prefetch q. */
713	jmp	1f
714	nop
7151:	/* Reload "stale" selectors. */
716	movl	$GSEL(GDATA_SEL, SEL_KPL),%eax
717	movw	%ax,%ds
718	movw	%ax,%es
719	movw	%ax,%ss
720	movl	$GSEL(GCPU_SEL, SEL_KPL),%eax
721	movw	%ax,%fs
722	/* Reload code selector by doing intersegment return. */
723	popl	%eax
724	pushl	$GSEL(GCODE_SEL, SEL_KPL)
725	pushl	%eax
726	lret
727
728#ifdef DDB
729ENTRY(setjmp)
730	movl	4(%esp),%eax
731	movl	%ebx,(%eax)		# save ebx
732	movl	%esp,4(%eax)		# save esp
733	movl	%ebp,8(%eax)		# save ebp
734	movl	%esi,12(%eax)		# save esi
735	movl	%edi,16(%eax)		# save edi
736	movl	(%esp),%edx		# get rta
737	movl	%edx,20(%eax)		# save eip
738	xorl	%eax,%eax		# return (0);
739	ret
740
741ENTRY(longjmp)
742	movl	4(%esp),%eax
743	movl	(%eax),%ebx		# restore ebx
744	movl	4(%eax),%esp		# restore esp
745	movl	8(%eax),%ebp		# restore ebp
746	movl	12(%eax),%esi		# restore esi
747	movl	16(%eax),%edi		# restore edi
748	movl	20(%eax),%edx		# get rta
749	movl	%edx,(%esp)		# put in return frame
750	xorl	%eax,%eax		# return (1);
751	incl	%eax
752	ret
753#endif /* DDB */
754
755/*****************************************************************************/
756
757/*
758 * cpu_switchto(struct proc *old, struct proc *new)
759 * Switch from the "old" proc to the "new" proc. If "old" is NULL, we
760 * don't need to bother saving old context.
761 */
762ENTRY(cpu_switchto)
763	pushl	%ebx
764	pushl	%esi
765	pushl	%edi
766
767	movl	16(%esp), %esi
768	movl	20(%esp), %edi
769
770	/* If old process exited, don't bother. */
771	testl	%esi,%esi
772	jz	switch_exited
773
774	/* Save old stack pointers. */
775	movl	P_ADDR(%esi),%ebx
776	movl	%esp,PCB_ESP(%ebx)
777	movl	%ebp,PCB_EBP(%ebx)
778
779switch_exited:
780	/* Restore saved context. */
781
782	/* No interrupts while loading new state. */
783	cli
784
785	/* Record new process. */
786	movl	%edi, CPUVAR(CURPROC)
787	movb	$SONPROC, P_STAT(%edi)
788
789	/* Restore stack pointers. */
790	movl	P_ADDR(%edi),%ebx
791	movl	PCB_ESP(%ebx),%esp
792	movl	PCB_EBP(%ebx),%ebp
793
794	/* Record new pcb. */
795	movl	%ebx, CPUVAR(CURPCB)
796
797	/* record the bits needed for future U-->K transition */
798	movl	PCB_KSTACK(%ebx),%eax
799	subl	$FRAMESIZE,%eax
800	movl	%eax,CPUVAR(KERN_ESP)
801
802	/*
803	 * Activate the address space.  The pcb copy of %cr3 will
804	 * be refreshed from the pmap, and because we're
805	 * curproc they'll both be reloaded into the CPU.
806	 */
807	pushl	%edi
808	pushl	%esi
809	call	pmap_switch
810	addl	$8,%esp
811
812	/* Restore cr0 (including FPU state). */
813	movl	PCB_CR0(%ebx),%ecx
814#ifdef MULTIPROCESSOR
815	/*
816	 * If our floating point registers are on a different CPU,
817	 * clear CR0_TS so we'll trap rather than reuse bogus state.
818	 */
819	movl	CPUVAR(SELF), %esi
820	cmpl	PCB_FPCPU(%ebx), %esi
821	jz	1f
822	orl	$CR0_TS,%ecx
8231:
824#endif
825	movl	%ecx,%cr0
826
827	/* Interrupts are okay again. */
828	sti
829
830	popl	%edi
831	popl	%esi
832	popl	%ebx
833	ret
834
835ENTRY(cpu_idle_enter)
836	movl	cpu_idle_enter_fcn,%eax
837	cmpl	$0,%eax
838	je	1f
839	jmpl	*%eax
8401:
841	ret
842
843ENTRY(cpu_idle_cycle)
844	movl	cpu_idle_cycle_fcn,%eax
845	cmpl	$0,%eax
846	je	1f
847	call	*%eax
848	ret
8491:
850	sti
851	hlt
852	ret
853
854ENTRY(cpu_idle_leave)
855	movl	cpu_idle_leave_fcn,%eax
856	cmpl	$0,%eax
857	je	1f
858	jmpl	*%eax
8591:
860	ret
861
862/*
863 * savectx(struct pcb *pcb);
864 * Update pcb, saving current processor state.
865 */
866ENTRY(savectx)
867	movl	4(%esp),%edx		# edx = p->p_addr
868
869	/* Save stack pointers. */
870	movl	%esp,PCB_ESP(%edx)
871	movl	%ebp,PCB_EBP(%edx)
872
873	movl	PCB_FLAGS(%edx),%ecx
874	orl	$PCB_SAVECTX,%ecx
875	movl	%ecx,PCB_FLAGS(%edx)
876
877	ret
878
879/*****************************************************************************/
880
881/*
882 * Trap and fault vector routines
883 *
884 * On exit from the kernel to user mode, we always need to check for ASTs.  In
885 * addition, we need to do this atomically; otherwise an interrupt may occur
886 * which causes an AST, but it won't get processed until the next kernel entry
887 * (possibly the next clock tick).  Thus, we disable interrupt before checking,
888 * and only enable them again on the final `iret' or before calling the AST
889 * handler.
890 */
891
892#define	TRAP(a)		pushl $(a) ; jmp alltraps
893#define	ZTRAP(a)	pushl $0 ; TRAP(a)
894
895IDTVEC(div)
896	ZTRAP(T_DIVIDE)
897IDTVEC(dbg)
898	subl	$4,%esp
899	pushl	%eax
900	movl	%dr6,%eax
901	movl	%eax,4(%esp)
902	andb	$~0xf,%al
903	movl	%eax,%dr6
904	popl	%eax
905	TRAP(T_TRCTRAP)
906
907IDTVEC(nmi)
908	/*
909	 * we came through a task gate; now U+K of the idle thread is
910	 * enabled; NMIs are blocked until next iret; IRQs are disabled;
911	 * all segment descriptors are useable
912	 *
913	 * first of all, switch back to the U+K we were actually running
914	 * on before
915	 */
916	movl	CPUVAR(CURPMAP),%eax
917	movl	PM_PDIRPA(%eax),%eax
918	movl	%eax,%cr3
919
920	/*
921	 * when we came from within the kernel, iret will not
922	 * switch back to the stack we came from but will keep
923	 * running on the NMI stack. in that case we switch
924	 * manually back to the stack we were running on and
925	 * build the iretframe there.
926	 */
927
928	/* was there a ring transition? */
929	movl	CPUVAR(TSS),%eax
930	testb	$SEL_RPL,TSS_CS(%eax)
931	jne	1f
932
933	/*
934	 * no ring transition, switch back to original stack, build
935	 * frame from state saved in TSS.
936	 */
937	movl	TSS_ESP(%eax),%esp
938	subl	$12,%esp
939	movl	TSS_EFLAGS(%eax),%ebx
940	movl	%ebx,8(%esp)
941	movl	TSS_CS(%eax),%ebx
942	movl	%ebx,4(%esp)
943	movl	TSS_EIP(%eax),%ebx
944	movl	%ebx,0(%esp)
945	pushl	$0
946	pushl	$T_NMI
947	jmp	2f
948
949	/*
950	 * ring transition, stay on stack, build frame from state
951	 * saved in TSS.
952	 */
9531:	subl	$20,%esp
954	pushl	$0
955	pushl	$T_NMI
956	movl	TSS_SS(%eax),%ebx
957	movl	%ebx,IRF_SS(%esp)
958	movl	TSS_ESP(%eax),%ebx
959	movl	%ebx,IRF_ESP(%esp)
960	movl	TSS_EFLAGS(%eax),%ebx
961	movl	%ebx,IRF_EFLAGS(%esp)
962	movl	TSS_CS(%eax),%ebx
963	movl	%ebx,IRF_CS(%esp)
964	movl	TSS_EIP(%eax),%ebx
965	movl	%ebx,IRF_EIP(%esp)
966
967	/* clear PSL_NT */
9682:	pushfl
969	popl	%eax
970	andl	$~PSL_NT,%eax
971	pushl	%eax
972	popfl
973
974	/* clear CR0_TS XXX hshoexer: needed? */
975	movl	%cr0,%eax
976	andl	$~CR0_TS,%eax
977	movl	%eax,%cr0
978
979	/* unbusy descriptors and reload common TSS */
980	movl	CPUVAR(GDT),%eax
981	movl	$GSEL(GNMITSS_SEL, SEL_KPL),%ebx
982	andl	$~0x200,4-SEL_KPL(%eax,%ebx,1)
983	movl	$GSEL(GTSS_SEL, SEL_KPL),%ebx
984	andl	$~0x200,4-SEL_KPL(%eax,%ebx,1)
985	ltr	%bx
986
987	/* load GPRs and segment registers with saved values from common TSS */
988	movl	CPUVAR(TSS),%eax
989	movl	TSS_ECX(%eax),%ecx
990	movl	TSS_EDX(%eax),%edx
991	movl	TSS_ESI(%eax),%esi
992	movl	TSS_EDI(%eax),%edi
993	movl	TSS_EBP(%eax),%ebp
994	movw	TSS_FS(%eax),%fs
995	movw	TSS_GS(%eax),%gs
996	movw	TSS_ES(%eax),%es
997	/* saved %ds might be invalid, thus push now and pop later */
998	movl	TSS_DS(%eax),%ebx
999	pushl	%ebx
1000	movl	TSS_EBX(%eax),%ebx
1001	movl	TSS_EAX(%eax),%eax
1002	popl	%ds
1003
1004	/*
1005	 * we can now proceed and save everything on the stack as
1006	 * if no task switch had happened.
1007	 */
1008	jmp alltraps
1009IDTVEC(bpt)
1010	ZTRAP(T_BPTFLT)
1011IDTVEC(ofl)
1012	ZTRAP(T_OFLOW)
1013IDTVEC(bnd)
1014	ZTRAP(T_BOUND)
1015IDTVEC(ill)
1016	ZTRAP(T_PRIVINFLT)
1017IDTVEC(dna)
1018#if NNPX > 0
1019	pushl	$0			# dummy error code
1020	pushl	$T_DNA
1021	INTRENTRY(dna)
1022	sti
1023	pushl	CPUVAR(SELF)
1024	call	*npxdna_func
1025	addl	$4,%esp
1026	testl	%eax,%eax
1027	jz	calltrap
1028#ifdef DIAGNOSTIC
1029	movl	$0xfd,%esi
1030#endif
1031	cli
1032	INTRFASTEXIT
1033#else
1034	ZTRAP(T_DNA)
1035#endif
1036IDTVEC(dble)
1037	TRAP(T_DOUBLEFLT)
1038IDTVEC(fpusegm)
1039	ZTRAP(T_FPOPFLT)
1040IDTVEC(tss)
1041	TRAP(T_TSSFLT)
1042IDTVEC(missing)
1043	TRAP(T_SEGNPFLT)
1044IDTVEC(stk)
1045	TRAP(T_STKFLT)
1046
1047IDTVEC(prot)
1048	pushl	$T_PROTFLT
1049	/* If iret faults, we'll get a trap at doreti_iret+3 with CPL == 0. */
1050	pushl	%eax
1051	leal	doreti_iret+3,%eax
1052	cmpl	%eax,12(%esp)	/* over %eax, trapno and err to %eip */
1053	popl	%eax
1054	jne	97f
1055	pushl	%ebp
1056	pushl	%eax
1057	pushl	%fs
1058	INTR_ENABLE_U_PLUS_K
1059	/*
1060	 * we have an iretframe on trampoline stack, above it the
1061	 * remainder of the original iretframe iret faulted on.
1062	 */
1063	movl	CPUVAR(KERN_ESP),%eax
1064	pushl	%eax
1065	pushl	$0xdeadbeef
1066	/*
1067	 * now we have a trampframe on trampoline stack, above it the
1068	 * remainder of the original iretframe iret faulted on.
1069	 */
1070	movl	%esp,%ebp
1071	movl	%eax,%esp
1072	subl	$SIZEOF_IRETFRAME+(5*4),%esp
1073	/* copy to iretframe on kernel stack */
1074	movl	TRF_EFLAGS(%ebp),%eax
1075	movl	%eax,IRF_EFLAGS(%esp)
1076	movl	TRF_CS(%ebp),%eax
1077	movl	%eax,IRF_CS(%esp)
1078	movl	TRF_EIP(%ebp),%eax
1079	movl	%eax,IRF_EIP(%esp)
1080	movl	TRF_ERR(%ebp),%eax
1081	movl	%eax,IRF_ERR(%esp)
1082	movl	TRF_TRAPNO(%ebp),%eax
1083	movl	%eax,IRF_TRAPNO(%esp)
1084	/* copy remainder of faulted iretframe */
1085	movl	40(%ebp),%eax		/* eip */
1086	movl	%eax,20(%esp)
1087	movl	44(%ebp),%eax		/* cs */
1088	movl	%eax,24(%esp)
1089	movl	48(%ebp),%eax		/* eflags */
1090	movl	%eax,28(%esp)
1091	movl	52(%ebp),%eax		/* esp */
1092	movl	%eax,32(%esp)
1093	movl	56(%ebp),%eax		/* ss */
1094	movl	%eax,36(%esp)
1095	movl	TRF_FS(%ebp),%eax
1096	movw	%ax,%fs
1097	movl	TRF_EAX(%ebp),%eax
1098	movl	TRF_EBP(%ebp),%ebp
1099	/*
1100	 * we have an iretframe on kernel stack, above it the
1101	 * remainder of the original iretframe iret faulted on.
1102	 * for INTRENTRY(prot) it looks like the fault happened
1103	 * on the kernel stack
1104	 */
110597:	INTRENTRY(prot)
1106	sti
1107	jmp	calltrap
1108IDTVEC(f00f_redirect)
1109	pushl	$T_PAGEFLT
1110	INTRENTRY(f00f_redirect)
1111	sti
1112	testb	$PGEX_U,TF_ERR(%esp)
1113	jnz	calltrap
1114	movl	%cr2,%eax
1115	subl	idt,%eax
1116	cmpl	$(6*8),%eax
1117	jne	calltrap
1118	movb	$T_PRIVINFLT,TF_TRAPNO(%esp)
1119	jmp	calltrap
1120IDTVEC(page)
1121	TRAP(T_PAGEFLT)
1122IDTVEC(rsvd)
1123	ZTRAP(T_RESERVED)
1124IDTVEC(mchk)
1125	ZTRAP(T_MACHK)
1126IDTVEC(simd)
1127	ZTRAP(T_XFTRAP)
1128IDTVEC(intrspurious)
1129	/*
1130	 * The Pentium Pro local APIC may erroneously call this vector for a
1131	 * default IR7.  Just ignore it.
1132	 *
1133	 * (The local APIC does this when CPL is raised while it's on the
1134	 * way to delivering an interrupt.. presumably enough has been set
1135	 * up that it's inconvenient to abort delivery completely..)
1136	 */
1137	iret
1138IDTVEC(fpu)
1139#if NNPX > 0
1140	/*
1141	 * Handle like an interrupt so that we can call npxintr to clear the
1142	 * error.  It would be better to handle npx interrupts as traps but
1143	 * this is difficult for nested interrupts.
1144	 */
1145	subl	$8,%esp			/* space for tf_{err,trapno} */
1146	INTRENTRY(fpu)
1147	sti
1148	pushl	CPL			# if_ppl in intrframe
1149	pushl	%esp			# push address of intrframe
1150	incl	uvmexp+V_TRAP
1151	call	npxintr
1152	addl	$8,%esp			# pop address and if_ppl
1153#ifdef DIAGNOSTIC
1154	movl	$0xfc,%esi
1155#endif
1156	cli
1157	INTRFASTEXIT
1158#else
1159	ZTRAP(T_ARITHTRAP)
1160#endif
1161IDTVEC(align)
1162	TRAP(T_ALIGNFLT)
1163	/* 18 - 31 reserved for future exp */
1164
1165/*
1166 * If an error is detected during trap, syscall, or interrupt exit, trap() will
1167 * change %eip to point to one of these labels.  We clean up the stack, if
1168 * necessary, and resume as if we were handling a general protection fault.
1169 * This will cause the process to get a SIGBUS.
1170 */
1171KUENTRY(resume_iret)
1172	ZTRAP(T_PROTFLT)
1173NENTRY(resume_pop_ds)
1174	pushl	%es
1175	movl	$GSEL(GDATA_SEL, SEL_KPL),%eax
1176	movw	%ax,%es
1177NENTRY(resume_pop_es)
1178	pushl	%gs
1179	xorl	%eax,%eax	/* $GSEL(GNULL_SEL, SEL_KPL) == 0 */
1180	movw	%ax,%gs
1181NENTRY(resume_pop_gs)
1182	pushl	%fs
1183	movl	$GSEL(GCPU_SEL, SEL_KPL),%eax
1184	movw	%ax,%fs
1185NENTRY(resume_pop_fs)
1186	movl	$T_PROTFLT,TF_TRAPNO(%esp)
1187	sti
1188	jmp	calltrap
1189
1190/*
1191 * All traps go through here. Call the generic trap handler, and
1192 * check for ASTs afterwards.
1193 */
1194KUENTRY(alltraps)
1195	INTRENTRY(alltraps)
1196	sti
1197calltrap:
1198#ifdef DIAGNOSTIC
1199	movl	CPL,%ebx
1200#endif /* DIAGNOSTIC */
1201#if !defined(GPROF) && defined(DDBPROF)
1202	cmpl	$T_BPTFLT,TF_TRAPNO(%esp)
1203	jne	.Lreal_trap
1204
1205	pushl	%esp
1206	subl	$4, %esp
1207	pushl	%eax
1208	leal	dt_prov_kprobe, %eax
1209	movl	%eax, 4(%esp)
1210	popl	%eax
1211	call	dt_prov_kprobe_hook
1212	addl	$8, %esp
1213	cmpl	$0, %eax
1214	je	.Lreal_trap
1215
1216	/*
1217	 * Abuse the error field to indicate that INTRFASTEXIT needs
1218	 * to emulate the patched instruction.
1219	 */
1220	cmpl	$1, %eax
1221	je	.Lset_emulate_push_rbp
1222
1223	cmpl	$2, %eax
1224	je	.Lset_emulate_ret
1225
1226.Lset_emulate_push_rbp:
1227	movl	$INTR_FAKE_TRAP_PUSH_RPB, TF_ERR(%esp)
1228	jmp	.Lalltraps_check_asts
1229.Lset_emulate_ret:
1230	movl	$INTR_FAKE_TRAP_POP_RBP, TF_ERR(%esp)
1231	jmp	.Lalltraps_check_asts
1232.Lreal_trap:
1233#endif /* !defined(GPROF) && defined(DDBPROF) */
1234	pushl	%esp
1235	call	trap
1236	addl	$4,%esp
1237
1238.Lalltraps_check_asts:
1239	/* Check for ASTs on exit to user mode. */
1240	cli
1241	CHECK_ASTPENDING(%ecx)
1242	je	1f
1243	testb	$SEL_RPL,TF_CS(%esp)
1244	jz	1f
12455:	CLEAR_ASTPENDING(%ecx)
1246	sti
1247	pushl	%esp
1248	call	ast
1249	addl	$4,%esp
1250	jmp	.Lalltraps_check_asts
12511:
1252#if !defined(GPROF) && defined(DDBPROF)
1253	/*
1254	 * If we are returning from a probe trap we need to fix the
1255	 * stack layout and emulate the patched instruction.
1256	 *
1257	 * The code below does that by trashing %eax, so it MUST be
1258	 * restored afterward.
1259	 */
1260	cmpl	$INTR_FAKE_TRAP_PUSH_RPB, TF_ERR(%esp)
1261	je	.Lprobe_fixup_push_rbp
1262	cmpl	$INTR_FAKE_TRAP_POP_RBP, TF_ERR(%esp)
1263	je	.Lprobe_fixup_pop_rbp
1264#endif /* !defined(GPROF) && defined(DDBPROF) */
1265#ifndef DIAGNOSTIC
1266	INTRFASTEXIT
1267#else
1268	cmpl	CPL,%ebx
1269	jne	3f
1270#ifdef DIAGNOSTIC
1271	movl	$0xfb,%esi
1272#endif
1273	INTRFASTEXIT
12743:	sti
1275	pushl	$spl_lowered
1276	call	printf
1277	addl	$4,%esp
1278#if defined(DDB) && 0
1279	int	$3
1280#endif /* DDB */
1281	movl	%ebx,CPL
1282	jmp	.Lalltraps_check_asts
1283
1284	.section .rodata
1285spl_lowered:
1286	.asciz	"WARNING: SPL NOT LOWERED ON TRAP EXIT\n"
1287#endif /* DIAGNOSTIC */
1288
1289	.text
1290#if !defined(GPROF) && defined(DDBPROF)
1291.Lprobe_fixup_push_rbp:
1292	/* Restore all register unwinding the stack. */
1293	INTR_RESTORE_ALL
1294
1295	/*
1296	 * Use the space left by ``err'' and ``trapno'' to emulate
1297	 * "pushl %ebp".
1298	 *
1299	 * Temporarily save %eax.
1300	 */
1301	movl	%eax,0(%esp)
1302
1303	/* Shift hardware-saved registers: eip, cs, eflags */
1304	movl	8(%esp),%eax
1305	movl	%eax,4(%esp)
1306	movl	12(%esp),%eax
1307	movl	%eax,8(%esp)
1308	movl	16(%esp),%eax
1309	movl	%eax,12(%esp)
1310
1311	/* Store %ebp in the expected location to finish the emulation. */
1312	movl	%ebp,16(%esp)
1313
1314	popl	%eax
1315	iret
1316.Lprobe_fixup_pop_rbp:
1317	/* Restore all register unwinding the stack. */
1318	INTR_RESTORE_ALL
1319
1320	movl	%eax, 0(%esp)
1321
1322	/* pop %ebp */
1323	movl	20(%esp), %ebp
1324	/* Shift hardware-saved registers: eflags, cs, eip */
1325	movl	16(%esp), %eax
1326	movl	%eax, 20(%esp)
1327	movl	12(%esp), %eax
1328	movl	%eax, 16(%esp)
1329	movl	8(%esp), %eax
1330	movl	%eax, 12(%esp)
1331
1332	/* Pop eax and restore the stack pointer */
1333	popl	%eax
1334	addl	$8, %esp
1335	iret
1336#endif /* !defined(GPROF) && defined(DDBPROF) */
1337
1338	.text
1339#ifdef DIAGNOSTIC
1340.Lintr_exit_not_blocked:
1341	movl	warn_once,%eax
1342	testl	%eax,%eax
1343	jnz	1f
1344	incl	%eax
1345	movl	%eax,warn_once
1346	pushl	%esi		/* marker indicating where we came from */
1347	pushl	%edx		/* EFLAGS are in %edx */
1348	pushl	$.Lnot_blocked
1349	call	printf
1350	addl	$12,%esp
1351#ifdef DDB
1352	int	$3
1353#endif	/* DDB */
13541:	cli
1355	jmp	intr_fast_exit
1356
1357	.data
1358	.global warn_once
1359warn_once:
1360	.long	0
1361	.section .rodata
1362.Lnot_blocked:
1363	.asciz	"WARNING: INTERRUPTS NOT BLOCKED ON INTERRUPT RETURN 0x%x 0x%x\n"
1364	.text
1365#endif
1366
1367/*
1368 * Trap gate entry for syscall
1369 */
1370IDTVEC(syscall)
1371	subl	$8,%esp			/* space for tf_{err,trapno} */
1372	INTRENTRY(syscall)
1373	sti
1374	pushl	%esp
1375	call	syscall
1376	addl	$4,%esp
1377
1378.Lsyscall_check_asts:
1379	/* Check for ASTs on exit to user mode. */
1380	cli
1381	CHECK_ASTPENDING(%ecx)
1382	je	1f
1383	/* Always returning to user mode here. */
1384	CLEAR_ASTPENDING(%ecx)
1385	sti
1386	pushl	%esp
1387	call	ast
1388	addl	$4,%esp
1389	jmp	.Lsyscall_check_asts
13901:
1391#ifdef DIAGNOSTIC
1392	movl	$0xff,%esi
1393#endif
1394	jmp intr_fast_exit
1395
1396NENTRY(intr_fast_exit)
1397#ifdef DIAGNOSTIC
1398	pushfl
1399	popl	%edx
1400	testl	$PSL_I,%edx
1401	jnz	.Lintr_exit_not_blocked
1402#endif
1403	/* we have a full trapframe */
1404	INTR_RESTORE_ALL
1405	/* now we have an iretframe */
1406	testb	$SEL_RPL,IRF_CS(%esp)
1407	/* recursing into kernel: stay on kernel stack using iretframe */
1408	je	doreti_iret
1409
1410	/* leaving kernel: build trampframe on cpu stack */
1411	pushl	%ebp
1412	pushl	%eax
1413	pushl	%fs
1414        movl	$GSEL(GCPU_SEL, SEL_KPL),%eax
1415	movw	%ax,%fs
1416	movl	CPUVAR(INTR_ESP),%eax
1417	pushl	%eax
1418	pushl	$0xcafecafe
1419	/* now we have an trampframe, copy frame to cpu stack */
1420	movl	%eax,%ebp
1421	movl	TRF_EIP(%esp),%eax
1422	movl	%eax,TRF_EIP(%ebp)
1423	movl	TRF_CS(%esp),%eax
1424	movl	%eax,TRF_CS(%ebp)
1425	movl	TRF_EFLAGS(%esp),%eax
1426	movl	%eax,TRF_EFLAGS(%ebp)
1427	movl	TRF_ESP(%esp),%eax
1428	movl	%eax,TRF_ESP(%ebp)
1429	movl	TRF_SS(%esp),%eax
1430	movl	%eax,TRF_SS(%ebp)
1431	movl	TRF__DEADBEEF(%esp),%eax
1432	movl	%eax,TRF__DEADBEEF(%ebp)
1433	movl	TRF__KERN_ESP(%esp),%eax
1434	movl	%eax,TRF__KERN_ESP(%ebp)
1435	movl	TRF_FS(%esp),%eax
1436	movl	%eax,TRF_FS(%ebp)
1437	movl	TRF_EAX(%esp),%eax
1438	movl	%eax,TRF_EAX(%ebp)
1439	movl	TRF_EBP(%esp),%eax
1440	movl	%eax,TRF_EBP(%ebp)
1441	/* switch to cpu stack, where we copied the trampframe */
1442	movl	%ebp,%esp
1443	movl	CPUVAR(USER_CR3),%eax
1444	testl	%eax,%eax
1445	jz	1f
1446	jmp	iret_tramp
1447
1448KUENTRY(iret_tramp)
1449	movl	%eax,%cr3
1450	/* we have a trampframe; restore registers and adjust to iretframe */
14511:	popl	%eax
1452	popl	%eax
1453	popl	%fs
1454	popl	%eax
1455	popl	%ebp
1456	.globl	doreti_iret
1457doreti_iret:
1458	/* we have an iretframe */
1459	addl	$IRF_EIP,%esp
1460	iret
1461
1462#include <i386/i386/vector.s>
1463#include <i386/isa/icu.s>
1464
1465#if !defined(SMALL_KERNEL)
1466ENTRY(sse2_pagezero)
1467	pushl	%ebx
1468	movl	8(%esp),%ecx
1469	movl	%ecx,%eax
1470	addl	$4096,%eax
1471	xor	%ebx,%ebx
14721:
1473	movnti	%ebx,(%ecx)
1474	addl	$4,%ecx
1475	cmpl	%ecx,%eax
1476	jne	1b
1477	sfence
1478	popl	%ebx
1479	ret
1480
1481ENTRY(i686_pagezero)
1482	pushl	%edi
1483	pushl	%ebx
1484
1485	movl	12(%esp), %edi
1486	movl	$1024, %ecx
1487
1488	.align  4,0x90
14891:
1490	xorl	%eax, %eax
1491	repe
1492	scasl
1493	jnz	2f
1494
1495	popl	%ebx
1496	popl	%edi
1497	ret
1498
1499	.align  4,0x90
15002:
1501	incl	%ecx
1502	subl	$4, %edi
1503
1504	movl	%ecx, %edx
1505	cmpl	$16, %ecx
1506
1507	jge	3f
1508
1509	movl	%edi, %ebx
1510	andl	$0x3f, %ebx
1511	shrl	%ebx
1512	shrl	%ebx
1513	movl	$16, %ecx
1514	subl	%ebx, %ecx
1515
15163:
1517	subl	%ecx, %edx
1518	rep
1519	stosl
1520
1521	movl	%edx, %ecx
1522	testl	%edx, %edx
1523	jnz	1b
1524
1525	popl	%ebx
1526	popl	%edi
1527	ret
1528#endif
1529
1530/*
1531 * int cpu_paenable(void *);
1532 */
1533ENTRY(cpu_paenable)
1534	movl	$-1, %eax
1535	testl	$CPUID_PAE, cpu_feature
1536	jz	1f
1537
1538	pushl	%esi
1539	pushl	%edi
1540	movl	12(%esp), %esi
1541	movl	%cr3, %edi
1542	orl	$0xfe0, %edi    /* PDPT will be in the last four slots! */
1543	movl	%edi, %cr3
1544	addl	$KERNBASE, %edi /* and make it back virtual again */
1545	movl	$8, %ecx
1546	rep
1547	movsl
1548
1549	movl	$MSR_EFER, %ecx
1550	rdmsr
1551	orl	$EFER_NXE, %eax
1552	wrmsr
1553
1554	movl	%cr4, %eax
1555	orl	$CR4_PAE, %eax
1556	movl	%eax, %cr4      /* BANG!!! */
1557
1558	movl	12(%esp), %eax
1559	subl	$KERNBASE, %eax
1560	movl	%eax, %cr3      /* reload real PDPT */
1561	movl	$4*NBPG, %eax
1562	movl	%eax, PTDsize
1563
1564	xorl	%eax, %eax
1565	popl	%edi
1566	popl	%esi
15671:
1568	ret
1569
1570#if NLAPIC > 0
1571#include <i386/i386/apicvec.s>
1572#endif
1573
1574	.section .rodata
1575	.globl _stac
1576_stac:
1577	stac
1578
1579	.globl _clac
1580_clac:
1581	clac
1582