1/*
2 * librm: a library for interfacing to real-mode code
3 *
4 * Michael Brown <mbrown@fensystems.co.uk>
5 *
6 */
7
8FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
9
10/* Drag in general configuration */
11#include <config/general.h>
12
13/* Drag in local definitions */
14#include "librm.h"
15
16/* CR0: protection enabled */
17#define CR0_PE ( 1 << 0 )
18
19/* CR0: paging */
20#define CR0_PG ( 1 << 31 )
21
22/* CR4: physical address extensions */
23#define CR4_PAE ( 1 << 5 )
24
25/* Extended feature enable MSR (EFER) */
26#define MSR_EFER 0xc0000080
27
28/* EFER: long mode enable */
29#define EFER_LME ( 1 << 8 )
30
31/* Page: present */
32#define PG_P 0x01
33
34/* Page: read/write */
35#define PG_RW 0x02
36
37/* Page: user/supervisor */
38#define PG_US 0x04
39
40/* Page: page size */
41#define PG_PS 0x80
42
43/* Size of various paging-related data structures */
44#define SIZEOF_PTE_LOG2 3
45#define SIZEOF_PTE ( 1 << SIZEOF_PTE_LOG2 )
46#define SIZEOF_PT_LOG2 12
47#define SIZEOF_PT ( 1 << SIZEOF_PT_LOG2 )
48#define SIZEOF_4KB_PAGE_LOG2 12
49#define SIZEOF_4KB_PAGE ( 1 << SIZEOF_4KB_PAGE_LOG2 )
50#define SIZEOF_2MB_PAGE_LOG2 21
51#define SIZEOF_2MB_PAGE ( 1 << SIZEOF_2MB_PAGE_LOG2 )
52#define SIZEOF_LOW_4GB_LOG2 32
53#define SIZEOF_LOW_4GB ( 1 << SIZEOF_LOW_4GB_LOG2 )
54
55/* Size of various C data structures */
56#define SIZEOF_I386_SEG_REGS	12
57#define SIZEOF_I386_REGS	32
58#define SIZEOF_REAL_MODE_REGS	( SIZEOF_I386_SEG_REGS + SIZEOF_I386_REGS )
59#define SIZEOF_I386_FLAGS	4
60#define SIZEOF_I386_ALL_REGS	( SIZEOF_REAL_MODE_REGS + SIZEOF_I386_FLAGS )
61#define SIZEOF_X86_64_REGS	128
62
63/* Size of an address */
64#ifdef __x86_64__
65#define SIZEOF_ADDR 8
66#else
67#define SIZEOF_ADDR 4
68#endif
69
70/* Default code size */
71#ifdef __x86_64__
72#define CODE_DEFAULT code64
73#else
74#define CODE_DEFAULT code32
75#endif
76
77/* Selectively assemble code for 32-bit/64-bit builds */
78#ifdef __x86_64__
79#define if32 if 0
80#define if64 if 1
81#else
82#define if32 if 1
83#define if64 if 0
84#endif
85
86/****************************************************************************
87 * Global descriptor table
88 *
89 * Call init_librm to set up the GDT before attempting to use any
90 * protected-mode code.
91 *
92 * NOTE: This must be located before prot_to_real, otherwise gas
93 * throws a "can't handle non absolute segment in `ljmp'" error due to
94 * not knowing the value of REAL_CS when the ljmp is encountered.
95 *
96 * Note also that putting ".word gdt_end - gdt - 1" directly into
97 * gdt_limit, rather than going via gdt_length, will also produce the
98 * "non absolute segment" error.  This is most probably a bug in gas.
99 ****************************************************************************
100 */
101	.section ".data16.gdt", "aw", @progbits
102	.align 16
103gdt:
104gdtr:		/* The first GDT entry is unused, the GDTR can fit here. */
105gdt_limit:		.word gdt_length - 1
106gdt_base:		.long 0
107			.word 0 /* padding */
108
109	.org	gdt + VIRTUAL_CS, 0
110virtual_cs:	/* 32 bit protected mode code segment, virtual addresses */
111	.word	0xffff, 0
112	.byte	0, 0x9f, 0xcf, 0
113
114	.org	gdt + VIRTUAL_DS, 0
115virtual_ds:	/* 32 bit protected mode data segment, virtual addresses */
116	.word	0xffff, 0
117	.byte	0, 0x93, 0xcf, 0
118
119	.org	gdt + PHYSICAL_CS, 0
120physical_cs:	/* 32 bit protected mode code segment, physical addresses */
121	.word	0xffff, 0
122	.byte	0, 0x9f, 0xcf, 0
123
124	.org	gdt + PHYSICAL_DS, 0
125physical_ds:	/* 32 bit protected mode data segment, physical addresses */
126	.word	0xffff, 0
127	.byte	0, 0x93, 0xcf, 0
128
129	.org	gdt + REAL_CS, 0
130real_cs: 	/* 16 bit real mode code segment */
131	.word	0xffff, 0
132	.byte	0, 0x9b, 0x00, 0
133
134	.org	gdt + REAL_DS, 0
135real_ds:	/* 16 bit real mode data segment */
136	.word	0xffff, 0
137	.byte	0, 0x93, 0x00, 0
138
139	.org	gdt + P2R_DS, 0
140p2r_ds:		/* 16 bit real mode data segment for prot_to_real transition */
141	.word	0xffff, ( P2R_DS << 4 )
142	.byte	0, 0x93, 0x00, 0
143
144	.org	gdt + LONG_CS, 0
145long_cs:	/* 64 bit long mode code segment */
146	.word	0, 0
147	.byte	0, 0x9a, 0x20, 0
148
149gdt_end:
150	.equ	gdt_length, gdt_end - gdt
151
152/****************************************************************************
153 * Stored real-mode and protected-mode stack pointers
154 *
155 * The real-mode stack pointer is stored here whenever real_to_prot
156 * is called and restored whenever prot_to_real is called.  The
157 * converse happens for the protected-mode stack pointer.
158 *
159 * Despite initial appearances this scheme is, in fact re-entrant,
160 * because program flow dictates that we always return via the point
161 * we left by.  For example:
162 *    PXE API call entry
163 *  1   real => prot
164 *        ...
165 *        Print a text string
166 *	    ...
167 *  2       prot => real
168 *            INT 10
169 *  3       real => prot
170 *	    ...
171 *        ...
172 *  4   prot => real
173 *    PXE API call exit
174 *
175 * At point 1, the RM mode stack value, say RPXE, is stored in
176 * rm_ss,sp.  We want this value to still be present in rm_ss,sp when
177 * we reach point 4.
178 *
179 * At point 2, the RM stack value is restored from RPXE.  At point 3,
180 * the RM stack value is again stored in rm_ss,sp.  This *does*
181 * overwrite the RPXE that we have stored there, but it's the same
182 * value, since the code between points 2 and 3 has managed to return
183 * to us.
184 ****************************************************************************
185 */
186	.section ".bss.rm_ss_sp", "aw", @nobits
187	.globl rm_sp
188rm_sp:	.word 0
189	.globl rm_ss
190rm_ss:	.word 0
191
192	.section ".data.pm_esp", "aw", @progbits
193pm_esp:	.long VIRTUAL(_estack)
194
195/****************************************************************************
196 * Temporary static data buffer
197 *
198 * This is used to reduce the amount of real-mode stack space consumed
199 * during mode transitions, since we are sometimes called with very
200 * little real-mode stack space available.
201 ****************************************************************************
202 */
203	/* Temporary static buffer usage by virt_call */
204	.struct	0
205VC_TMP_GDT:		.space	6
206VC_TMP_IDT:		.space	6
207VC_TMP_PAD:		.space	4 /* for alignment */
208.if64
209VC_TMP_CR3:		.space	4
210VC_TMP_CR4:		.space	4
211VC_TMP_EMER:		.space	8
212.endif
213#ifdef TIVOLI_VMM_WORKAROUND
214VC_TMP_FXSAVE:		.space	512
215#endif
216VC_TMP_END:
217	.previous
218
219	/* Temporary static buffer usage by real_call */
220	.struct 0
221RC_TMP_FUNCTION:	.space	4
222RC_TMP_END:
223	.previous
224
225	/* Shared temporary static buffer */
226	.section ".bss16.rm_tmpbuf", "aw", @nobits
227	.align 16
228rm_tmpbuf:
229	.space	VC_TMP_END
230	.size	rm_tmpbuf, . - rm_tmpbuf
231
232/****************************************************************************
233 * Virtual address offsets
234 *
235 * These are used by the protected-mode code to map between virtual
236 * and physical addresses, and to access variables in the .text16 or
237 * .data16 segments.
238 ****************************************************************************
239 */
240	.struct 0
241VA_VIRT_OFFSET:	.space	SIZEOF_ADDR
242VA_TEXT16:	.space	SIZEOF_ADDR
243VA_DATA16:	.space	SIZEOF_ADDR
244VA_SIZE:
245	.previous
246
247	/* Internal copies, used only by librm itself */
248	.section ".bss16.rm_virt_addrs", "aw", @nobits
249rm_virt_addrs:	.space	VA_SIZE
250	.equ	rm_virt_offset, ( rm_virt_addrs + VA_VIRT_OFFSET )
251	.equ	rm_text16, ( rm_virt_addrs + VA_TEXT16 )
252	.equ	rm_data16, ( rm_virt_addrs + VA_DATA16 )
253
254	/* Externally visible variables, used by C code */
255	.section ".bss.virt_addrs", "aw", @nobits
256virt_addrs:	.space	VA_SIZE
257	.globl	virt_offset
258	.equ	virt_offset, ( virt_addrs + VA_VIRT_OFFSET )
259	.globl	text16
260	.equ	text16, ( virt_addrs + VA_TEXT16 )
261	.globl	data16
262	.equ	data16, ( virt_addrs + VA_DATA16 )
263
264/****************************************************************************
265 * init_librm (real-mode far call, 16-bit real-mode far return address)
266 *
267 * Initialise the GDT ready for transitions to protected mode.
268 *
269 * Parameters:
270 *   %cs : .text16 segment
271 *   %ds : .data16 segment
272 *   %edi : Physical base of protected-mode code
273 ****************************************************************************
274 */
275	.section ".text16.init_librm", "ax", @progbits
276	.code16
277	.globl init_librm
278init_librm:
279	/* Preserve registers */
280	pushl	%eax
281	pushl	%ebx
282	pushl	%edi
283
284	/* Store rm_virt_offset and set up virtual_cs and virtual_ds segments */
285	subl	$VIRTUAL(_textdata), %edi
286	movl	%edi, rm_virt_offset
287.if64 ;	setae	(rm_virt_offset+4) ; .endif
288	movl	%edi, %eax
289	movw	$virtual_cs, %bx
290	call	set_seg_base
291	movw	$virtual_ds, %bx
292	call	set_seg_base
293
294	/* Store rm_cs and rm_text16, set up real_cs segment */
295	xorl	%eax, %eax
296	movw	%cs, %ax
297	movw	%ax, %cs:rm_cs
298	shll	$4, %eax
299	movw	$real_cs, %bx
300	call	set_seg_base
301.if32 ;	subl	%edi, %eax ; .endif
302	movl	%eax, rm_text16
303
304	/* Store rm_ds and rm_data16, set up real_ds segment and GDT base */
305	xorl	%eax, %eax
306	movw	%ds, %ax
307	movw	%ax, %cs:rm_ds
308	shll	$4, %eax
309	movw	$real_ds, %bx
310	call	set_seg_base
311	movl	%eax, gdt_base
312	addl	$gdt, gdt_base
313.if32 ;	subl	%edi, %eax ; .endif
314	movl	%eax, rm_data16
315
316	/* Configure virt_call for protected mode, if applicable */
317.if64 ;	movl	$VIRTUAL(vc_pmode), %cs:vc_jmp_offset ; .endif
318
319	/* Switch to protected mode */
320	virtcall init_librm_pmode
321	.section ".text.init_librm", "ax", @progbits
322	.code32
323init_librm_pmode:
324
325	/* Store virt_offset, text16, and data16 */
326	pushw	%ds
327	movw	$REAL_DS, %ax
328	movw	%ax, %ds
329	movl	$rm_virt_addrs, %esi
330	movl	$VIRTUAL(virt_addrs), %edi
331	movl	$( VA_SIZE / 4 ), %ecx
332	rep movsl
333	popw	%ds
334
335.if64 ;	/* Initialise long mode, if applicable */
336	movl	VIRTUAL(virt_offset), %edi
337	leal	VIRTUAL(p2l_ljmp_target)(%edi), %eax
338	movl	%eax, VIRTUAL(p2l_ljmp_offset)
339	call	init_pages
340.endif
341	/* Return to real mode */
342	ret
343	.section ".text16.init_librm", "ax", @progbits
344	.code16
345init_librm_rmode:
346
347	/* Configure virt_call for long mode, if applicable */
348.if64 ;	movl	$VIRTUAL(vc_lmode), %cs:vc_jmp_offset ; .endif
349
350	/* Initialise IDT */
351	virtcall init_idt
352
353	/* Restore registers */
354	popl	%edi
355	popl	%ebx
356	popl	%eax
357	lret
358
359	.section ".text16.set_seg_base", "ax", @progbits
360	.code16
361set_seg_base:
3621:	movw	%ax, 2(%bx)
363	rorl	$16, %eax
364	movb	%al, 4(%bx)
365	movb	%ah, 7(%bx)
366	roll	$16, %eax
367	ret
368
369/****************************************************************************
370 * real_to_prot (real-mode near call, 32-bit virtual return address)
371 *
372 * Switch from 16-bit real-mode to 32-bit protected mode with virtual
373 * addresses.  The real-mode %ss:sp is stored in rm_ss and rm_sp, and
374 * the protected-mode %esp is restored from the saved pm_esp.
375 * Interrupts are disabled.  All other registers may be destroyed.
376 *
377 * The return address for this function should be a 32-bit virtual
378 * address.
379 *
380 * Parameters:
381 *   %ecx : number of bytes to move from RM stack to PM stack
382 *   %edx : number of bytes to copy from RM temporary buffer to PM stack
383 *
384 ****************************************************************************
385 */
386	.section ".text16.real_to_prot", "ax", @progbits
387	.code16
388real_to_prot:
389	/* Enable A20 line */
390	call	enable_a20
391	/* A failure at this point is fatal, and there's nothing we
392	 * can do about it other than lock the machine to make the
393	 * problem immediately visible.
394	 */
3951:	jc	1b
396
397	/* Make sure we have our data segment available */
398	movw	%cs:rm_ds, %ds
399
400	/* Add protected-mode return address to length of data to be copied */
401	addw	$4, %cx /* %ecx must be less than 64kB anyway */
402
403	/* Real-mode %ss:%sp => %ebp and virtual address => %esi */
404	xorl	%eax, %eax
405	movw	%ss, %ax
406	shll	$4, %eax
407	movzwl	%sp, %ebp
408	addr32 leal (%eax,%ebp), %esi
409	subl	rm_virt_offset, %esi
410	shll	$12, %eax
411	orl	%eax, %ebp
412
413	/* Real-mode data segment virtual address => %ebx */
414	movl	rm_data16, %ebx
415.if64 ; subl	rm_virt_offset, %ebx ; .endif
416
417	/* Load protected-mode global descriptor table */
418	data32 lgdt gdtr
419
420	/* Zero segment registers.  This wastes around 12 cycles on
421	 * real hardware, but saves a substantial number of emulated
422	 * instructions under KVM.
423	 */
424	xorw	%ax, %ax
425	movw	%ax, %ds
426	movw	%ax, %es
427	movw	%ax, %fs
428	movw	%ax, %gs
429	movw	%ax, %ss
430
431	/* Switch to protected mode (with paging disabled if applicable) */
432	cli
433	movl	%cr0, %eax
434.if64 ;	andl	$~CR0_PG, %eax ; .endif
435	orb	$CR0_PE, %al
436	movl	%eax, %cr0
437	data32 ljmp	$VIRTUAL_CS, $VIRTUAL(r2p_pmode)
438	.section ".text.real_to_prot", "ax", @progbits
439	.code32
440r2p_pmode:
441	/* Set up protected-mode data segments and stack pointer */
442	movw	$VIRTUAL_DS, %ax
443	movw	%ax, %ds
444	movw	%ax, %es
445	movw	%ax, %fs
446	movw	%ax, %gs
447	movw	%ax, %ss
448	movl	VIRTUAL(pm_esp), %esp
449
450	/* Load protected-mode interrupt descriptor table */
451	lidt	VIRTUAL(idtr32)
452
453	/* Record real-mode %ss:sp (after removal of data) */
454	addl	%ecx, %ebp
455	movl	%ebp, VIRTUAL(rm_sp)
456
457	/* Move data from RM stack to PM stack */
458	subl	%edx, %esp
459	subl	%ecx, %esp
460	movl	%esp, %edi
461	rep movsb
462
463	/* Copy data from RM temporary buffer to PM stack */
464	leal	rm_tmpbuf(%ebx), %esi
465	movl	%edx, %ecx
466	rep movsb
467
468	/* Return to virtual address */
469	ret
470
471/****************************************************************************
472 * prot_to_real (protected-mode near call, 32-bit real-mode return address)
473 *
474 * Switch from 32-bit protected mode with virtual addresses to 16-bit
475 * real mode.  The protected-mode %esp is stored in pm_esp and the
476 * real-mode %ss:sp is restored from the saved rm_ss and rm_sp.  The
477 * high word of the real-mode %esp is set to zero.  All real-mode data
478 * segment registers are loaded from the saved rm_ds.  Interrupts are
479 * *not* enabled, since we want to be able to use prot_to_real in an
480 * ISR.  All other registers may be destroyed.
481 *
482 * The return address for this function should be a 32-bit (sic)
483 * real-mode offset within .code16.
484 *
485 * Parameters:
486 *   %ecx : number of bytes to move from PM stack to RM stack
487 *   %edx : number of bytes to move from PM stack to RM temporary buffer
488 *   %esi : real-mode global and interrupt descriptor table registers
489 *
490 ****************************************************************************
491 */
492	.section ".text.prot_to_real", "ax", @progbits
493	.code32
494prot_to_real:
495	/* Copy real-mode global descriptor table register to RM code segment */
496	movl	VIRTUAL(text16), %edi
497.if64 ;	subl	VIRTUAL(virt_offset), %edi ; .endif
498	leal	rm_gdtr(%edi), %edi
499	movsw
500	movsl
501
502	/* Load real-mode interrupt descriptor table register */
503	lidt	(%esi)
504
505	/* Add return address to data to be moved to RM stack */
506	addl	$4, %ecx
507
508	/* Real-mode %ss:sp => %ebp and virtual address => %edi */
509	movl	VIRTUAL(rm_sp), %ebp
510	subl	%ecx, %ebp
511	movzwl	VIRTUAL(rm_ss), %eax
512	shll	$4, %eax
513	movzwl	%bp, %edi
514	addl	%eax, %edi
515	subl	VIRTUAL(virt_offset), %edi
516
517	/* Move data from PM stack to RM stack */
518	movl	%esp, %esi
519	rep movsb
520
521	/* Move data from PM stack to RM temporary buffer */
522	movl	VIRTUAL(data16), %edi
523.if64 ;	subl	VIRTUAL(virt_offset), %edi ; .endif
524	addl	$rm_tmpbuf, %edi
525	movl	%edx, %ecx
526	rep movsb
527
528	/* Record protected-mode %esp (after removal of data) */
529	movl	%esi, VIRTUAL(pm_esp)
530
531	/* Load real-mode segment limits */
532	movw	$P2R_DS, %ax
533	movw	%ax, %ds
534	movw	%ax, %es
535	movw	%ax, %fs
536	movw	%ax, %gs
537	movw	%ax, %ss
538	ljmp	$REAL_CS, $p2r_rmode
539	.section ".text16.prot_to_real", "ax", @progbits
540	.code16
541p2r_rmode:
542	/* Load real-mode GDT */
543	data32 lgdt %cs:rm_gdtr
544	/* Switch to real mode */
545	movl	%cr0, %eax
546	andb	$0!CR0_PE, %al
547	movl	%eax, %cr0
548p2r_ljmp_rm_cs:
549	ljmp	$0, $1f
5501:
551	/* Set up real-mode data segments and stack pointer */
552	movw	%cs:rm_ds, %ax
553	movw	%ax, %ds
554	movw	%ax, %es
555	movw	%ax, %fs
556	movw	%ax, %gs
557	movl	%ebp, %eax
558	shrl	$16, %eax
559	movw	%ax, %ss
560	movzwl	%bp, %esp
561
562	/* Return to real-mode address */
563	data32 ret
564
565
566	/* Real-mode code and data segments.  Assigned by the call to
567	 * init_librm.  rm_cs doubles as the segment part of the jump
568	 * instruction used by prot_to_real.  Both are located in
569	 * .text16 rather than .data16: rm_cs since it forms part of
570	 * the jump instruction within the code segment, and rm_ds
571	 * since real-mode code needs to be able to locate the data
572	 * segment with no other reference available.
573	 */
574	.globl rm_cs
575	.equ	rm_cs, ( p2r_ljmp_rm_cs + 3 )
576
577	.section ".text16.data.rm_ds", "aw", @progbits
578	.globl rm_ds
579rm_ds:	.word 0
580
581	/* Real-mode global and interrupt descriptor table registers */
582	.section ".text16.data.rm_gdtr", "aw", @progbits
583rm_gdtr:
584	.word 0 /* Limit */
585	.long 0 /* Base */
586
587/****************************************************************************
588 * phys_to_prot (protected-mode near call, 32-bit physical return address)
589 *
590 * Switch from 32-bit protected mode with physical addresses to 32-bit
591 * protected mode with virtual addresses.  %esp is adjusted to a
592 * virtual address.  All other registers are preserved.
593 *
594 * The return address for this function should be a 32-bit physical
595 * (sic) address.
596 *
597 ****************************************************************************
598 */
599	.section ".text.phys_to_prot", "ax", @progbits
600	.code32
601	.globl phys_to_prot
602phys_to_prot:
603	/* Preserve registers */
604	pushl	%eax
605	pushl	%ebp
606
607	/* Switch to virtual code segment */
608	cli
609	ljmp	$VIRTUAL_CS, $VIRTUAL(1f)
6101:
611	/* Switch to virtual data segment and adjust %esp */
612	movw	$VIRTUAL_DS, %ax
613	movw	%ax, %ds
614	movw	%ax, %es
615	movw	%ax, %fs
616	movw	%ax, %gs
617	movw	%ax, %ss
618	movl	VIRTUAL(virt_offset), %ebp
619	subl	%ebp, %esp
620
621	/* Adjust return address to a virtual address */
622	subl	%ebp, 8(%esp)
623
624	/* Restore registers and return */
625	popl	%ebp
626	popl	%eax
627	ret
628
629.if32	/* Expose as _phys_to_virt for use by COMBOOT, if applicable */
630	.globl	_phys_to_virt
631	.equ	_phys_to_virt, phys_to_prot
632.endif
633
634/****************************************************************************
635 * prot_to_phys (protected-mode near call, 32-bit virtual return address)
636 *
637 * Switch from 32-bit protected mode with virtual addresses to 32-bit
638 * protected mode with physical addresses.  %esp is adjusted to a
639 * physical address.  All other registers are preserved.
640 *
641 * The return address for this function should be a 32-bit virtual
642 * (sic) address.
643 *
644 ****************************************************************************
645 */
646	.section ".text.prot_to_phys", "ax", @progbits
647	.code32
648prot_to_phys:
649	/* Preserve registers */
650	pushl	%eax
651	pushl	%ebp
652
653	/* Adjust return address to a physical address */
654	movl	VIRTUAL(virt_offset), %ebp
655	addl	%ebp, 8(%esp)
656
657	/* Switch to physical code segment */
658	cli
659	pushl	$PHYSICAL_CS
660	leal	VIRTUAL(1f)(%ebp), %eax
661	pushl	%eax
662	lret
6631:
664	/* Switch to physical data segment and adjust %esp */
665	movw	$PHYSICAL_DS, %ax
666	movw	%ax, %ds
667	movw	%ax, %es
668	movw	%ax, %fs
669	movw	%ax, %gs
670	movw	%ax, %ss
671	addl	%ebp, %esp
672
673	/* Restore registers and return */
674	popl	%ebp
675	popl	%eax
676	ret
677
678.if32	/* Expose as _virt_to_phys for use by COMBOOT, if applicable */
679	.globl	_virt_to_phys
680	.equ	_virt_to_phys, prot_to_phys
681.endif
682
683/****************************************************************************
684 * intr_to_prot (protected-mode near call, 32-bit virtual return address)
685 *
686 * Switch from 32-bit protected mode with a virtual code segment and
687 * either a physical or virtual stack segment to 32-bit protected mode
688 * with normal virtual addresses.  %esp is adjusted if necessary to a
689 * virtual address.  All other registers are preserved.
690 *
691 * The return address for this function should be a 32-bit virtual
692 * address.
693 *
694 ****************************************************************************
695 */
696	.section ".text.intr_to_prot", "ax", @progbits
697	.code32
698	.globl intr_to_prot
699intr_to_prot:
700	/* Preserve registers */
701	pushl	%eax
702
703	/* Check whether stack segment is physical or virtual */
704	movw	%ss, %ax
705	cmpw	$VIRTUAL_DS, %ax
706	movw	$VIRTUAL_DS, %ax
707
708	/* Reload data segment registers */
709	movw	%ax, %ds
710	movw	%ax, %es
711	movw	%ax, %fs
712	movw	%ax, %gs
713
714	/* Reload stack segment and adjust %esp if necessary */
715	je	1f
716	movw	%ax, %ss
717	subl	VIRTUAL(virt_offset), %esp
7181:
719	/* Restore registers and return */
720	popl	%eax
721	ret
722
723	/* Expose as _intr_to_virt for use by GDB */
724	.globl	_intr_to_virt
725	.equ	_intr_to_virt, intr_to_prot
726
727/****************************************************************************
728 * prot_to_long (protected-mode near call, 32-bit virtual return address)
729 *
730 * Switch from 32-bit protected mode with virtual addresses to 64-bit
731 * long mode.  The protected-mode %esp is adjusted to a physical
732 * address.  All other registers are preserved.
733 *
734 * The return address for this function should be a 32-bit (sic)
735 * virtual address.
736 *
737 ****************************************************************************
738 */
739	.if64
740
741	.section ".text.prot_to_long", "ax", @progbits
742	.code32
743prot_to_long:
744	/* Preserve registers */
745	pushl	%eax
746	pushl	%ecx
747	pushl	%edx
748
749	/* Set up PML4 */
750	movl	VIRTUAL(pml4), %eax
751	movl	%eax, %cr3
752
753	/* Enable PAE */
754	movl	%cr4, %eax
755	orb	$CR4_PAE, %al
756	movl	%eax, %cr4
757
758	/* Enable long mode */
759	movl	$MSR_EFER, %ecx
760	rdmsr
761	orw	$EFER_LME, %ax
762	wrmsr
763
764	/* Enable paging */
765	movl	%cr0, %eax
766	orl	$CR0_PG, %eax
767	movl	%eax, %cr0
768
769	/* Restore registers */
770	popl	%edx
771	popl	%ecx
772	popl	%eax
773
774	/* Construct 64-bit return address */
775	pushl	(%esp)
776	movl	$0xffffffff, 4(%esp)
777p2l_ljmp:
778	/* Switch to long mode (using a physical %rip) */
779	ljmp	$LONG_CS, $0
780	.code64
781p2l_lmode:
782	/* Adjust and zero-extend %esp to a physical address */
783	addl	virt_offset, %esp
784
785	/* Use long-mode IDT */
786	lidt	idtr64
787
788	/* Return to virtual address */
789	ret
790
791	/* Long mode jump offset and target.  Required since an ljmp
792	 * in protected mode will zero-extend the offset, and so
793	 * cannot reach an address within the negative 2GB as used by
794	 * -mcmodel=kernel.  Assigned by the call to init_librm.
795	 */
796	.equ	p2l_ljmp_offset, ( p2l_ljmp + 1 )
797	.equ	p2l_ljmp_target, p2l_lmode
798
799	.endif
800
801/****************************************************************************
802 * long_to_prot (long-mode near call, 64-bit virtual return address)
803 *
804 * Switch from 64-bit long mode to 32-bit protected mode with virtual
805 * addresses.  The long-mode %rsp is adjusted to a virtual address.
806 * All other registers are preserved.
807 *
808 * The return address for this function should be a 64-bit (sic)
809 * virtual address.
810 *
811 ****************************************************************************
812 */
813	.if64
814
815	.section ".text.long_to_prot", "ax", @progbits
816	.code64
817long_to_prot:
818	/* Switch to protected mode */
819	ljmp	*l2p_vector
820	.code32
821l2p_pmode:
822	/* Adjust %esp to a virtual address */
823	subl	VIRTUAL(virt_offset), %esp
824
825	/* Preserve registers */
826	pushl	%eax
827	pushl	%ecx
828	pushl	%edx
829
830	/* Disable paging */
831	movl	%cr0, %eax
832	andl	$~CR0_PG, %eax
833	movl	%eax, %cr0
834
835	/* Disable PAE (in case external non-PAE-aware code enables paging) */
836	movl	%cr4, %eax
837	andb	$~CR4_PAE, %al
838	movl	%eax, %cr4
839
840	/* Disable long mode */
841	movl	$MSR_EFER, %ecx
842	rdmsr
843	andw	$~EFER_LME, %ax
844	wrmsr
845
846	/* Restore registers */
847	popl	%edx
848	popl	%ecx
849	popl	%eax
850
851	/* Use protected-mode IDT */
852	lidt	VIRTUAL(idtr32)
853
854	/* Return */
855	ret	$4
856
857	/* Long mode jump vector.  Required since there is no "ljmp
858	 * immediate" instruction in long mode.
859	 */
860	.section ".data.l2p_vector", "aw", @progbits
861l2p_vector:
862	.long	VIRTUAL(l2p_pmode), VIRTUAL_CS
863
864	.endif
865
866/****************************************************************************
867 * long_save_regs (long-mode near call, 64-bit virtual return address)
868 *
869 * Preserve registers that are accessible only in long mode.  This
870 * includes %r8-%r15 and the upper halves of %rax, %rbx, %rcx, %rdx,
871 * %rsi, %rdi, and %rbp.
872 *
873 ****************************************************************************
874 */
875	.if64
876
877	.section ".text.long_preserve_regs", "ax", @progbits
878	.code64
879long_preserve_regs:
880	/* Preserve registers */
881	pushq	%rax
882	pushq	%rcx
883	pushq	%rdx
884	pushq	%rbx
885	pushq	%rsp
886	pushq	%rbp
887	pushq	%rsi
888	pushq	%rdi
889	pushq	%r8
890	pushq	%r9
891	pushq	%r10
892	pushq	%r11
893	pushq	%r12
894	pushq	%r13
895	pushq	%r14
896	pushq	%r15
897
898	/* Return */
899	jmp	*SIZEOF_X86_64_REGS(%rsp)
900
901	.endif
902
903/****************************************************************************
904 * long_restore_regs (long-mode near call, 64-bit virtual return address)
905 *
906 * Restore registers that are accessible only in long mode.  This
907 * includes %r8-%r15 and the upper halves of %rax, %rbx, %rcx, %rdx,
908 * %rsi, %rdi, and %rbp.
909 *
910 ****************************************************************************
911 */
912	.if64
913
914	.section ".text.long_restore_regs", "ax", @progbits
915	.code64
916long_restore_regs:
917	/* Move return address above register dump */
918	popq	SIZEOF_X86_64_REGS(%rsp)
919
920	/* Restore registers */
921	popq	%r15
922	popq	%r14
923	popq	%r13
924	popq	%r12
925	popq	%r11
926	popq	%r10
927	popq	%r9
928	popq	%r8
929	movl	%edi, (%rsp)
930	popq	%rdi
931	movl	%esi, (%rsp)
932	popq	%rsi
933	movl	%ebp, (%rsp)
934	popq	%rbp
935	leaq	8(%rsp), %rsp /* discard */
936	movl	%ebx, (%rsp)
937	popq	%rbx
938	movl	%edx, (%rsp)
939	popq	%rdx
940	movl	%ecx, (%rsp)
941	popq	%rcx
942	movl	%eax, (%rsp)
943	popq	%rax
944
945	/* Return */
946	ret
947
948	.endif
949
950/****************************************************************************
951 * virt_call (real-mode near call, 16-bit real-mode near return address)
952 *
953 * Call a specific C function in 32-bit protected mode or 64-bit long
954 * mode (as applicable).  The prototype of the C function must be
955 *   void function ( struct i386_all_regs *ix86 );
956 * ix86 will point to a struct containing the real-mode registers
957 * at entry to virt_call().
958 *
959 * All registers will be preserved across virt_call(), unless the C
960 * function explicitly overwrites values in ix86.  Interrupt status
961 * and GDT will also be preserved.  Gate A20 will be enabled.
962 *
963 * Note that virt_call() does not rely on the real-mode stack
964 * remaining intact in order to return, since everything relevant is
965 * copied to the protected-mode stack for the duration of the call.
966 * In particular, this means that a real-mode prefix can make a call
967 * to main() which will return correctly even if the prefix's stack
968 * gets vapourised during the Etherboot run.  (The prefix cannot rely
969 * on anything else on the stack being preserved, so should move any
970 * critical data to registers before calling main()).
971 *
972 * Parameters:
973 *   function : 32-bit virtual address of function to call
974 *
975 * Example usage:
976 *	pushl	$pxe_api_call
977 *	call	virt_call
978 * to call in to the C function
979 *      void pxe_api_call ( struct i386_all_regs *ix86 );
980 ****************************************************************************
981 */
982	.struct	0
983VC_OFFSET_IX86:		.space	SIZEOF_I386_ALL_REGS
984VC_OFFSET_PADDING:	.space	2 /* for alignment */
985VC_OFFSET_RETADDR:	.space	2
986VC_OFFSET_PARAMS:
987VC_OFFSET_FUNCTION:	.space	4
988VC_OFFSET_END:
989	.previous
990
991	.section ".text16.virt_call", "ax", @progbits
992	.code16
993	.globl virt_call
994virt_call:
995	/* Preserve registers and flags on external RM stack */
996	pushw	%ss /* padding */
997	pushfl
998	pushal
999	pushw	%gs
1000	pushw	%fs
1001	pushw	%es
1002	pushw	%ds
1003	pushw	%ss
1004	pushw	%cs
1005
1006	/* Claim ownership of temporary static buffer */
1007	cli
1008	movw	%cs:rm_ds, %ds
1009
1010#ifdef TIVOLI_VMM_WORKAROUND
1011	/* Preserve FPU, MMX and SSE state in temporary static buffer */
1012	fxsave	( rm_tmpbuf + VC_TMP_FXSAVE )
1013#endif
1014	/* Preserve GDT and IDT in temporary static buffer */
1015	sidt	( rm_tmpbuf + VC_TMP_IDT )
1016	sgdt	( rm_tmpbuf + VC_TMP_GDT )
1017
1018.if64 ;	/* Preserve control registers, if applicable */
1019	movl	$MSR_EFER, %ecx
1020	rdmsr
1021	movl	%eax, ( rm_tmpbuf + VC_TMP_EMER + 0 )
1022	movl	%edx, ( rm_tmpbuf + VC_TMP_EMER + 4 )
1023	movl	%cr4, %eax
1024	movl	%eax, ( rm_tmpbuf + VC_TMP_CR4 )
1025	movl	%cr3, %eax
1026	movl	%eax, ( rm_tmpbuf + VC_TMP_CR3 )
1027.endif
1028	/* For sanity's sake, clear the direction flag as soon as possible */
1029	cld
1030
1031	/* Switch to protected mode and move register dump to PM stack */
1032	movl	$VC_OFFSET_END, %ecx
1033	movl	$VC_TMP_END, %edx
1034	pushl	$VIRTUAL(vc_pmode)
1035vc_jmp:	jmp	real_to_prot
1036	.section ".text.virt_call", "ax", @progbits
1037	.code32
1038vc_pmode:
1039	/* Call function (in protected mode) */
1040	pushl	%esp
1041	call	*(VC_OFFSET_FUNCTION+4)(%esp)
1042	popl	%eax /* discard */
1043
1044.if64 ; /* Switch to long mode */
1045	jmp	1f
1046vc_lmode:
1047	call	prot_to_long
1048	.code64
1049
1050	/* Call function (in long mode) */
1051	movq	%rsp, %rdi
1052	movslq	VC_OFFSET_FUNCTION(%rsp), %rax
1053	callq	*%rax
1054
1055	/* Switch to protected mode */
1056	call	long_to_prot
10571:	.code32
1058.endif
1059	/* Switch to real mode and move register dump back to RM stack */
1060	movl	$VC_OFFSET_END, %ecx
1061	movl	$VC_TMP_END, %edx
1062	leal	VC_TMP_GDT(%esp, %ecx), %esi
1063	pushl	$vc_rmode
1064	jmp	prot_to_real
1065	.section ".text16.virt_call", "ax", @progbits
1066	.code16
1067vc_rmode:
1068.if64 ;	/* Restore control registers, if applicable */
1069	movw	%sp, %bp
1070	movl	( rm_tmpbuf + VC_TMP_CR3 ), %eax
1071	movl	%eax, %cr3
1072	movl	( rm_tmpbuf + VC_TMP_CR4 ), %eax
1073	movl	%eax, %cr4
1074	movl	( rm_tmpbuf + VC_TMP_EMER + 0 ), %eax
1075	movl	( rm_tmpbuf + VC_TMP_EMER + 4 ), %edx
1076	movl	$MSR_EFER, %ecx
1077	wrmsr
1078.endif
1079
1080#ifdef TIVOLI_VMM_WORKAROUND
1081	/* Restore FPU, MMX and SSE state from temporary static buffer */
1082	fxrstor	( rm_tmpbuf + VC_TMP_FXSAVE )
1083#endif
1084	/* Restore registers and flags and return */
1085	popl	%eax /* skip %cs and %ss */
1086	popw	%ds
1087	popw	%es
1088	popw	%fs
1089	popw	%gs
1090	popal
1091	/* popal skips %esp.  We therefore want to do "movl -20(%sp),
1092	 * %esp", but -20(%sp) is not a valid 80386 expression.
1093	 * Fortunately, prot_to_real() zeroes the high word of %esp, so
1094	 * we can just use -20(%esp) instead.
1095	 */
1096	addr32 movl -20(%esp), %esp
1097	popfl
1098	popw	%ss /* padding */
1099
1100	/* Return and discard function parameters */
1101	ret	$( VC_OFFSET_END - VC_OFFSET_PARAMS )
1102
1103
1104	/* Protected-mode jump target */
1105	.equ	vc_jmp_offset, ( vc_jmp - 4 )
1106
1107/****************************************************************************
1108 * real_call (protected-mode near call, 32-bit virtual return address)
1109 * real_call (long-mode near call, 64-bit virtual return address)
1110 *
1111 * Call a real-mode function from protected-mode or long-mode code.
1112 *
1113 * The non-segment register values will be passed directly to the
1114 * real-mode code.  The segment registers will be set as per
1115 * prot_to_real.  The non-segment register values set by the real-mode
1116 * function will be passed back to the protected-mode or long-mode
1117 * caller.  A result of this is that this routine cannot be called
1118 * directly from C code, since it clobbers registers that the C ABI
1119 * expects the callee to preserve.
1120 *
1121 * librm.h defines a convenient macro REAL_CODE() for using real_call.
1122 * See librm.h and realmode.h for details and examples.
1123 *
1124 * Parameters:
1125 *   function : offset within .text16 of real-mode function to call
1126 *
1127 * Returns: none
1128 ****************************************************************************
1129 */
1130	.struct	0
1131RC_OFFSET_REGS:		.space	SIZEOF_I386_REGS
1132RC_OFFSET_REGS_END:
1133RC_OFFSET_FUNCTION_COPY:.space	4
1134.if64
1135RC_OFFSET_LREGS:	.space	SIZEOF_X86_64_REGS
1136RC_OFFSET_LREG_RETADDR:	.space	SIZEOF_ADDR
1137.endif
1138RC_OFFSET_RETADDR:	.space	SIZEOF_ADDR
1139RC_OFFSET_PARAMS:
1140RC_OFFSET_FUNCTION:	.space	SIZEOF_ADDR
1141RC_OFFSET_END:
1142	.previous
1143
1144	.section ".text.real_call", "ax", @progbits
1145	.CODE_DEFAULT
1146	.globl real_call
1147real_call:
1148.if64 ;	/* Preserve registers and switch to protected mode, if applicable */
1149	call	long_preserve_regs
1150	call	long_to_prot
1151	.code32
1152.endif
1153	/* Create register dump and function pointer copy on PM stack */
1154	pushl	( RC_OFFSET_FUNCTION - RC_OFFSET_FUNCTION_COPY - 4 )(%esp)
1155	pushal
1156
1157	/* Switch to real mode and move register dump to RM stack  */
1158	movl	$RC_OFFSET_REGS_END, %ecx
1159	movl	$RC_TMP_END, %edx
1160	pushl	$rc_rmode
1161	movl	$VIRTUAL(rm_default_gdtr_idtr), %esi
1162	jmp	prot_to_real
1163	.section ".text16.real_call", "ax", @progbits
1164	.code16
1165rc_rmode:
1166	/* Call real-mode function */
1167	popal
1168	call	*( rm_tmpbuf + RC_TMP_FUNCTION )
1169	pushal
1170
1171	/* For sanity's sake, clear the direction flag as soon as possible */
1172	cld
1173
1174	/* Switch to protected mode and move register dump back to PM stack */
1175	movl	$RC_OFFSET_REGS_END, %ecx
1176	xorl	%edx, %edx
1177	pushl	$VIRTUAL(rc_pmode)
1178	jmp	real_to_prot
1179	.section ".text.real_call", "ax", @progbits
1180	.code32
1181rc_pmode:
1182	/* Restore registers */
1183	popal
1184
1185.if64 ; /* Switch to long mode and restore registers, if applicable */
1186	call	prot_to_long
1187	.code64
1188	call	long_restore_regs
1189.endif
1190	/* Return and discard function parameters */
1191	ret	$( RC_OFFSET_END - RC_OFFSET_PARAMS )
1192
1193
1194	/* Default real-mode global and interrupt descriptor table registers */
1195	.section ".data.rm_default_gdtr_idtr", "aw", @progbits
1196rm_default_gdtr_idtr:
1197	.word 0		/* Global descriptor table limit */
1198	.long 0		/* Global descriptor table base */
1199	.word 0x03ff	/* Interrupt descriptor table limit */
1200	.long 0		/* Interrupt descriptor table base */
1201
1202/****************************************************************************
1203 * phys_call (protected-mode near call, 32-bit virtual return address)
1204 * phys_call (long-mode near call, 64-bit virtual return address)
1205 *
1206 * Call a function with flat 32-bit physical addressing
1207 *
1208 * The non-segment register values will be passed directly to the
1209 * function.  The segment registers will be set for flat 32-bit
1210 * physical addressing.  The non-segment register values set by the
1211 * function will be passed back to the caller.
1212 *
1213 * librm.h defines a convenient macro PHYS_CODE() for using phys_call.
1214 *
1215 * Parameters:
1216 *   function : virtual (sic) address of function to call
1217 *
1218 ****************************************************************************
1219 */
1220	.struct 0
1221.if64
1222PHC_OFFSET_LREGS:	.space	SIZEOF_X86_64_REGS
1223PHC_OFFSET_LREG_RETADDR:.space	SIZEOF_ADDR
1224.endif
1225PHC_OFFSET_RETADDR:	.space	SIZEOF_ADDR
1226PHC_OFFSET_PARAMS:
1227PHC_OFFSET_FUNCTION:	.space	SIZEOF_ADDR
1228PHC_OFFSET_END:
1229	.previous
1230
1231	.section ".text.phys_call", "ax", @progbits
1232	.CODE_DEFAULT
1233	.globl phys_call
1234phys_call:
1235.if64 ;	/* Preserve registers and switch to protected mode, if applicable */
1236	call	long_preserve_regs
1237	call	long_to_prot
1238	.code32
1239.endif
1240	/* Adjust function pointer to a physical address */
1241	pushl	%ebp
1242	movl	VIRTUAL(virt_offset), %ebp
1243	addl	%ebp, ( PHC_OFFSET_FUNCTION + 4 /* saved %ebp */ )(%esp)
1244	popl	%ebp
1245
1246	/* Switch to physical addresses */
1247	call	prot_to_phys
1248
1249	/* Call function */
1250	call	*PHC_OFFSET_FUNCTION(%esp)
1251
1252	/* For sanity's sake, clear the direction flag as soon as possible */
1253	cld
1254
1255	/* Switch to virtual addresses */
1256	call	phys_to_prot
1257
1258.if64 ; /* Switch to long mode and restore registers, if applicable */
1259	call	prot_to_long
1260	.code64
1261	call	long_restore_regs
1262.endif
1263	/* Return and discard function parameters */
1264	ret	$( PHC_OFFSET_END - PHC_OFFSET_PARAMS )
1265
1266/****************************************************************************
1267 * phys_to_long (protected-mode near call, 32-bit physical return address)
1268 *
1269 * Used by COMBOOT.
1270 *
1271 ****************************************************************************
1272 */
1273	.if64
1274
1275	.section ".text.phys_to_long", "ax", @progbits
1276	.code32
1277phys_to_long:
1278
1279	/* Switch to virtual addresses */
1280	call	phys_to_prot
1281
1282	/* Convert to 32-bit virtual return address */
1283	pushl	%eax
1284	movl	VIRTUAL(virt_offset), %eax
1285	subl	%eax, 4(%esp)
1286	popl	%eax
1287
1288	/* Switch to long mode and return */
1289	jmp	prot_to_long
1290
1291	/* Expose as _phys_to_virt for use by COMBOOT */
1292	.globl  _phys_to_virt
1293	.equ    _phys_to_virt, phys_to_long
1294
1295	.endif
1296
1297/****************************************************************************
1298 * long_to_phys (long-mode near call, 64-bit virtual return address)
1299 *
1300 * Used by COMBOOT.
1301 *
1302 ****************************************************************************
1303 */
1304	.if64
1305
1306	.section ".text.long_to_phys", "ax", @progbits
1307	.code64
1308long_to_phys:
1309
1310	/* Switch to protected mode */
1311	call	long_to_prot
1312	.code32
1313
1314	/* Convert to 32-bit virtual return address */
1315	popl	(%esp)
1316
1317	/* Switch to physical addresses and return */
1318	jmp	prot_to_phys
1319
1320	/* Expose as _virt_to_phys for use by COMBOOT */
1321	.globl  _virt_to_phys
1322	.equ    _virt_to_phys, long_to_phys
1323
1324	.endif
1325
1326/****************************************************************************
1327 * flatten_real_mode (real-mode near call)
1328 *
1329 * Switch to flat real mode
1330 *
1331 ****************************************************************************
1332 */
1333	.section ".text16.flatten_real_mode", "ax", @progbits
1334	.code16
1335	.globl flatten_real_mode
1336flatten_real_mode:
1337	/* Modify GDT to use flat real mode */
1338	movb	$0x8f, real_cs + 6
1339	movb	$0x8f, real_ds + 6
1340	/* Call dummy protected-mode function */
1341	virtcall flatten_dummy
1342	/* Restore GDT */
1343	movb	$0x00, real_cs + 6
1344	movb	$0x00, real_ds + 6
1345	/* Return */
1346	ret
1347
1348	.section ".text.flatten_dummy", "ax", @progbits
1349	.CODE_DEFAULT
1350flatten_dummy:
1351	ret
1352
1353/****************************************************************************
1354 * Interrupt wrapper
1355 *
1356 * Used by the protected-mode and long-mode interrupt vectors to call
1357 * the interrupt() function.
1358 *
1359 * May be entered with either physical or virtual stack segment.
1360 ****************************************************************************
1361 */
1362	.section ".text.interrupt_wrapper", "ax", @progbits
1363	.code32
1364	.globl interrupt_wrapper
1365interrupt_wrapper:
1366	/* Preserve registers (excluding already-saved %eax) */
1367	pushl	%ebx
1368	pushl	%ecx
1369	pushl	%edx
1370	pushl	%esi
1371	pushl	%edi
1372	pushl	%ebp
1373
1374	/* Expand IRQ number to whole %eax register */
1375	movzbl	%al, %eax
1376
1377.if64 ; /* Skip transition to long mode, if applicable */
1378	xorl	%edx, %edx
1379	movw	%cs, %bx
1380	cmpw	$LONG_CS, %bx
1381	je	1f
1382.endif
1383	/* Preserve segment registers and original %esp */
1384	pushl	%ds
1385	pushl	%es
1386	pushl	%fs
1387	pushl	%gs
1388	pushl	%ss
1389	pushl	%esp
1390
1391	/* Switch to virtual addressing */
1392	call	intr_to_prot
1393
1394	/* Pass 32-bit interrupt frame pointer in %edx */
1395	movl	%esp, %edx
1396	xorl	%ecx, %ecx
1397.if64
1398	/* Switch to long mode */
1399	call	prot_to_long
1400	.code64
1401
14021:	/* Preserve long-mode registers */
1403	pushq	%r8
1404	pushq	%r9
1405	pushq	%r10
1406	pushq	%r11
1407	pushq	%r12
1408	pushq	%r13
1409	pushq	%r14
1410	pushq	%r15
1411
1412	/* Expand IRQ number to whole %rdi register */
1413	movl	%eax, %edi
1414
1415	/* Pass 32-bit interrupt frame pointer (if applicable) in %rsi */
1416	testl	%edx, %edx
1417	je	1f
1418	movl	%edx, %esi
1419	addl	virt_offset, %esi
14201:
1421	/* Pass 64-bit interrupt frame pointer in %rdx */
1422	movq	%rsp, %rdx
1423.endif
1424	/* Call interrupt handler */
1425	call	interrupt
1426.if64
1427	/* Restore long-mode registers */
1428	popq	%r15
1429	popq	%r14
1430	popq	%r13
1431	popq	%r12
1432	popq	%r11
1433	popq	%r10
1434	popq	%r9
1435	popq	%r8
1436
1437	/* Skip transition back to protected mode, if applicable */
1438	cmpw	$LONG_CS, %bx
1439	je	1f
1440
1441	/* Switch to protected mode */
1442	call	long_to_prot
1443	.code32
1444	cmpw	$LONG_CS, %bx
1445.endif
1446	/* Restore segment registers and original %esp */
1447	lss	(%esp), %esp
1448	popl	%ss
1449	popl	%gs
1450	popl	%fs
1451	popl	%es
1452	popl	%ds
1453
14541:	/* Restore registers */
1455	popl	%ebp
1456	popl	%edi
1457	popl	%esi
1458	popl	%edx
1459	popl	%ecx
1460	popl	%ebx
1461	popl	%eax
1462
1463	/* Return from interrupt (with REX prefix if required) */
1464.if64 ; jne 1f ; .byte 0x48 ; .endif
14651:	iret
1466
1467/****************************************************************************
1468 * Page tables
1469 *
1470 ****************************************************************************
1471 */
1472	.section ".pages", "aw", @nobits
1473	.align	SIZEOF_PT
1474
1475	/* Page map level 4 entries (PML4Es)
1476	 *
1477	 * This comprises
1478	 *
1479	 * - PML4E[0x000] covering [0x0000000000000000-0x0000007fffffffff]
1480	 * - PML4E[0x1ff] covering [0xffffff8000000000-0xffffffffffffffff]
1481	 *
1482	 * These point to the PDPT.  This creates some aliased
1483	 * addresses within unused portions of the 64-bit address
1484	 * space, but allows us to use just a single PDPT.
1485	 *
1486	 * - PDE[...] covering arbitrary 2MB portions of I/O space
1487	 *
1488	 * These are 2MB pages created by ioremap() to cover I/O
1489	 * device addresses.
1490	 */
1491pml4e:
1492	.space	SIZEOF_PT
1493	.size	pml4e, . - pml4e
1494
1495	.globl	io_pages
1496	.equ	io_pages, pml4e
1497
1498	/* Page directory pointer table entries (PDPTEs)
1499	 *
1500	 * This comprises:
1501	 *
1502	 * - PDPTE[0x000] covering [0x0000000000000000-0x000000003fffffff]
1503	 * - PDPTE[0x001] covering [0x0000000040000000-0x000000007fffffff]
1504	 * - PDPTE[0x002] covering [0x0000000080000000-0x00000000bfffffff]
1505	 * - PDPTE[0x003] covering [0x00000000c0000000-0x00000000ffffffff]
1506	 *
1507	 * These point to the appropriate page directories (in pde_low)
1508	 * used to identity-map the whole of the 32-bit address space.
1509	 *
1510	 * - PDPTE[0x004] covering [0x0000000100000000-0x000000013fffffff]
1511	 *
1512	 * This points back to the PML4, allowing the PML4 to be
1513	 * (ab)used to hold 2MB pages used for I/O device addresses.
1514	 *
1515	 * - PDPTE[0x1ff] covering [0xffffffffc0000000-0xffffffffffffffff]
1516	 *
1517	 * This points back to the PDPT itself, allowing the PDPT to be
1518	 * (ab)used to hold PDEs covering .textdata.
1519	 *
1520	 * - PDE[N-M] covering [_textdata,_end)
1521	 *
1522	 * These are used to point to the page tables (in pte_textdata)
1523	 * used to map our .textdata section.  Note that each PDE
1524	 * covers 2MB, so we are likely to use only a single PDE in
1525	 * practice.
1526	 */
1527pdpte:
1528	.space	SIZEOF_PT
1529	.size	pdpte, . - pdpte
1530	.equ	pde_textdata, pdpte /* (ab)use */
1531
1532	/* Page directory entries (PDEs) for the low 4GB
1533	 *
1534	 * This comprises 2048 2MB pages to identity-map the whole of
1535	 * the 32-bit address space.
1536	 */
1537pde_low:
1538	.equ	PDE_LOW_PTES, ( SIZEOF_LOW_4GB / SIZEOF_2MB_PAGE )
1539	.equ	PDE_LOW_PTS, ( ( PDE_LOW_PTES * SIZEOF_PTE ) / SIZEOF_PT )
1540	.space	( PDE_LOW_PTS * SIZEOF_PT )
1541	.size	pde_low, . - pde_low
1542
1543	/* Page table entries (PTEs) for .textdata
1544	 *
1545	 * This comprises enough 4kB pages to map the whole of
1546	 * .textdata.  The required number of PTEs is calculated by
1547	 * the linker script.
1548	 *
1549	 * Note that these mappings do not cover the PTEs themselves.
1550	 * This does not matter, since code running with paging
1551	 * enabled never needs to access these PTEs.
1552	 */
1553pte_textdata:
1554	/* Allocated by linker script; must be at the end of .textdata */
1555
1556	.section ".bss.pml4", "aw", @nobits
1557pml4:	.long	0
1558
1559/****************************************************************************
1560 * init_pages (protected-mode near call)
1561 *
1562 * Initialise the page tables ready for long mode.
1563 *
1564 * Parameters:
1565 *   %edi : virt_offset
1566 ****************************************************************************
1567 */
1568	.section ".text.init_pages", "ax", @progbits
1569	.code32
1570init_pages:
1571	/* Initialise PML4Es for low 4GB and negative 2GB */
1572	leal	( VIRTUAL(pdpte) + ( PG_P | PG_RW | PG_US ) )(%edi), %eax
1573	movl	%eax, VIRTUAL(pml4e)
1574	movl	%eax, ( VIRTUAL(pml4e) + SIZEOF_PT - SIZEOF_PTE )
1575
1576	/* Initialise PDPTE for negative 1GB */
1577	movl	%eax, ( VIRTUAL(pdpte) + SIZEOF_PT - SIZEOF_PTE )
1578
1579	/* Initialise PDPTE for I/O space */
1580	leal	( VIRTUAL(pml4e) + ( PG_P | PG_RW | PG_US ) )(%edi), %eax
1581	movl	%eax, ( VIRTUAL(pdpte) + ( PDE_LOW_PTS * SIZEOF_PTE ) )
1582
1583	/* Initialise PDPTEs for low 4GB */
1584	movl	$PDE_LOW_PTS, %ecx
1585	leal	( VIRTUAL(pde_low) + ( PDE_LOW_PTS * SIZEOF_PT ) + \
1586		  ( PG_P | PG_RW | PG_US ) )(%edi), %eax
15871:	subl	$SIZEOF_PT, %eax
1588	movl	%eax, ( VIRTUAL(pdpte) - SIZEOF_PTE )(,%ecx,SIZEOF_PTE)
1589	loop	1b
1590
1591	/* Initialise PDEs for low 4GB */
1592	movl	$PDE_LOW_PTES, %ecx
1593	leal	( 0 + ( PG_P | PG_RW | PG_US | PG_PS ) ), %eax
15941:	subl	$SIZEOF_2MB_PAGE, %eax
1595	movl	%eax, ( VIRTUAL(pde_low) - SIZEOF_PTE )(,%ecx,SIZEOF_PTE)
1596	loop	1b
1597
1598	/* Initialise PDEs for .textdata */
1599	movl	$_textdata_pdes, %ecx
1600	leal	( VIRTUAL(_etextdata) + ( PG_P | PG_RW | PG_US ) )(%edi), %eax
1601	movl	$VIRTUAL(_textdata), %ebx
1602	shrl	$( SIZEOF_2MB_PAGE_LOG2 - SIZEOF_PTE_LOG2 ), %ebx
1603	andl	$( SIZEOF_PT - 1 ), %ebx
16041:	subl	$SIZEOF_PT, %eax
1605	movl	%eax, (VIRTUAL(pde_textdata) - SIZEOF_PTE)(%ebx,%ecx,SIZEOF_PTE)
1606	loop	1b
1607
1608	/* Initialise PTEs for .textdata */
1609	movl	$_textdata_ptes, %ecx
1610	leal	( VIRTUAL(_textdata) + ( PG_P | PG_RW | PG_US ) )(%edi), %eax
1611	addl	$_textdata_paged_len, %eax
16121:	subl	$SIZEOF_4KB_PAGE, %eax
1613	movl	%eax, ( VIRTUAL(pte_textdata) - SIZEOF_PTE )(,%ecx,SIZEOF_PTE)
1614	loop	1b
1615
1616	/* Record PML4 physical address */
1617	leal	VIRTUAL(pml4e)(%edi), %eax
1618	movl	%eax, VIRTUAL(pml4)
1619
1620	/* Return */
1621	ret
1622