xref: /dragonfly/sys/cpu/x86_64/include/asmacros.h (revision cd9c4877)

/*
 * Copyright (c) 1993 The Regents of the University of California.
 * Copyright (c) 2008 The DragonFly Project.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: src/sys/amd64/include/asmacros.h,v 1.32 2006/10/28 06:04:29 bde Exp $
 */

#ifndef _CPU_ASMACROS_H_
#define _CPU_ASMACROS_H_

#include <sys/cdefs.h>
#include <machine/specialreg.h>

/* XXX too much duplication in various asm*.h's. */

/*
 * CNAME is used to manage the relationship between symbol names in C
 * and the equivalent assembly language names.  CNAME is given a name as
 * it would be used in a C program.  It expands to the equivalent assembly
 * language name.
 */
#define CNAME(csym)		csym

#define ALIGN_DATA	.p2align 3	/* 8 byte alignment, zero filled */
#define ALIGN_TEXT	.p2align 4,0x90	/* 16-byte alignment, nop filled */
#define SUPERALIGN_TEXT	.p2align 4,0x90	/* 16-byte alignment, nop filled */

#define GEN_ENTRY(name)		ALIGN_TEXT; .globl CNAME(name); \
				.type CNAME(name),@function; CNAME(name):
#define NON_GPROF_ENTRY(name)	GEN_ENTRY(name)
#define NON_GPROF_RET		.byte 0xc3	/* opcode for `ret' */

#define	END(name)		.size name, . - name

/*
 * ALTENTRY() has to align because it is before a corresponding ENTRY().
 * ENTRY() has to align to because there may be no ALTENTRY() before it.
 * If there is a previous ALTENTRY() then the alignment code for ENTRY()
 * is empty.
 */
#define ALTENTRY(name)		GEN_ENTRY(name)
#define	CROSSJUMP(jtrue, label, jfalse)	jtrue label
#define	CROSSJUMPTARGET(label)
#define ENTRY(name)		GEN_ENTRY(name)
#define FAKE_MCOUNT(caller)
#define MCOUNT
#define MCOUNT_LABEL(name)
#define MEXITCOUNT

#ifdef LOCORE
/*
 * Convenience macro for declaring interrupt entry points.
 */
#define	IDTVEC(name)	ALIGN_TEXT; .globl __CONCAT(X,name); \
			.type __CONCAT(X,name),@function; __CONCAT(X,name):

/*
 * stack frame macro support - supports mmu isolation, swapgs, and
 * stack frame pushing and popping.
 */

/*
 * Kernel pmap isolation to work-around the massive Intel mmu bug
 * that allows kernel memory to be sussed out due to speculative memory
 * reads and instruction execution creating timing differences that can
 * be detected by userland.  e.g. force speculative read, speculatively
 * execute a cmp/branch sequence, detect timing.  Iterate cmp $values
 * to suss-out content of speculatively read kernel memory.
 *
 * We do this by creating a trampoline area for all user->kernel and
 * kernel->user transitions.  The trampoline area allows us to limit
 * the reach the kernel map in the isolated version of the user pmap
 * to JUST the trampoline area (for all cpus), tss, and vector area.
 *
 * It is very important that these transitions not access any memory
 * outside of the trampoline page while the isolated user process pmap
 * is active in %cr3.
 *
 * The trampoline does not add much overhead when pmap isolation is
 * disabled, so we just run with it regardless.  Of course, when pmap
 * isolation is enabled, the %cr3 loads add 150-250ns to every system
 * call as well as (without PCID) smash the TLB.
 *
 * KMMUENTER -	Executed by the trampoline when a user->kernel transition
 *		is detected.  The stack pointer points into the pcpu
 *		trampoline space and is available for register save/restore.
 *		Other registers have not yet been saved.  %gs points at
 *		the kernel pcpu structure.
 *
 *		Caller has already determined that a transition is in
 *		progress and has already issued the swapgs.  hwtf indicates
 *		how much hardware has already pushed.
 *
 * KMMUEXIT  -	Executed when a kernel->user transition is made.  The stack
 *		pointer points into the pcpu trampoline space and we are
 *		almost ready to iretq.  %gs still points at the kernel pcpu
 *		structure.
 *
 *		Caller has already determined that a transition is in
 *		progress.  hwtf indicates how much hardware has already
 *		pushed.
 */

/*
 * KMMUENTER_CORE - Handles ISOMMU, IBRS, and IBPB.  Caller has already
 *		    saved %rcx and %rdx.  We have to deal with %rax.
 *
 *		    XXX If IBPB is not supported, try to clear the
 *		    call return hw cache w/ many x chained call sequence?
 *
 * NOTE - IBRS2 - We are leaving IBRS on full-time.  However, Intel
 *		  believes it is not safe unless the MSR is poked on each
 *		  user->kernel transition, so poke the MSR for both IBRS1
 *		  and IBRS2.
 */
#define KMMUENTER_CORE							\
	testq	$PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; 		\
	je	40f ;							\
	movq	PCPU(trampoline)+TR_PCB_CR3,%rcx ;			\
	movq	%rcx,%cr3 ;						\
40:	movl	PCPU(trampoline)+TR_PCB_SPEC_CTRL,%edx ;		\
	testq	%rdx, %rdx ;						\
	je	43f ;							\
	movq	%rax, PCPU(trampoline)+TR_RAX ;				\
	testq	$SPEC_CTRL_DUMMY_ENABLE,%rdx ;				\
	je	41f ;							\
	movq	%rdx, %rax ;						\
	andq	$SPEC_CTRL_IBRS|SPEC_CTRL_STIBP, %rax ;			\
	movq	$MSR_SPEC_CTRL,%rcx ;					\
	xorl	%edx,%edx ;						\
	wrmsr ;								\
	movl	PCPU(trampoline)+TR_PCB_SPEC_CTRL,%edx ;		\
41:	testq	$SPEC_CTRL_DUMMY_IBPB,%rdx ;				\
	je	42f ;							\
	movl	$MSR_PRED_CMD,%ecx ;					\
	movl	$1,%eax ;						\
	xorl	%edx,%edx ;						\
	wrmsr ;								\
42:	movq	PCPU(trampoline)+TR_RAX, %rax ;				\
43:									\

/*
 * Enter with trampoline, hardware pushed up to %rip
 */
#define KMMUENTER_TFRIP							\
	subq	$TR_RIP, %rsp ;						\
	movq	%rcx, TR_RCX(%rsp) ;					\
	movq	%rdx, TR_RDX(%rsp) ;					\
	KMMUENTER_CORE ;						\
	movq	%rsp, %rcx ;		/* trampoline rsp */		\
	movq	PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */	\
	movq	TR_SS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RSP(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RFLAGS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_CS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RIP(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RDX(%rcx), %rdx ;					\
	movq	TR_RCX(%rcx), %rcx					\

/*
 * Enter with trampoline, hardware pushed up to ERR
 */
#define KMMUENTER_TFERR							\
	subq	$TR_ERR, %rsp ;						\
	movq	%rcx, TR_RCX(%rsp) ;					\
	movq	%rdx, TR_RDX(%rsp) ;					\
	KMMUENTER_CORE ;						\
	movq	%rsp, %rcx ;		/* trampoline rsp */		\
	movq	PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */	\
	movq	TR_SS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RSP(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RFLAGS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_CS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RIP(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_ERR(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RDX(%rcx), %rdx ;					\
	movq	TR_RCX(%rcx), %rcx					\

/*
 * Enter with trampoline, hardware pushed up to ERR and
 * we need to save %cr2 early (before potentially reloading %cr3).
 */
#define KMMUENTER_TFERR_SAVECR2						\
	subq	$TR_ERR, %rsp ;						\
	movq	%rcx, TR_RCX(%rsp) ;					\
	movq	%rdx, TR_RDX(%rsp) ;					\
	movq	%cr2, %rcx ;						\
	movq	%rcx, PCPU(trampoline)+TR_CR2 ;				\
	KMMUENTER_CORE ;						\
	movq	%rsp, %rcx ;		/* trampoline rsp */		\
	movq	PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */	\
	movq	TR_SS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RSP(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RFLAGS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_CS(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RIP(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_ERR(%rcx), %rdx ;					\
	pushq	%rdx ;							\
	movq	TR_RDX(%rcx), %rdx ;					\
	movq	TR_RCX(%rcx), %rcx					\

/*
 * Set %cr3 if necessary on syscall entry.  No registers may be
 * disturbed.
 *
 * NOTE: TR_CR2 is used by the caller to save %rsp, we cannot use it here.
 */
#define KMMUENTER_SYSCALL						\
	movq	%rcx, PCPU(trampoline)+TR_RCX ;				\
	movq	%rdx, PCPU(trampoline)+TR_RDX ;				\
	KMMUENTER_CORE ;						\
	movq	PCPU(trampoline)+TR_RDX, %rdx ;				\
	movq	PCPU(trampoline)+TR_RCX, %rcx 				\

/*
 * KMMUEXIT_CORE handles IBRS, STIBP, and MDS, but not ISOMMU
 *
 * We don't re-execute the IBPB barrier on exit atm.
 *
 * The MDS barrier (Microarchitectural Data Sampling) should be executed
 * prior to any return to user-mode, if supported and enabled.  This is
 * Intel-only.
 *
 * WARNING! %rsp may not be usable (it could be pointing to the user
 *	    stack at this point).  And we must save/restore any registers
 *	    we use.
 */
#define KMMUEXIT_CORE							\
	testl	$SPEC_CTRL_DUMMY_ENABLE|SPEC_CTRL_MDS_ENABLE, PCPU(trampoline)+TR_PCB_SPEC_CTRL+4 ; \
	je	43f ;							\
	movq	%rax, PCPU(trampoline)+TR_RAX ;				\
	movl	PCPU(trampoline)+TR_PCB_SPEC_CTRL+4, %eax ;		\
	testq	$SPEC_CTRL_MDS_ENABLE, %rax ;				\
	je	41f ;							\
	movq	$GSEL(GDATA_SEL, SEL_KPL), PCPU(trampoline)+TR_RCX ;	\
	verw	PCPU(trampoline)+TR_RCX ;				\
41:	testq	$SPEC_CTRL_DUMMY_ENABLE, %rax ;				\
	je	42f ;							\
	movq	%rcx, PCPU(trampoline)+TR_RCX ;				\
	movq	%rdx, PCPU(trampoline)+TR_RDX ;				\
	andq	$SPEC_CTRL_IBRS|SPEC_CTRL_STIBP, %rax ;			\
	movq	$MSR_SPEC_CTRL,%rcx ;					\
	xorl	%edx,%edx ;						\
	wrmsr ;								\
	movq	PCPU(trampoline)+TR_RDX, %rdx ;				\
	movq	PCPU(trampoline)+TR_RCX, %rcx ;				\
42:	movq	PCPU(trampoline)+TR_RAX, %rax ;				\
43:

/*
 * We are positioned at the base of the trapframe.  Advance the trapframe
 * and handle MMU isolation.  MMU isolation requires us to copy the
 * hardware frame to the trampoline area before setting %cr3 to the
 * isolated map.  We then set the %rsp for iretq to TR_RIP in the
 * trampoline area (after restoring the register we saved in TR_ERR).
 */
#define KMMUEXIT							\
	addq	$TF_RIP,%rsp ;						\
	KMMUEXIT_CORE ;							\
	testq	$PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; 		\
	je	50f ;							\
	movq	%rcx, PCPU(trampoline)+TR_ERR ;	/* save in TR_ERR */	\
	popq	%rcx ;				/* copy %rip */		\
	movq	%rcx, PCPU(trampoline)+TR_RIP ;				\
	popq	%rcx ;				/* copy %cs */		\
	movq	%rcx, PCPU(trampoline)+TR_CS ;				\
	popq	%rcx ;				/* copy %rflags */	\
	movq	%rcx, PCPU(trampoline)+TR_RFLAGS ;			\
	popq	%rcx ;				/* copy %rsp */		\
	movq	%rcx, PCPU(trampoline)+TR_RSP ;				\
	popq	%rcx ;				/* copy %ss */		\
	movq	%rcx, PCPU(trampoline)+TR_SS ;				\
	movq	%gs:0,%rcx ;						\
	addq	$GD_TRAMPOLINE+TR_ERR,%rcx ;				\
	movq	%rcx,%rsp ;						\
	movq	PCPU(trampoline)+TR_PCB_CR3_ISO,%rcx ;			\
	movq	%rcx,%cr3 ;						\
	popq	%rcx ;		/* positioned at TR_RIP after this */	\
50:									\

/*
 * Warning: user stack pointer already loaded into %rsp at this
 * point.  We still have the kernel %gs.
 *
 * Caller will sysexit, we do not have to copy anything to the
 * trampoline area.
 */
#define KMMUEXIT_SYSCALL						\
	KMMUEXIT_CORE ;							\
	testq	$PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; 		\
	je	50f ;							\
	movq	%rcx, PCPU(trampoline)+TR_RCX ;				\
	movq	PCPU(trampoline)+TR_PCB_CR3_ISO,%rcx ;			\
	movq	%rcx,%cr3 ;						\
	movq	PCPU(trampoline)+TR_RCX, %rcx ;				\
50:									\

/*
 * Macros to create and destroy a trap frame.  rsp has already been shifted
 * to the base of the trapframe in the thread structure.
 */
#define PUSH_FRAME_REGS							\
	movq	%rdi,TF_RDI(%rsp) ;					\
	movq	%rsi,TF_RSI(%rsp) ;					\
	movq	%rdx,TF_RDX(%rsp) ;					\
	movq	%rcx,TF_RCX(%rsp) ;					\
	movq	%r8,TF_R8(%rsp) ;					\
	movq	%r9,TF_R9(%rsp) ;					\
	movq	%rax,TF_RAX(%rsp) ;					\
	movq	%rbx,TF_RBX(%rsp) ;					\
	movq	%rbp,TF_RBP(%rsp) ;					\
	movq	%r10,TF_R10(%rsp) ;					\
	movq	%r11,TF_R11(%rsp) ;					\
	movq	%r12,TF_R12(%rsp) ;					\
	movq	%r13,TF_R13(%rsp) ;					\
	movq	%r14,TF_R14(%rsp) ;					\
	movq	%r15,TF_R15(%rsp) ;					\
					/* SECURITY CLEAR REGS */	\
	xorq	%rax,%rax ;						\
	movq	%rax,%rbx ;						\
	movq	%rax,%rcx ;						\
	movq	%rax,%rdx ;						\
	movq	%rax,%rdi ;						\
	movq	%rax,%rsi ;						\
	movq	%rax,%rbp ;						\
	movq	%rax,%r8 ;						\
	movq	%rax,%r9 ;						\
	movq	%rax,%r10 ;						\
	movq	%rax,%r11 ;						\
	movq	%rax,%r12 ;						\
	movq	%rax,%r13 ;						\
	movq	%rax,%r14 ;						\
	movq	%rax,%r15 						\


/*
 * PUSH_FRAME is the first thing executed upon interrupt entry.  We are
 * responsible for swapgs execution and the KMMUENTER dispatch.
 *
 * NOTE - PUSH_FRAME code doesn't mess with %gs or the stack, or assume it can
 *	  use PCPU(trampoline), if the trap/exception is from supevisor mode.
 *	  It only messes with that stuff when the trap/exception is from user
 *	  mode.  Our DBG and NMI code depend on this behavior.
 */
#define PUSH_FRAME_TFRIP						\
	testb	$SEL_RPL_MASK,TF_CS-TF_RIP(%rsp) ; /* from userland? */	\
	jz	1f ;							\
	swapgs ;		/* from userland */			\
	KMMUENTER_TFRIP ;	/* from userland */			\
1:									\
	subq	$TF_RIP,%rsp ;						\
	PUSH_FRAME_REGS ; 						\
	SMAP_CLOSE							\

#define PUSH_FRAME_TFERR						\
	testb	$SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) ; /* from userland? */	\
	jz	1f ;							\
	swapgs ;		/* from userland */			\
	KMMUENTER_TFERR ;	/* from userland */			\
1:									\
	subq	$TF_ERR,%rsp ;						\
	PUSH_FRAME_REGS ;						\
	SMAP_CLOSE							\

#define PUSH_FRAME_TFERR_SAVECR2					\
	testb	$SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) ;			\
	jz	1f ;							\
	swapgs ;		/* from userland */			\
	KMMUENTER_TFERR_SAVECR2 ;/* from userland */			\
	subq	$TF_ERR,%rsp ;						\
	PUSH_FRAME_REGS ;						\
	movq	PCPU(trampoline)+TR_CR2, %r10 ;				\
	jmp 2f ;							\
1:									\
	subq	$TF_ERR,%rsp ;						\
	PUSH_FRAME_REGS ;						\
	movq	%cr2, %r10 ;						\
2:									\
	movq	%r10, TF_ADDR(%rsp) ;					\
	SMAP_CLOSE							\

/*
 * POP_FRAME is issued just prior to the iretq, or just prior to a
 * jmp doreti_iret.  These must be passed in to the macro.
 */
#define POP_FRAME(lastinsn)						\
	movq	TF_RDI(%rsp),%rdi ;					\
	movq	TF_RSI(%rsp),%rsi ;					\
	movq	TF_RDX(%rsp),%rdx ;					\
	movq	TF_RCX(%rsp),%rcx ;					\
	movq	TF_R8(%rsp),%r8 ;					\
	movq	TF_R9(%rsp),%r9 ;					\
	movq	TF_RAX(%rsp),%rax ;					\
	movq	TF_RBX(%rsp),%rbx ;					\
	movq	TF_RBP(%rsp),%rbp ;					\
	movq	TF_R10(%rsp),%r10 ;					\
	movq	TF_R11(%rsp),%r11 ;					\
	movq	TF_R12(%rsp),%r12 ;					\
	movq	TF_R13(%rsp),%r13 ;					\
	movq	TF_R14(%rsp),%r14 ;					\
	movq	TF_R15(%rsp),%r15 ;					\
	cli ;								\
	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* return to user? */	\
	jz	1f ;							\
	KMMUEXIT ;		/* return to user */			\
	swapgs ;		/* return to user */			\
	jmp	2f ;							\
1:									\
	addq	$TF_RIP,%rsp ;	/* setup for iretq */			\
2:									\
	lastinsn

/*
 * Access per-CPU data.
 */
#define	PCPU(member)		%gs:gd_ ## member
#define PCPU_E8(member,idx)	%gs:gd_ ## member(,idx,8)
#define	PCPU_ADDR(member, reg)					\
	movq %gs:PC_PRVSPACE, reg ;				\
	addq $PC_ ## member, reg

/*
 * This adds a code placemarker (the 3-byte NOP) and references
 * it from a section, allowing it to be replaced with a STAC or CLAC
 * instruction (also 3-byte instructions) when the system supports
 * SMAP.
 */
#define SMAP_OPEN						\
	.globl	__start_set_smap_open ;				\
	.globl	__stop_set_smap_open ;				\
60:	nop	%eax ;						\
	.section set_smap_open,"aw" ;				\
	.quad	60b ;						\
	.previous						\

#define SMAP_CLOSE						\
	.globl	__start_set_smap_close ;			\
	.globl	__stop_set_smap_close ;				\
60:	nop	%eax ;						\
	.section set_smap_close,"aw" ;				\
	.quad	60b ;						\
	.previous						\

#endif /* LOCORE */

#endif /* !_CPU_ASMACROS_H_ */