xref: /dragonfly/sys/platform/pc64/apic/apic_vector.s (revision 7f29793e)

/*
 *	from: vector.s, 386BSD 0.1 unknown origin
 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
 */

#if 0
#include "opt_auto_eoi.h"
#endif

#include <machine/asmacros.h>
#include <machine/lock.h>
#include <machine/psl.h>
#include <machine/trap.h>
#include <machine/segments.h>

#include <machine_base/icu/icu.h>
#include <bus/isa/isa.h>

#include "assym.s"

#include "apicreg.h"
#include <machine_base/apic/ioapic_ipl.h>
#include <machine/intr_machdep.h>

#ifdef foo
/* convert an absolute IRQ# into bitmask */
#define IRQ_LBIT(irq_num)	(1UL << (irq_num & 0x3f))
#endif

#define IRQ_SBITS(irq_num)	((irq_num) & 0x3f)

/* convert an absolute IRQ# into gd_ipending index */
#define IRQ_LIDX(irq_num)	((irq_num) >> 6)

#define MPLOCKED     lock ;

#define APIC_PUSH_FRAME_TFRIP						\
	PUSH_FRAME_TFRIP ;	/* 15 regs + space for 5 extras */	\
	movq $0,TF_XFLAGS(%rsp) ;					\
	movq $0,TF_TRAPNO(%rsp) ;					\
	movq $0,TF_ADDR(%rsp) ;						\
	movq $0,TF_FLAGS(%rsp) ;					\
	movq $0,TF_ERR(%rsp) ;						\
	cld ;								\

/*
 * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
 * segment register being changed (e.g. by procfs), which is why syscalls
 * have to use doreti.
 */
#define APIC_POP_FRAME(lastinsn)					\
	POP_FRAME(lastinsn) 						\

#define IOAPICADDR(irq_num) \
	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
#define REDIRIDX(irq_num) \
	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX
#define IOAPICFLAGS(irq_num) \
	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS

#define MASK_IRQ(irq_num)						\
	testl	$IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;	\
	jne	7f ;			/* masked, don't mask */	\
	orl	$IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;	\
						/* set the mask bit */	\
	movq	IOAPICADDR(irq_num), %rcx ;	/* ioapic addr */	\
	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
	movl	%eax, (%rcx) ;			/* write the index */	\
	orl	$IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */	\
7: ;						/* already masked */	\

/*
 * Test to see whether we are handling an edge or level triggered INT.
 *  Level-triggered INTs must still be masked as we don't clear the source,
 *  and the EOI cycle would cause redundant INTs to occur.
 */
#define MASK_LEVEL_IRQ(irq_num)						\
	testl	$IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;		\
	jz	9f ;				/* edge, don't mask */	\
	MASK_IRQ(irq_num) ;						\
9: ;									\

/*
 * Test to see if the source is currntly masked, clear if so.
 */
#define UNMASK_IRQ(irq_num)					\
	cmpl	$0,%eax ;						\
	jnz	8f ;							\
	IOAPIC_IMASK_LOCK ;						\
	testl	$IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;	\
	je	7f ;			/* bit clear, not masked */	\
	andl	$~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;	\
						/* clear mask bit */	\
	movq	IOAPICADDR(irq_num),%rcx ;	/* ioapic addr */	\
	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
	movl	%eax,(%rcx) ;			/* write the index */	\
	andl	$~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
7: ;									\
	IOAPIC_IMASK_UNLOCK ;						\
8: ;									\

/*
 * Interrupt call handlers run in the following sequence:
 *
 *	- Push the trap frame required by doreti
 *	- Mask the interrupt and reenable its source
 *	- If we cannot take the interrupt set its ipending bit and
 *	  doreti.
 *	- If we can take the interrupt clear its ipending bit,
 *	  call the handler, then unmask and doreti.
 *
 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 */

#define	INTR_HANDLER(irq_num)						\
	.text ;								\
	SUPERALIGN_TEXT ;						\
IDTVEC(ioapic_intr##irq_num) ;						\
	APIC_PUSH_FRAME_TFRIP ;						\
	FAKE_MCOUNT(TF_RIP(%rsp)) ;					\
	IOAPIC_IMASK_LOCK ;						\
	MASK_LEVEL_IRQ(irq_num) ;					\
	movq	lapic_eoi, %rax ;					\
	IOAPIC_IMASK_UNLOCK ;						\
	callq	*%rax ;							\
	movq	PCPU(curthread),%rbx ;					\
	testl	$-1,TD_NEST_COUNT(%rbx) ;				\
	jne	1f ;							\
	testl	$-1,TD_CRITCOUNT(%rbx) ;				\
	je	2f ;							\
1: ;									\
	/* in critical section, make interrupt pending */		\
	/* set the pending bit and return, leave interrupt masked */	\
	movq	$1,%rcx ;						\
	shlq	$IRQ_SBITS(irq_num),%rcx ;				\
	movq	$IRQ_LIDX(irq_num),%rdx ;				\
	orq	%rcx,PCPU_E8(ipending,%rdx) ;				\
	orl	$RQF_INTPEND,PCPU(reqflags) ;				\
	jmp	5f ;							\
2: ;									\
	/* clear pending bit, run handler */				\
	movq	$1,%rcx ;						\
	shlq	$IRQ_SBITS(irq_num),%rcx ;				\
	notq	%rcx ;							\
	movq	$IRQ_LIDX(irq_num),%rdx ;				\
	andq	%rcx,PCPU_E8(ipending,%rdx) ;				\
	pushq	$irq_num ;		/* trapframe -> intrframe */	\
	movq	%rsp, %rdi ;		/* pass frame by reference */	\
	incl	TD_CRITCOUNT(%rbx) ;					\
	sti ;								\
	call	ithread_fast_handler ;	/* returns 0 to unmask */	\
	cli ;				/* interlock avoid stacking */	\
	decl	TD_CRITCOUNT(%rbx) ;					\
	addq	$8, %rsp ;		/* intrframe -> trapframe */	\
	UNMASK_IRQ(irq_num) ;						\
5: ;									\
	MEXITCOUNT ;							\
	jmp	doreti ;						\

/*
 * Handle "spurious INTerrupts".
 *
 * NOTE: This is different than the "spurious INTerrupt" generated by an
 *	 8259 PIC for missing INTs.  See the APIC documentation for details.
 *	 This routine should NOT do an 'EOI' cycle.
 *
 * NOTE: Even though we don't do anything here we must still swapgs if
 *	 coming from a user frame in case the iretq faults... just use
 *	 the nominal APIC_PUSH_FRAME sequence to get it done.
 */
	.text
	SUPERALIGN_TEXT
	.globl Xspuriousint
Xspuriousint:
	APIC_PUSH_FRAME_TFRIP
	/* No EOI cycle used here */
	FAKE_MCOUNT(TF_RIP(%rsp))
	MEXITCOUNT
	APIC_POP_FRAME(jmp doreti_iret)

/*
 * Handle TLB shootdowns.
 *
 * NOTE: interrupts are left disabled.
 */
	.text
	SUPERALIGN_TEXT
	.globl	Xinvltlb
Xinvltlb:
	APIC_PUSH_FRAME_TFRIP
	movq	lapic_eoi, %rax
	callq	*%rax			/* End Of Interrupt to APIC */
	FAKE_MCOUNT(TF_RIP(%rsp))
	incl    PCPU(cnt) + V_IPI
	movq	PCPU(curthread),%rbx
	incl    PCPU(intr_nesting_level)
	incl    TD_CRITCOUNT(%rbx)
	subq	$8,%rsp			/* make same as interrupt frame */
	movq	%rsp,%rdi		/* pass frame by reference */
	orl	$RQF_XINVLTLB,PCPU(reqflags) /* HVM interlock */
	call	smp_inval_intr		/* called w/interrupts disabled */
	addq	$8,%rsp			/* turn into trapframe */
	decl	TD_CRITCOUNT(%rbx)
	decl	PCPU(intr_nesting_level)
	MEXITCOUNT
	/*APIC_POP_FRAME*/
	jmp	doreti			/* doreti b/c intrs enabled */

/*
 * Handle sniffs - sniff %rip and %rsp.
 */
	.text
	SUPERALIGN_TEXT
	.globl	Xsniff
Xsniff:
	APIC_PUSH_FRAME_TFRIP
	movq	lapic_eoi, %rax
	callq	*%rax			/* End Of Interrupt to APIC */
	FAKE_MCOUNT(TF_RIP(%rsp))
	incl    PCPU(cnt) + V_IPI
	movq	%rsp,%rdi
	call	CNAME(hard_sniff)	/* systat -pv and flame sniff */
	MEXITCOUNT
	APIC_POP_FRAME(jmp doreti_iret)

/*
 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 *
 *  - We cannot call doreti
 *  - Signals its receipt.
 *  - Waits for permission to restart.
 *  - Processing pending IPIQ events while waiting.
 *  - Signals its restart.
 */

	.text
	SUPERALIGN_TEXT
	.globl Xcpustop
Xcpustop:
	APIC_PUSH_FRAME_TFRIP
	movq	lapic_eoi, %rax
	callq	*%rax			/* End Of Interrupt to APIC */

	movl	PCPU(cpuid), %eax
	imull	$PCB_SIZE, %eax
	leaq	CNAME(stoppcbs), %rdi
	addq	%rax, %rdi
	call	CNAME(savectx)		/* Save process context */

	/*
	 * Indicate that we have stopped and loop waiting for permission
	 * to start again.  We must still process IPI events while in a
	 * stopped state.
	 *
	 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
	 * (e.g. Xtimer, Xinvltlb).
	 */
#if CPUMASK_ELEMENTS != 4
#error "assembly incompatible with cpumask_t"
#endif
	movq	PCPU(cpumask)+0,%rax	/* stopped_cpus |= 1 << cpuid */
	MPLOCKED orq %rax, stopped_cpus+0
	movq	PCPU(cpumask)+8,%rax
	MPLOCKED orq %rax, stopped_cpus+8
	movq	PCPU(cpumask)+16,%rax
	MPLOCKED orq %rax, stopped_cpus+16
	movq	PCPU(cpumask)+24,%rax
	MPLOCKED orq %rax, stopped_cpus+24

	movq	PCPU(curthread),%rbx
	incl    PCPU(intr_nesting_level)
	incl    TD_CRITCOUNT(%rbx)
	sti
1:
	andl	$~RQF_IPIQ,PCPU(reqflags)
	call	lwkt_smp_stopped
	pause

	subq	%rdi,%rdi
	movq	started_cpus+0,%rax	/* while (!(started_cpus & (1<<id))) */
	andq	PCPU(cpumask)+0,%rax
	orq	%rax,%rdi
	movq	started_cpus+8,%rax
	andq	PCPU(cpumask)+8,%rax
	orq	%rax,%rdi
	movq	started_cpus+16,%rax
	andq	PCPU(cpumask)+16,%rax
	orq	%rax,%rdi
	movq	started_cpus+24,%rax
	andq	PCPU(cpumask)+24,%rax
	orq	%rax,%rdi
	testq	%rdi,%rdi
	jz	1b

	movq	PCPU(other_cpus)+0,%rax	/* started_cpus &= ~(1 << cpuid) */
	MPLOCKED andq %rax, started_cpus+0
	movq	PCPU(other_cpus)+8,%rax
	MPLOCKED andq %rax, started_cpus+8
	movq	PCPU(other_cpus)+16,%rax
	MPLOCKED andq %rax, started_cpus+16
	movq	PCPU(other_cpus)+24,%rax
	MPLOCKED andq %rax, started_cpus+24

	movq	PCPU(other_cpus)+0,%rax	/* stopped_cpus &= ~(1 << cpuid) */
	MPLOCKED andq %rax, stopped_cpus+0
	movq	PCPU(other_cpus)+8,%rax
	MPLOCKED andq %rax, stopped_cpus+8
	movq	PCPU(other_cpus)+16,%rax
	MPLOCKED andq %rax, stopped_cpus+16
	movq	PCPU(other_cpus)+24,%rax
	MPLOCKED andq %rax, stopped_cpus+24

	cmpl	$0,PCPU(cpuid)
	jnz	2f

	movq	CNAME(cpustop_restartfunc), %rax
	testq	%rax, %rax
	jz	2f
	movq	$0, CNAME(cpustop_restartfunc)	/* One-shot */

	call	*%rax
2:
	decl	TD_CRITCOUNT(%rbx)
	decl	PCPU(intr_nesting_level)
	MEXITCOUNT
	/*APIC_POP_FRAME*/
	jmp	doreti

	/*
	 * For now just have one ipiq IPI, but what we really want is
	 * to have one for each source cpu to the APICs don't get stalled
	 * backlogging the requests.
	 */
	.text
	SUPERALIGN_TEXT
	.globl Xipiq
Xipiq:
	APIC_PUSH_FRAME_TFRIP
	movq	lapic_eoi, %rax
	callq	*%rax			/* End Of Interrupt to APIC */
	FAKE_MCOUNT(TF_RIP(%rsp))

	incl    PCPU(cnt) + V_IPI
	movq	PCPU(curthread),%rbx
	testl	$-1,TD_CRITCOUNT(%rbx)
	jne	1f
	subq	$8,%rsp			/* make same as interrupt frame */
	movq	%rsp,%rdi		/* pass frame by reference */
	incl	PCPU(intr_nesting_level)
	incl	TD_CRITCOUNT(%rbx)
	subq	%rax,%rax
	sti
	xchgl	%eax,PCPU(npoll)	/* (atomic op) allow another Xipi */
	call	lwkt_process_ipiq_frame
	cli 				/* interlock avoid stacking */
	decl	TD_CRITCOUNT(%rbx)
	decl	PCPU(intr_nesting_level)
	addq	$8,%rsp			/* turn into trapframe */
	MEXITCOUNT
	jmp	doreti
1:
	orl	$RQF_IPIQ,PCPU(reqflags)
	MEXITCOUNT
	APIC_POP_FRAME(jmp doreti_iret)

	.text
	SUPERALIGN_TEXT
	.globl Xtimer
Xtimer:
	APIC_PUSH_FRAME_TFRIP
	movq	lapic_eoi, %rax
	callq	*%rax			/* End Of Interrupt to APIC */
	FAKE_MCOUNT(TF_RIP(%rsp))

	subq	$8,%rsp			/* make same as interrupt frame */
	movq	%rsp,%rdi		/* pass frame by reference */
	call	pcpu_timer_always
	addq	$8,%rsp			/* turn into trapframe */

	incl    PCPU(cnt) + V_TIMER
	movq	TF_RIP(%rsp),%rbx	/* sample addr before checking crit */
	movq	%rbx,PCPU(sample_pc)
	movq	PCPU(curthread),%rbx
	testl	$-1,TD_CRITCOUNT(%rbx)
	jne	1f
	testl	$-1,TD_NEST_COUNT(%rbx)
	jne	1f
	subq	$8,%rsp			/* make same as interrupt frame */
	movq	%rsp,%rdi		/* pass frame by reference */
	incl	PCPU(intr_nesting_level)
	incl	TD_CRITCOUNT(%rbx)
	sti
	call	pcpu_timer_process_frame
	cli 				/* interlock avoid stacking */
	decl	TD_CRITCOUNT(%rbx)
	decl	PCPU(intr_nesting_level)
	addq	$8,%rsp			/* turn into trapframe */
	MEXITCOUNT
	jmp	doreti
1:
	orl	$RQF_TIMER,PCPU(reqflags)
	MEXITCOUNT
	APIC_POP_FRAME(jmp doreti_iret)

MCOUNT_LABEL(bintr)
	INTR_HANDLER(0)
	INTR_HANDLER(1)
	INTR_HANDLER(2)
	INTR_HANDLER(3)
	INTR_HANDLER(4)
	INTR_HANDLER(5)
	INTR_HANDLER(6)
	INTR_HANDLER(7)
	INTR_HANDLER(8)
	INTR_HANDLER(9)
	INTR_HANDLER(10)
	INTR_HANDLER(11)
	INTR_HANDLER(12)
	INTR_HANDLER(13)
	INTR_HANDLER(14)
	INTR_HANDLER(15)
	INTR_HANDLER(16)
	INTR_HANDLER(17)
	INTR_HANDLER(18)
	INTR_HANDLER(19)
	INTR_HANDLER(20)
	INTR_HANDLER(21)
	INTR_HANDLER(22)
	INTR_HANDLER(23)
	INTR_HANDLER(24)
	INTR_HANDLER(25)
	INTR_HANDLER(26)
	INTR_HANDLER(27)
	INTR_HANDLER(28)
	INTR_HANDLER(29)
	INTR_HANDLER(30)
	INTR_HANDLER(31)
	INTR_HANDLER(32)
	INTR_HANDLER(33)
	INTR_HANDLER(34)
	INTR_HANDLER(35)
	INTR_HANDLER(36)
	INTR_HANDLER(37)
	INTR_HANDLER(38)
	INTR_HANDLER(39)
	INTR_HANDLER(40)
	INTR_HANDLER(41)
	INTR_HANDLER(42)
	INTR_HANDLER(43)
	INTR_HANDLER(44)
	INTR_HANDLER(45)
	INTR_HANDLER(46)
	INTR_HANDLER(47)
	INTR_HANDLER(48)
	INTR_HANDLER(49)
	INTR_HANDLER(50)
	INTR_HANDLER(51)
	INTR_HANDLER(52)
	INTR_HANDLER(53)
	INTR_HANDLER(54)
	INTR_HANDLER(55)
	INTR_HANDLER(56)
	INTR_HANDLER(57)
	INTR_HANDLER(58)
	INTR_HANDLER(59)
	INTR_HANDLER(60)
	INTR_HANDLER(61)
	INTR_HANDLER(62)
	INTR_HANDLER(63)
	INTR_HANDLER(64)
	INTR_HANDLER(65)
	INTR_HANDLER(66)
	INTR_HANDLER(67)
	INTR_HANDLER(68)
	INTR_HANDLER(69)
	INTR_HANDLER(70)
	INTR_HANDLER(71)
	INTR_HANDLER(72)
	INTR_HANDLER(73)
	INTR_HANDLER(74)
	INTR_HANDLER(75)
	INTR_HANDLER(76)
	INTR_HANDLER(77)
	INTR_HANDLER(78)
	INTR_HANDLER(79)
	INTR_HANDLER(80)
	INTR_HANDLER(81)
	INTR_HANDLER(82)
	INTR_HANDLER(83)
	INTR_HANDLER(84)
	INTR_HANDLER(85)
	INTR_HANDLER(86)
	INTR_HANDLER(87)
	INTR_HANDLER(88)
	INTR_HANDLER(89)
	INTR_HANDLER(90)
	INTR_HANDLER(91)
	INTR_HANDLER(92)
	INTR_HANDLER(93)
	INTR_HANDLER(94)
	INTR_HANDLER(95)
	INTR_HANDLER(96)
	INTR_HANDLER(97)
	INTR_HANDLER(98)
	INTR_HANDLER(99)
	INTR_HANDLER(100)
	INTR_HANDLER(101)
	INTR_HANDLER(102)
	INTR_HANDLER(103)
	INTR_HANDLER(104)
	INTR_HANDLER(105)
	INTR_HANDLER(106)
	INTR_HANDLER(107)
	INTR_HANDLER(108)
	INTR_HANDLER(109)
	INTR_HANDLER(110)
	INTR_HANDLER(111)
	INTR_HANDLER(112)
	INTR_HANDLER(113)
	INTR_HANDLER(114)
	INTR_HANDLER(115)
	INTR_HANDLER(116)
	INTR_HANDLER(117)
	INTR_HANDLER(118)
	INTR_HANDLER(119)
	INTR_HANDLER(120)
	INTR_HANDLER(121)
	INTR_HANDLER(122)
	INTR_HANDLER(123)
	INTR_HANDLER(124)
	INTR_HANDLER(125)
	INTR_HANDLER(126)
	INTR_HANDLER(127)
	INTR_HANDLER(128)
	INTR_HANDLER(129)
	INTR_HANDLER(130)
	INTR_HANDLER(131)
	INTR_HANDLER(132)
	INTR_HANDLER(133)
	INTR_HANDLER(134)
	INTR_HANDLER(135)
	INTR_HANDLER(136)
	INTR_HANDLER(137)
	INTR_HANDLER(138)
	INTR_HANDLER(139)
	INTR_HANDLER(140)
	INTR_HANDLER(141)
	INTR_HANDLER(142)
	INTR_HANDLER(143)
	INTR_HANDLER(144)
	INTR_HANDLER(145)
	INTR_HANDLER(146)
	INTR_HANDLER(147)
	INTR_HANDLER(148)
	INTR_HANDLER(149)
	INTR_HANDLER(150)
	INTR_HANDLER(151)
	INTR_HANDLER(152)
	INTR_HANDLER(153)
	INTR_HANDLER(154)
	INTR_HANDLER(155)
	INTR_HANDLER(156)
	INTR_HANDLER(157)
	INTR_HANDLER(158)
	INTR_HANDLER(159)
	INTR_HANDLER(160)
	INTR_HANDLER(161)
	INTR_HANDLER(162)
	INTR_HANDLER(163)
	INTR_HANDLER(164)
	INTR_HANDLER(165)
	INTR_HANDLER(166)
	INTR_HANDLER(167)
	INTR_HANDLER(168)
	INTR_HANDLER(169)
	INTR_HANDLER(170)
	INTR_HANDLER(171)
	INTR_HANDLER(172)
	INTR_HANDLER(173)
	INTR_HANDLER(174)
	INTR_HANDLER(175)
	INTR_HANDLER(176)
	INTR_HANDLER(177)
	INTR_HANDLER(178)
	INTR_HANDLER(179)
	INTR_HANDLER(180)
	INTR_HANDLER(181)
	INTR_HANDLER(182)
	INTR_HANDLER(183)
	INTR_HANDLER(184)
	INTR_HANDLER(185)
	INTR_HANDLER(186)
	INTR_HANDLER(187)
	INTR_HANDLER(188)
	INTR_HANDLER(189)
	INTR_HANDLER(190)
	INTR_HANDLER(191)
MCOUNT_LABEL(eintr)

	.data

#if CPUMASK_ELEMENTS != 4
#error "assembly incompatible with cpumask_t"
#endif
/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
	.globl stopped_cpus, started_cpus
stopped_cpus:
	.quad	0
	.quad	0
	.quad	0
	.quad	0
started_cpus:
	.quad	0
	.quad	0
	.quad	0
	.quad	0

	.globl CNAME(cpustop_restartfunc)
CNAME(cpustop_restartfunc):
	.quad 0

	.text