xref: /dragonfly/sys/platform/pc64/apic/lapic.c (revision db2ec6f8)

/*
 * Copyright (c) 1996, by Steve Passe
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. The name of the developer may NOT be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: src/sys/i386/i386/mpapic.c,v 1.37.2.7 2003/01/25 02:31:47 peter Exp $
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/bus.h>
#include <sys/machintr.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <machine/globaldata.h>
#include <machine/clock.h>
#include <machine/limits.h>
#include <machine/smp.h>
#include <machine/md_var.h>
#include <machine/pmap.h>
#include <machine/specialreg.h>
#include <machine_base/apic/lapic.h>
#include <machine_base/apic/ioapic.h>
#include <machine_base/apic/ioapic_abi.h>
#include <machine_base/apic/apicvar.h>
#include <machine_base/icu/icu_var.h>
#include <machine/segments.h>
#include <sys/spinlock2.h>

#include <machine/cputypes.h>
#include <machine/intr_machdep.h>

#if !defined(KTR_LAPIC)
#define KTR_LAPIC	KTR_ALL
#endif
KTR_INFO_MASTER(lapic);
KTR_INFO(KTR_LAPIC, lapic, mem_eoi, 0, "mem_eoi");
KTR_INFO(KTR_LAPIC, lapic, msr_eoi, 0, "msr_eoi");
#define log_lapic(name)     KTR_LOG(lapic_ ## name)

extern int naps;

volatile lapic_t *lapic_mem;

static void	lapic_timer_calibrate(void);
static void	lapic_timer_set_divisor(int);
static void	lapic_timer_fixup_handler(void *);
static void	lapic_timer_restart_handler(void *);

static int	lapic_timer_c1e_test = -1;	/* auto-detect */
TUNABLE_INT("hw.lapic_timer_c1e_test", &lapic_timer_c1e_test);

static int	lapic_timer_enable = 1;
TUNABLE_INT("hw.lapic_timer_enable", &lapic_timer_enable);

static int	lapic_timer_tscdeadline = 1;
TUNABLE_INT("hw.lapic_timer_tscdeadline", &lapic_timer_tscdeadline);

static int	lapic_calibrate_test = 0;
TUNABLE_INT("hw.lapic_calibrate_test", &lapic_calibrate_test);

static int	lapic_calibrate_fast = 1;
TUNABLE_INT("hw.lapic_calibrate_fast", &lapic_calibrate_fast);

static void	lapic_timer_tscdlt_reload(struct cputimer_intr *, sysclock_t);
static void	lapic_mem_timer_intr_reload(struct cputimer_intr *, sysclock_t);
static void	lapic_msr_timer_intr_reload(struct cputimer_intr *, sysclock_t);
static void	lapic_timer_intr_enable(struct cputimer_intr *);
static void	lapic_timer_intr_restart(struct cputimer_intr *);
static void	lapic_timer_intr_pmfixup(struct cputimer_intr *);

static struct cputimer_intr lapic_cputimer_intr = {
	.freq = 0,
	.reload = lapic_mem_timer_intr_reload,
	.enable = lapic_timer_intr_enable,
	.config = cputimer_intr_default_config,
	.restart = lapic_timer_intr_restart,
	.pmfixup = lapic_timer_intr_pmfixup,
	.initclock = cputimer_intr_default_initclock,
	.pcpuhand = NULL,
	.next = SLIST_ENTRY_INITIALIZER,
	.name = "lapic",
	.type = CPUTIMER_INTR_LAPIC,
	.prio = CPUTIMER_INTR_PRIO_LAPIC,
	.caps = CPUTIMER_INTR_CAP_NONE,
	.priv = NULL
};

static int		lapic_timer_divisor_idx = -1;
static const uint32_t	lapic_timer_divisors[] = {
	APIC_TDCR_2,	APIC_TDCR_4,	APIC_TDCR_8,	APIC_TDCR_16,
	APIC_TDCR_32,	APIC_TDCR_64,	APIC_TDCR_128,	APIC_TDCR_1
};
#define APIC_TIMER_NDIVISORS (int)(NELEM(lapic_timer_divisors))

static int	lapic_use_tscdeadline = 0;

/*
 * APIC ID <-> CPU ID mapping structures.
 */
int	cpu_id_to_apic_id[NAPICID];
int	apic_id_to_cpu_id[NAPICID];
int	lapic_enable = 1;
int	lapic_usable = 0;
int	x2apic_enable = 1;

SYSCTL_INT(_hw, OID_AUTO, x2apic_enable, CTLFLAG_RD, &x2apic_enable, 0, "");

/* Separate cachelines for each cpu's info. */
struct deadlines {
	uint64_t timestamp;
	uint64_t downcount_time;
	uint64_t padding[6];
};
static struct deadlines *tsc_deadlines = NULL;

static void	lapic_mem_eoi(void);
static int	lapic_mem_ipi(int dest_type, int vector, int delivery_mode);
static void	lapic_mem_single_ipi(int cpu, int vector, int delivery_mode);

static void	lapic_msr_eoi(void);
static int	lapic_msr_ipi(int dest_type, int vector, int delivery_mode);
static void	lapic_msr_single_ipi(int cpu, int vector, int delivery_mode);

void		(*lapic_eoi)(void);
int		(*apic_ipi)(int dest_type, int vector, int delivery_mode);
void		(*single_apic_ipi)(int cpu, int vector, int delivery_mode);

static __inline void
lapic_mem_icr_set(uint32_t apic_id, uint32_t icr_lo_val)
{
	uint32_t icr_lo, icr_hi;

	icr_hi = (LAPIC_MEM_READ(icr_hi) & ~APIC_ID_MASK) |
	    (apic_id << APIC_ID_SHIFT);
	icr_lo = (LAPIC_MEM_READ(icr_lo) & APIC_ICRLO_RESV_MASK) | icr_lo_val;

	LAPIC_MEM_WRITE(icr_hi, icr_hi);
	LAPIC_MEM_WRITE(icr_lo, icr_lo);
}

static __inline void
lapic_msr_icr_set(uint32_t apic_id, uint32_t icr_lo_val)
{
	LAPIC_MSR_WRITE(MSR_X2APIC_ICR,
	    ((uint64_t)apic_id << 32) | ((uint64_t)icr_lo_val));
}

/*
 * Enable LAPIC, configure interrupts.
 */
void
lapic_init(boolean_t bsp)
{
	uint32_t timer;
	u_int   temp;

	if (bsp) {
		/* Decide whether we want to use TSC Deadline mode. */
		if (lapic_timer_tscdeadline != 0 &&
		    (cpu_feature2 & CPUID2_TSCDLT) &&
		    tsc_invariant && tsc_frequency != 0) {
			lapic_use_tscdeadline = 1;
			tsc_deadlines =
				kmalloc(sizeof(struct deadlines) * (naps + 1),
					M_DEVBUF,
					M_WAITOK | M_ZERO | M_CACHEALIGN);
		}
	}

	/*
	 * Install vectors
	 *
	 * Since IDT is shared between BSP and APs, these vectors
	 * only need to be installed once; we do it on BSP.
	 */
	if (bsp) {
		if (cpu_vendor_id == CPU_VENDOR_AMD &&
		    CPUID_TO_FAMILY(cpu_id) >= 0x0f &&
		    CPUID_TO_FAMILY(cpu_id) < 0x17) {	/* XXX */
			uint32_t tcr;

			/*
			 * Set the LINTEN bit in the HyperTransport
			 * Transaction Control Register.
			 *
			 * This will cause EXTINT and NMI interrupts
			 * routed over the hypertransport bus to be
			 * fed into the LAPIC LINT0/LINT1.  If the bit
			 * isn't set, the interrupts will go to the
			 * general cpu INTR/NMI pins.  On a dual-core
			 * cpu the interrupt winds up going to BOTH cpus.
			 * The first cpu that does the interrupt ack
			 * cycle will get the correct interrupt.  The
			 * second cpu that does it will get a spurious
			 * interrupt vector (typically IRQ 7).
			 */
			outl(0x0cf8,
			    (1 << 31) |	/* enable */
			    (0 << 16) |	/* bus */
			    (0x18 << 11) | /* dev (cpu + 0x18) */
			    (0 << 8) |	/* func */
			    0x68	/* reg */
			    );
			tcr = inl(0xcfc);
			if ((tcr & 0x00010000) == 0) {
				kprintf("LAPIC: AMD LINTEN on\n");
				outl(0xcfc, tcr|0x00010000);
			}
			outl(0x0cf8, 0);
		}

		/* Install a 'Spurious INTerrupt' vector */
		setidt_global(XSPURIOUSINT_OFFSET, Xspuriousint,
		    SDT_SYSIGT, SEL_KPL, 0);

		/* Install a timer vector */
		setidt_global(XTIMER_OFFSET, Xtimer,
		    SDT_SYSIGT, SEL_KPL, 0);

		/* Install an inter-CPU IPI for TLB invalidation */
		setidt_global(XINVLTLB_OFFSET, Xinvltlb,
		    SDT_SYSIGT, SEL_KPL, 0);

		/* Install an inter-CPU IPI for IPIQ messaging */
		setidt_global(XIPIQ_OFFSET, Xipiq,
		    SDT_SYSIGT, SEL_KPL, 0);

		/* Install an inter-CPU IPI for CPU stop/restart */
		setidt_global(XCPUSTOP_OFFSET, Xcpustop,
		    SDT_SYSIGT, SEL_KPL, 0);

		/* Install an inter-CPU IPI for TLB invalidation */
		setidt_global(XSNIFF_OFFSET, Xsniff,
		    SDT_SYSIGT, SEL_KPL, 0);
	}

	/*
	 * Setup LINT0 as ExtINT on the BSP.  This is theoretically an
	 * aggregate interrupt input from the 8259.  The INTA cycle
	 * will be routed to the external controller (the 8259) which
	 * is expected to supply the vector.
	 *
	 * Must be setup edge triggered, active high.
	 *
	 * Disable LINT0 on BSP, if I/O APIC is enabled.
	 *
	 * Disable LINT0 on the APs.  It doesn't matter what delivery
	 * mode we use because we leave it masked.
	 */
	temp = LAPIC_READ(lvt_lint0);
	temp &= ~(APIC_LVT_MASKED | APIC_LVT_TRIG_MASK |
		  APIC_LVT_POLARITY_MASK | APIC_LVT_DM_MASK);
	if (bsp) {
		temp |= APIC_LVT_DM_EXTINT;
		if (ioapic_enable)
			temp |= APIC_LVT_MASKED;
	} else {
		temp |= APIC_LVT_DM_FIXED | APIC_LVT_MASKED;
	}
	LAPIC_WRITE(lvt_lint0, temp);

	/*
	 * Setup LINT1 as NMI.
	 *
	 * Must be setup edge trigger, active high.
	 *
	 * Enable LINT1 on BSP, if I/O APIC is enabled.
	 *
	 * Disable LINT1 on the APs.
	 */
	temp = LAPIC_READ(lvt_lint1);
	temp &= ~(APIC_LVT_MASKED | APIC_LVT_TRIG_MASK |
		  APIC_LVT_POLARITY_MASK | APIC_LVT_DM_MASK);
	temp |= APIC_LVT_MASKED | APIC_LVT_DM_NMI;
	if (bsp && ioapic_enable)
		temp &= ~APIC_LVT_MASKED;
	LAPIC_WRITE(lvt_lint1, temp);

	/*
	 * Mask the LAPIC error interrupt, LAPIC performance counter
	 * interrupt.
	 */
	LAPIC_WRITE(lvt_error, LAPIC_READ(lvt_error) | APIC_LVT_MASKED);
	LAPIC_WRITE(lvt_pcint, LAPIC_READ(lvt_pcint) | APIC_LVT_MASKED);

	/*
	 * Set LAPIC timer vector and mask the LAPIC timer interrupt.
	 */
	timer = LAPIC_READ(lvt_timer);
	timer &= ~APIC_LVTT_VECTOR;
	timer |= XTIMER_OFFSET;
	timer |= APIC_LVTT_MASKED;
	LAPIC_WRITE(lvt_timer, timer);

	/*
	 * Set the Task Priority Register as needed.   At the moment allow
	 * interrupts on all cpus (the APs will remain CLId until they are
	 * ready to deal).
	 */
	temp = LAPIC_READ(tpr);
	temp &= ~APIC_TPR_PRIO;		/* clear priority field */
	LAPIC_WRITE(tpr, temp);

	/*
	 * AMD specific setup
	 */
	if (cpu_vendor_id == CPU_VENDOR_AMD && lapic_mem != NULL &&
	    (LAPIC_MEM_READ(version) & APIC_VER_AMD_EXT_SPACE)) {
		uint32_t ext_feat;
		uint32_t count;
		uint32_t max_count;
		uint32_t lvt;
		uint32_t i;

		ext_feat = LAPIC_MEM_READ(ext_feat);
		count = (ext_feat & APIC_EXTFEAT_MASK) >> APIC_EXTFEAT_SHIFT;
		max_count = sizeof(lapic_mem->ext_lvt) /
		    sizeof(lapic_mem->ext_lvt[0]);
		if (count > max_count)
			count = max_count;
		for (i = 0; i < count; ++i) {
			lvt = LAPIC_MEM_READ(ext_lvt[i].lvt);

			lvt &= ~(APIC_LVT_POLARITY_MASK | APIC_LVT_TRIG_MASK |
				 APIC_LVT_DM_MASK | APIC_LVT_MASKED);
			lvt |= APIC_LVT_MASKED | APIC_LVT_DM_FIXED;

			switch(i) {
			case APIC_EXTLVT_IBS:
				break;
			case APIC_EXTLVT_MCA:
				break;
			case APIC_EXTLVT_DEI:
				break;
			case APIC_EXTLVT_SBI:
				break;
			default:
				break;
			}
			if (bsp) {
				kprintf("   LAPIC AMD elvt%d: 0x%08x",
					i, LAPIC_MEM_READ(ext_lvt[i].lvt));
				if (LAPIC_MEM_READ(ext_lvt[i].lvt) != lvt)
					kprintf(" -> 0x%08x", lvt);
				kprintf("\n");
			}
			LAPIC_MEM_WRITE(ext_lvt[i].lvt, lvt);
		}
	}

	/*
	 * Enable the LAPIC
	 */
	temp = LAPIC_READ(svr);
	temp |= APIC_SVR_ENABLE;	/* enable the LAPIC */
	temp &= ~APIC_SVR_FOCUS_DISABLE; /* enable lopri focus processor */

	if (LAPIC_READ(version) & APIC_VER_EOI_SUPP) {
		if (temp & APIC_SVR_EOI_SUPP) {
			temp &= ~APIC_SVR_EOI_SUPP;
			if (bsp)
				kprintf("    LAPIC disabling EOI supp\n");
		}
		/* (future, on KVM auto-EOI must be disabled) */
		if (vmm_guest == VMM_GUEST_KVM)
			temp &= ~APIC_SVR_EOI_SUPP;
	}

	/*
	 * Set the spurious interrupt vector.  The low 4 bits of the vector
	 * must be 1111.
	 */
	if ((XSPURIOUSINT_OFFSET & 0x0F) != 0x0F)
		panic("bad XSPURIOUSINT_OFFSET: 0x%08x", XSPURIOUSINT_OFFSET);
	temp &= ~APIC_SVR_VECTOR;
	temp |= XSPURIOUSINT_OFFSET;

	LAPIC_WRITE(svr, temp);

	/*
	 * Pump out a few EOIs to clean out interrupts that got through
	 * before we were able to set the TPR.
	 */
	LAPIC_WRITE(eoi, 0);
	LAPIC_WRITE(eoi, 0);
	LAPIC_WRITE(eoi, 0);

	if (bsp) {
		lapic_timer_calibrate();
		if (lapic_timer_enable) {
			if (cpu_thermal_feature & CPUID_THERMAL_ARAT) {
				/*
				 * Local APIC timer will not stop
				 * in deep C-state.
				 */
				lapic_cputimer_intr.caps |=
				    CPUTIMER_INTR_CAP_PS;
			}
			if (lapic_use_tscdeadline) {
				lapic_cputimer_intr.reload =
				    lapic_timer_tscdlt_reload;
			}
			cputimer_intr_register(&lapic_cputimer_intr);
			cputimer_intr_select(&lapic_cputimer_intr, 0);
		}
	} else if (!lapic_use_tscdeadline) {
		lapic_timer_set_divisor(lapic_timer_divisor_idx);
	}

	if (bootverbose)
		apic_dump("apic_initialize()");
}

static void
lapic_timer_set_divisor(int divisor_idx)
{
	KKASSERT(divisor_idx >= 0 && divisor_idx < APIC_TIMER_NDIVISORS);
	LAPIC_WRITE(dcr_timer, lapic_timer_divisors[divisor_idx]);
}

static void
lapic_timer_oneshot(u_int count)
{
	uint32_t value;

	value = LAPIC_READ(lvt_timer);
	value &= ~(APIC_LVTT_PERIODIC | APIC_LVTT_TSCDLT);
	LAPIC_WRITE(lvt_timer, value);
	LAPIC_WRITE(icr_timer, count);
}

static void
lapic_timer_oneshot_quick(u_int count)
{
	LAPIC_WRITE(icr_timer, count);
}

static void
lapic_timer_tscdeadline_quick(uint64_t diff)
{
	uint64_t val = rdtsc() + diff;

	wrmsr(MSR_TSC_DEADLINE, val);
	tsc_deadlines[mycpuid].timestamp = val;
}

static uint64_t
lapic_scale_to_tsc(unsigned value, unsigned scale)
{
	uint64_t val;

	val = value;
	val *= tsc_frequency;
	val += (scale - 1);
	val /= scale;
	return val;
}

#define MAX_MEASURE_RETRIES	100

static u_int64_t
do_tsc_calibration(u_int us, u_int64_t apic_delay_tsc)
{
	u_int64_t old_tsc1, old_tsc2, new_tsc1, new_tsc2;
	u_int64_t diff, count;
	u_int64_t a;
	u_int32_t start, end;
	int retries1 = 0, retries2 = 0;

retry1:
	lapic_timer_oneshot_quick(APIC_TIMER_MAX_COUNT);
	old_tsc1 = rdtsc_ordered();
	start = LAPIC_READ(ccr_timer);
	old_tsc2 = rdtsc_ordered();
	if (apic_delay_tsc > 0 && retries1 < MAX_MEASURE_RETRIES &&
	    old_tsc2 - old_tsc1 > 2 * apic_delay_tsc) {
		retries1++;
		goto retry1;
	}
	DELAY(us);
retry2:
	new_tsc1 = rdtsc_ordered();
	end = LAPIC_READ(ccr_timer);
	new_tsc2 = rdtsc_ordered();
	if (apic_delay_tsc > 0 && retries2 < MAX_MEASURE_RETRIES &&
	    new_tsc2 - new_tsc1 > 2 * apic_delay_tsc) {
		retries2++;
		goto retry2;
	}
	if (end == 0)
		return 0;

	count = start - end;

	/* Make sure the lapic can count for up to 2s */
	a = (unsigned)APIC_TIMER_MAX_COUNT;
	if (us < 2000000 && (u_int64_t)count * 2000000 >= a * us)
		return 0;

	if (lapic_calibrate_test > 0 && (retries1 > 0 || retries2 > 0)) {
		kprintf("%s: retries1=%d retries2=%d\n",
		    __func__, retries1, retries2);
	}

	diff = (new_tsc1 - old_tsc1) + (new_tsc2 - old_tsc2);
	/* XXX First estimate if the total TSC diff value makes sense */
	/* This will almost overflow, but only almost :) */
	count = (2 * count * tsc_frequency) / diff;

	return count;
}

static uint64_t
do_cputimer_calibration(u_int us)
{
	sysclock_t value;
	sysclock_t start, end;
	uint32_t beginning, finish;

	lapic_timer_oneshot(APIC_TIMER_MAX_COUNT);
	beginning = LAPIC_READ(ccr_timer);
	start = sys_cputimer->count();
	DELAY(us);
	end = sys_cputimer->count();
	finish = LAPIC_READ(ccr_timer);
	if (finish == 0)
		return 0;
	/* value is the LAPIC timer difference. */
	value = (uint32_t)(beginning - finish);
	/* end is the sys_cputimer difference. */
	end -= start;
	if (end == 0)
		return 0;
	value = muldivu64(value, sys_cputimer->freq, end);

	return value;
}

static void
lapic_timer_calibrate(void)
{
	sysclock_t value;
	u_int64_t apic_delay_tsc = 0;
	int use_tsc_calibration = 0;

	/* No need to calibrate lapic_timer, if we will use TSC Deadline mode */
	if (lapic_use_tscdeadline) {
		lapic_cputimer_intr.freq = tsc_frequency;
		kprintf(
		    "lapic: TSC Deadline Mode: frequency %lu Hz\n",
		    lapic_cputimer_intr.freq);
		return;
	}

	/*
	 * On real hardware, tsc_invariant == 0 wouldn't be an issue, but in
	 * a virtual machine the frequency may get changed by the host.
	 */
	if (tsc_frequency != 0 && tsc_invariant && lapic_calibrate_fast)
		use_tsc_calibration = 1;

	if (use_tsc_calibration) {
		u_int64_t min_apic_tsc = 0, max_apic_tsc = 0;
		u_int64_t old_tsc, new_tsc;
		uint32_t val;
		int i;

		/* warm up */
		lapic_timer_oneshot(APIC_TIMER_MAX_COUNT);
		for (i = 0; i < 10; i++)
			val = LAPIC_READ(ccr_timer);

		for (i = 0; i < 100; i++) {
			old_tsc = rdtsc_ordered();
			val = LAPIC_READ(ccr_timer);
			new_tsc = rdtsc_ordered();
			new_tsc -= old_tsc;
			apic_delay_tsc += new_tsc;
			if (min_apic_tsc == 0 ||
			    min_apic_tsc > new_tsc) {
				min_apic_tsc = new_tsc;
			}
			if (max_apic_tsc < new_tsc)
				max_apic_tsc = new_tsc;
		}
		apic_delay_tsc /= 100;
		kprintf(
		    "LAPIC latency (in TSC ticks): %lu min: %lu max: %lu\n",
		    apic_delay_tsc, min_apic_tsc, max_apic_tsc);
		apic_delay_tsc = min_apic_tsc;
	}

	if (!use_tsc_calibration) {
		int i;

		/*
		 * Do some exercising of the lapic timer access. This improves
		 * precision of the subsequent calibration run in at least some
		 * virtualization cases.
		 */
		lapic_timer_set_divisor(0);
		for (i = 0; i < 10; i++)
			(void)do_cputimer_calibration(100);
	}
	/* Try to calibrate the local APIC timer. */
	for (lapic_timer_divisor_idx = 0;
	     lapic_timer_divisor_idx < APIC_TIMER_NDIVISORS;
	     lapic_timer_divisor_idx++) {
		lapic_timer_set_divisor(lapic_timer_divisor_idx);
		if (use_tsc_calibration) {
			value = do_tsc_calibration(200*1000, apic_delay_tsc);
		} else {
			value = do_cputimer_calibration(2*1000*1000);
		}
		if (value != 0)
			break;
	}
	if (lapic_timer_divisor_idx >= APIC_TIMER_NDIVISORS)
		panic("lapic: no proper timer divisor?!");
	lapic_cputimer_intr.freq = value;

	kprintf("lapic: divisor index %d, frequency %lu Hz\n",
		lapic_timer_divisor_idx, lapic_cputimer_intr.freq);

	if (lapic_calibrate_test > 0) {
		uint64_t freq;
		int i;

		for (i = 1; i <= 20; i++) {
			if (use_tsc_calibration) {
				freq = do_tsc_calibration(i*100*1000,
							  apic_delay_tsc);
			} else {
				freq = do_cputimer_calibration(i*100*1000);
			}
			if (freq != 0)
				kprintf("%ums: %lu\n", i * 100, freq);
		}
	}
}

static void
lapic_timer_tscdlt_reload(struct cputimer_intr *cti, sysclock_t reload)
{
	struct globaldata *gd = mycpu;
	uint64_t diff, now, val;

	/*
	 * Set maximum deadline to 60 seconds
	 */
	if (reload > sys_cputimer->freq * 60)
		reload = sys_cputimer->freq * 60;
	diff = muldivu64(reload, tsc_frequency, sys_cputimer->freq);
	if (diff < 4)
		diff = 4;
	if (cpu_vendor_id == CPU_VENDOR_INTEL)
		cpu_lfence();
	else
		cpu_mfence();
	now = rdtsc();
	val = now + diff;
	if (gd->gd_timer_running) {
		uint64_t deadline = tsc_deadlines[mycpuid].timestamp;
		if (deadline == 0 || now > deadline || val < deadline) {
			wrmsr(MSR_TSC_DEADLINE, val);
			tsc_deadlines[mycpuid].timestamp = val;
		}
	} else {
		gd->gd_timer_running = 1;
		wrmsr(MSR_TSC_DEADLINE, val);
		tsc_deadlines[mycpuid].timestamp = val;
	}
}

static void
lapic_mem_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload)
{
	struct globaldata *gd = mycpu;

	if ((ssysclock_t)reload < 0)
		reload = 1;
	reload = muldivu64(reload, cti->freq, sys_cputimer->freq);
	if (reload < 2)
		reload = 2;
	if (reload > 0xFFFFFFFF)
		reload = 0xFFFFFFFF;

	if (gd->gd_timer_running) {
		if (reload < LAPIC_MEM_READ(ccr_timer))
			LAPIC_MEM_WRITE(icr_timer, (uint32_t)reload);
	} else {
		gd->gd_timer_running = 1;
		LAPIC_MEM_WRITE(icr_timer, (uint32_t)reload);
	}
}

static void
lapic_msr_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload)
{
	struct globaldata *gd = mycpu;

	if ((ssysclock_t)reload < 0)
		reload = 1;
	reload = muldivu64(reload, cti->freq, sys_cputimer->freq);
	if (reload < 2)
		reload = 2;
	if (reload > 0xFFFFFFFF)
		reload = 0xFFFFFFFF;

	if (gd->gd_timer_running) {
		if (reload < LAPIC_MSR_READ(MSR_X2APIC_CCR_TIMER))
			LAPIC_MSR_WRITE(MSR_X2APIC_ICR_TIMER, (uint32_t)reload);
	} else {
		gd->gd_timer_running = 1;
		LAPIC_MSR_WRITE(MSR_X2APIC_ICR_TIMER, (uint32_t)reload);
	}
}

static void
lapic_timer_intr_enable(struct cputimer_intr *cti __unused)
{
	uint32_t timer;

	timer = LAPIC_READ(lvt_timer);
	timer &= ~(APIC_LVTT_MASKED | APIC_LVTT_PERIODIC | APIC_LVTT_TSCDLT);
	if (lapic_use_tscdeadline)
		timer |= APIC_LVTT_TSCDLT;
	LAPIC_WRITE(lvt_timer, timer);
	if (lapic_use_tscdeadline)
		cpu_mfence();

	lapic_timer_fixup_handler(NULL);
}

static void
lapic_timer_fixup_handler(void *arg)
{
	int *started = arg;

	if (started != NULL)
		*started = 0;

	if (cpu_vendor_id == CPU_VENDOR_AMD) {
		int c1e_test = lapic_timer_c1e_test;

		if (c1e_test < 0) {
			if (vmm_guest == VMM_GUEST_NONE) {
				c1e_test = 1;
			} else {
				/*
				 * Don't do this C1E testing and adjustment
				 * on virtual machines, the best case for
				 * accessing this MSR is a NOOP; the worst
				 * cases could be pretty nasty, e.g. crash.
				 */
				c1e_test = 0;
			}
		}

		/*
		 * Detect the presence of C1E capability mostly on latest
		 * dual-cores (or future) k8 family.  This feature renders
		 * the local APIC timer dead, so we disable it by reading
		 * the Interrupt Pending Message register and clearing both
		 * C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27).
		 *
		 * Reference:
		 *   "BIOS and Kernel Developer's Guide for AMD NPT
		 *    Family 0Fh Processors"
		 *   #32559 revision 3.00
		 */
		if ((cpu_id & 0x00000f00) == 0x00000f00 &&
		    (cpu_id & 0x0fff0000) >= 0x00040000 &&
		    c1e_test) {
			uint64_t msr;

			msr = rdmsr(0xc0010055);
			if (msr & 0x18000000) {
				struct globaldata *gd = mycpu;

				kprintf("cpu%d: AMD C1E detected\n",
					gd->gd_cpuid);
				wrmsr(0xc0010055, msr & ~0x18000000ULL);

				/*
				 * We are kinda stalled;
				 * kick start again.
				 */
				gd->gd_timer_running = 1;
				if (lapic_use_tscdeadline) {
					/* Maybe reached in Virtual Machines? */
					lapic_timer_tscdeadline_quick(5000);
				} else {
					lapic_timer_oneshot_quick(2);
				}

				if (started != NULL)
					*started = 1;
			}
		}
	}
}

static void
lapic_timer_restart_handler(void *dummy __unused)
{
	int started;

	lapic_timer_fixup_handler(&started);
	if (!started) {
		struct globaldata *gd = mycpu;

		gd->gd_timer_running = 1;
		if (lapic_use_tscdeadline) {
			/* Maybe reached in Virtual Machines? */
			lapic_timer_tscdeadline_quick(5000);
		} else {
			lapic_timer_oneshot_quick(2);
		}
	}
}

/*
 * This function is called only by ACPICA code currently:
 * - AMD C1E fixup.  AMD C1E only seems to happen after ACPI
 *   module controls PM.  So once ACPICA is attached, we try
 *   to apply the fixup to prevent LAPIC timer from hanging.
 */
static void
lapic_timer_intr_pmfixup(struct cputimer_intr *cti __unused)
{
	lwkt_send_ipiq_mask(smp_active_mask,
			    lapic_timer_fixup_handler, NULL);
}

static void
lapic_timer_intr_restart(struct cputimer_intr *cti __unused)
{
	lwkt_send_ipiq_mask(smp_active_mask, lapic_timer_restart_handler, NULL);
}


/*
 * dump contents of local APIC registers
 */
void
apic_dump(char* str)
{
	kprintf("SMP: CPU%d %s:\n", mycpu->gd_cpuid, str);
	kprintf("     lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
		LAPIC_READ(lvt_lint0), LAPIC_READ(lvt_lint1), LAPIC_READ(tpr),
		LAPIC_READ(svr));
}

/*
 * Inter Processor Interrupt functions.
 */

static __inline void
lapic_mem_icr_unpend(const char *func)
{
	if (LAPIC_MEM_READ(icr_lo) & APIC_DELSTAT_PEND) {
		int64_t tsc;
		int loops = 1;

		tsc = rdtsc();
		while (LAPIC_MEM_READ(icr_lo) & APIC_DELSTAT_PEND) {
			cpu_pause();
			if ((tsc_sclock_t)(rdtsc() -
					   (tsc + tsc_frequency)) > 0) {
				tsc = rdtsc();
				if (++loops > 30) {
					panic("%s: cpu%d apic stalled",
					    func, mycpuid);
				} else {
					kprintf("%s: cpu%d apic stalled\n",
					    func, mycpuid);
				}
			}
		}
	}
}

/*
 * Send APIC IPI 'vector' to 'destType' via 'deliveryMode'.
 *
 *  destType is 1 of: APIC_DEST_SELF, APIC_DEST_ALLISELF, APIC_DEST_ALLESELF
 *  vector is any valid SYSTEM INT vector
 *  delivery_mode is 1 of: APIC_DELMODE_FIXED, APIC_DELMODE_LOWPRIO
 *
 * WARNINGS!
 *
 * We now implement a per-cpu interlock (gd->gd_npoll) to prevent more than
 * one IPI from being sent to any given cpu at a time.  Thus we no longer
 * have to process incoming IPIs while waiting for the status to clear.
 * No deadlock should be possible.
 *
 * We now physically disable interrupts for the lapic ICR operation.  If
 * we do not do this then it looks like an EOI sent to the lapic (which
 * occurs even with a critical section) can interfere with the command
 * register ready status and cause an IPI to be lost.
 *
 * e.g. an interrupt can occur, issue the EOI, IRET, and cause the command
 * register to busy just before we write to icr_lo, resulting in a lost
 * issuance.  This only appears to occur on Intel cpus and is not
 * documented.  It could simply be that cpus are so fast these days that
 * it was always an issue, but is only now rearing its ugly head.  This
 * is conjecture.
 */
static int
lapic_mem_ipi(int dest_type, int vector, int delivery_mode)
{
	lapic_mem_icr_unpend(__func__);
	lapic_mem_icr_set(0,
	    dest_type | APIC_LEVEL_ASSERT | delivery_mode | vector);
	return 0;
}

static int
lapic_msr_ipi(int dest_type, int vector, int delivery_mode)
{
	lapic_msr_icr_set(0,
	    dest_type | APIC_LEVEL_ASSERT | delivery_mode | vector);
	return 0;
}

/*
 * Interrupts must be hard-disabled by caller
 */
static void
lapic_mem_single_ipi(int cpu, int vector, int delivery_mode)
{
	lapic_mem_icr_unpend(__func__);
	lapic_mem_icr_set(CPUID_TO_APICID(cpu),
	    APIC_DEST_DESTFLD | APIC_LEVEL_ASSERT | delivery_mode | vector);
}

static void
lapic_msr_single_ipi(int cpu, int vector, int delivery_mode)
{
	lapic_msr_icr_set(CPUID_TO_APICID(cpu),
	    APIC_DEST_DESTFLD | APIC_LEVEL_ASSERT | delivery_mode | vector);
}

/*
 * Send APIC IPI 'vector' to 'target's via 'delivery_mode'.
 *
 * target is a bitmask of destination cpus.  Vector is any
 * valid system INT vector.  Delivery mode may be either
 * APIC_DELMODE_FIXED or APIC_DELMODE_LOWPRIO.
 *
 * Interrupts must be hard-disabled by caller
 */
void
selected_apic_ipi(cpumask_t target, int vector, int delivery_mode)
{
	while (CPUMASK_TESTNZERO(target)) {
		int n = BSFCPUMASK(target);
		CPUMASK_NANDBIT(target, n);
		single_apic_ipi(n, vector, delivery_mode);
	}
}

/*
 * Load a 'downcount time' in uSeconds.
 */
void
set_apic_timer(int us)
{
	u_int count;

	if (lapic_use_tscdeadline) {
		uint64_t val;

		val = lapic_scale_to_tsc(us, 1000000);
		val += rdtsc();
		/* No need to arm the lapic here, just track the timeout. */
		tsc_deadlines[mycpuid].downcount_time = val;
		return;
	}

	/*
	 * When we reach here, lapic timer's frequency
	 * must have been calculated as well as the
	 * divisor (lapic->dcr_timer is setup during the
	 * divisor calculation).
	 */
	KKASSERT(lapic_cputimer_intr.freq != 0 &&
		 lapic_timer_divisor_idx >= 0);

	count = ((us * (int64_t)lapic_cputimer_intr.freq) + 999999) / 1000000;
	lapic_timer_oneshot(count);
}


/*
 * Read remaining time in timer, in microseconds (rounded up).
 */
int
read_apic_timer(void)
{
	uint64_t val;

	if (lapic_use_tscdeadline) {
		uint64_t now;

		val = tsc_deadlines[mycpuid].downcount_time;
		now = rdtsc();
		if (val == 0 || now > val) {
			return 0;
		} else {
			val -= now;
			val *= 1000000;
			val += (tsc_frequency - 1);
			val /= tsc_frequency;
			if (val > INT_MAX)
				val = INT_MAX;
			return val;
		}
	}

	val = LAPIC_READ(ccr_timer);
	if (val == 0)
		return 0;

	KKASSERT(lapic_cputimer_intr.freq > 0);
	val *= 1000000;
	val += (lapic_cputimer_intr.freq - 1);
	val /= lapic_cputimer_intr.freq;
	if (val > INT_MAX)
		val = INT_MAX;
	return val;
}


/*
 * Spin-style delay, set delay time in uS, spin till it drains.
 */
void
u_sleep(int count)
{
	set_apic_timer(count);
	while (read_apic_timer())
		 /* spin */ ;
}

int
lapic_unused_apic_id(int start)
{
	int i;

	for (i = start; i < APICID_MAX; ++i) {
		if (APICID_TO_CPUID(i) == -1)
			return i;
	}
	return NAPICID;
}

void
lapic_map(vm_paddr_t lapic_addr)
{
	lapic_mem = pmap_mapdev_uncacheable(lapic_addr, sizeof(struct LAPIC));
}

void
lapic_x2apic_enter(boolean_t bsp)
{
	uint64_t apic_base;

	KASSERT(x2apic_enable, ("X2APIC mode is not enabled"));

	/*
	 * X2APIC mode is requested, if it has not been enabled by the BIOS,
	 * enable it now.
	 */
	apic_base = rdmsr(MSR_APICBASE);
	if ((apic_base & APICBASE_X2APIC) == 0) {
		wrmsr(MSR_APICBASE,
		    apic_base | APICBASE_X2APIC | APICBASE_ENABLED);
	}
	if (bsp) {
		lapic_eoi = lapic_msr_eoi;
		apic_ipi = lapic_msr_ipi;
		single_apic_ipi = lapic_msr_single_ipi;
		lapic_cputimer_intr.reload = lapic_msr_timer_intr_reload;
	}
}

static TAILQ_HEAD(, lapic_enumerator) lapic_enumerators =
	TAILQ_HEAD_INITIALIZER(lapic_enumerators);

int
lapic_config(void)
{
	struct lapic_enumerator *e;
	uint64_t apic_base;
	int error, i, ap_max;

	KKASSERT(lapic_enable);

	lapic_eoi = lapic_mem_eoi;
	apic_ipi = lapic_mem_ipi;
	single_apic_ipi = lapic_mem_single_ipi;

	TUNABLE_INT_FETCH("hw.x2apic_enable", &x2apic_enable);
	if (x2apic_enable < 0)
		x2apic_enable = 1;
	if ((cpu_feature2 & CPUID2_X2APIC) == 0) {
		/* X2APIC is not supported. */
		x2apic_enable = 0;
	} else {
		/*
		 * If the BIOS enabled the X2APIC mode, then we would stick
		 * with the X2APIC mode.
		 */
		apic_base = rdmsr(MSR_APICBASE);
		if (apic_base & APICBASE_X2APIC) {
			if (x2apic_enable == 0)
				kprintf("LAPIC: BIOS enabled X2APIC mode, force on\n");
			else
				kprintf("LAPIC: BIOS enabled X2APIC mode\n");
			x2apic_enable = 1;
		}
	}
	if (cpu_feature2 & CPUID2_X2APIC) {
		apic_base = rdmsr(MSR_APICBASE);
		if (apic_base & APICBASE_X2APIC)
			kprintf("LAPIC: BIOS already enabled X2APIC mode\n");
	}

	if (x2apic_enable) {
		/*
		 * Enter X2APIC mode.
		 */
		kprintf("LAPIC: enter X2APIC mode\n");
		lapic_x2apic_enter(TRUE);
	}

	for (i = 0; i < NAPICID; ++i)
		APICID_TO_CPUID(i) = -1;

	TAILQ_FOREACH(e, &lapic_enumerators, lapic_link) {
		error = e->lapic_probe(e);
		if (!error)
			break;
	}
	if (e == NULL) {
		kprintf("LAPIC: Can't find LAPIC\n");
		return ENXIO;
	}

	error = e->lapic_enumerate(e);
	if (error) {
		kprintf("LAPIC: enumeration failed\n");
		return ENXIO;
	}

	/* LAPIC is usable now. */
	lapic_usable = 1;

	ap_max = MAXCPU - 1;
	TUNABLE_INT_FETCH("hw.ap_max", &ap_max);
	if (ap_max > MAXCPU - 1)
		ap_max = MAXCPU - 1;

	if (naps > ap_max) {
		kprintf("LAPIC: Warning use only %d out of %d "
			"available APs\n",
			ap_max, naps);
		naps = ap_max;
	}

	return 0;
}

void
lapic_enumerator_register(struct lapic_enumerator *ne)
{
	struct lapic_enumerator *e;

	TAILQ_FOREACH(e, &lapic_enumerators, lapic_link) {
		if (e->lapic_prio < ne->lapic_prio) {
			TAILQ_INSERT_BEFORE(e, ne, lapic_link);
			return;
		}
	}
	TAILQ_INSERT_TAIL(&lapic_enumerators, ne, lapic_link);
}

void
lapic_set_cpuid(int cpu_id, int apic_id)
{
	CPUID_TO_APICID(cpu_id) = apic_id;
	APICID_TO_CPUID(apic_id) = cpu_id;
}

void
lapic_fixup_noioapic(void)
{
	u_int   temp;

	/* Only allowed on BSP */
	KKASSERT(mycpuid == 0);
	KKASSERT(!ioapic_enable);

	temp = LAPIC_READ(lvt_lint0);
	temp &= ~APIC_LVT_MASKED;
	LAPIC_WRITE(lvt_lint0, temp);

	temp = LAPIC_READ(lvt_lint1);
	temp |= APIC_LVT_MASKED;
	LAPIC_WRITE(lvt_lint1, temp);
}

static void
lapic_mem_eoi(void)
{
	log_lapic(mem_eoi);
	LAPIC_MEM_WRITE(eoi, 0);
}

static void
lapic_msr_eoi(void)
{
	log_lapic(msr_eoi);
	LAPIC_MSR_WRITE(MSR_X2APIC_EOI, 0);
}

static void
lapic_mem_seticr_sync(uint32_t apic_id, uint32_t icr_lo_val)
{
	lapic_mem_icr_set(apic_id, icr_lo_val);
	while (LAPIC_MEM_READ(icr_lo) & APIC_DELSTAT_PEND)
		/* spin */;
}

void
lapic_seticr_sync(uint32_t apic_id, uint32_t icr_lo_val)
{
	if (x2apic_enable)
		lapic_msr_icr_set(apic_id, icr_lo_val);
	else
		lapic_mem_seticr_sync(apic_id, icr_lo_val);
}

static void
lapic_sysinit(void *dummy __unused)
{
	if (lapic_enable) {
		int error;

		error = lapic_config();
		if (error)
			lapic_enable = 0;
	}
	if (!lapic_enable)
		x2apic_enable = 0;

	if (lapic_enable) {
		/* Initialize BSP's local APIC */
		lapic_init(TRUE);
	} else if (ioapic_enable) {
		kprintf("IOAPIC disabled - lapic was not enabled\n");
		ioapic_enable = 0;
		icu_reinit_noioapic();
	}
}
SYSINIT(lapic, SI_BOOT2_LAPIC, SI_ORDER_FIRST, lapic_sysinit, NULL);