1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 /*
26  * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27  */
28 
29 #include <sys/cpuvar.h>
30 #include <sys/psm.h>
31 #include <sys/archsystm.h>
32 #include <sys/apic.h>
33 #include <sys/sunddi.h>
34 #include <sys/ddi_impldefs.h>
35 #include <sys/mach_intr.h>
36 #include <sys/sysmacros.h>
37 #include <sys/trap.h>
38 #include <sys/x86_archext.h>
39 #include <sys/privregs.h>
40 #include <sys/psm_common.h>
41 
42 /* Function prototypes of local apic and X2APIC */
43 static uint64_t local_apic_read(uint32_t reg);
44 static void local_apic_write(uint32_t reg, uint64_t value);
45 static int get_local_apic_pri(void);
46 static void local_apic_write_task_reg(uint64_t value);
47 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
48 static uint64_t local_x2apic_read(uint32_t msr);
49 static void local_x2apic_write(uint32_t msr, uint64_t value);
50 static int get_local_x2apic_pri(void);
51 static void local_x2apic_write_task_reg(uint64_t value);
52 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
53 
54 /*
55  * According to the X2APIC specification:
56  *
57  *   xAPIC global enable    X2APIC enable         Description
58  *   (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
59  * -----------------------------------------------------------
60  *      0 			0 	APIC is disabled
61  * 	0			1	Invalid
62  *	1			0	APIC is enabled in xAPIC mode
63  *	1			1	APIC is enabled in X2APIC mode
64  * -----------------------------------------------------------
65  */
66 int	x2apic_enable = 1;
67 apic_mode_t apic_mode = LOCAL_APIC;	/* Default mode is Local APIC */
68 
69 /* Uses MMIO (Memory Mapped IO) */
70 static apic_reg_ops_t local_apic_regs_ops = {
71 	local_apic_read,
72 	local_apic_write,
73 	get_local_apic_pri,
74 	local_apic_write_task_reg,
75 	local_apic_write_int_cmd,
76 	apic_send_EOI,
77 };
78 
79 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
80 static apic_reg_ops_t x2apic_regs_ops = {
81 	local_x2apic_read,
82 	local_x2apic_write,
83 	get_local_x2apic_pri,
84 	local_x2apic_write_task_reg,
85 	local_x2apic_write_int_cmd,
86 	apic_send_EOI,
87 };
88 
89 int apic_have_32bit_cr8 = 0;
90 
91 /* The default ops is local APIC (Memory Mapped IO) */
92 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops;
93 
94 /*
95  * APIC register ops related data sturctures and functions.
96  */
97 void apic_send_EOI();
98 void apic_send_directed_EOI(uint32_t irq);
99 
100 #define	X2APIC_ENABLE_BIT	10
101 
102 /*
103  * Local APIC Implementation
104  */
105 static uint64_t
106 local_apic_read(uint32_t reg)
107 {
108 	return ((uint32_t)apicadr[reg]);
109 }
110 
111 static void
112 local_apic_write(uint32_t reg, uint64_t value)
113 {
114 	apicadr[reg] = (uint32_t)value;
115 }
116 
117 static int
118 get_local_apic_pri(void)
119 {
120 #if defined(__amd64)
121 	return ((int)getcr8());
122 #else
123 	if (apic_have_32bit_cr8)
124 		return ((int)getcr8());
125 	return (apicadr[APIC_TASK_REG]);
126 #endif
127 }
128 
129 static void
130 local_apic_write_task_reg(uint64_t value)
131 {
132 #if defined(__amd64)
133 	setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
134 #else
135 	if (apic_have_32bit_cr8)
136 		setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
137 	else
138 		apicadr[APIC_TASK_REG] = (uint32_t)value;
139 #endif
140 }
141 
142 static void
143 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
144 {
145 	apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
146 	apicadr[APIC_INT_CMD1] = cmd1;
147 }
148 
149 /*
150  * X2APIC Implementation.
151  */
152 static uint64_t
153 local_x2apic_read(uint32_t msr)
154 {
155 	uint64_t i;
156 
157 	i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
158 	return (i);
159 }
160 
161 static void
162 local_x2apic_write(uint32_t msr, uint64_t value)
163 {
164 	uint64_t tmp;
165 
166 	if (msr != APIC_EOI_REG) {
167 		tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
168 		tmp = (tmp & 0xffffffff00000000) | value;
169 	} else {
170 		tmp = 0;
171 	}
172 
173 	wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
174 }
175 
176 static int
177 get_local_x2apic_pri(void)
178 {
179 	return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
180 }
181 
182 static void
183 local_x2apic_write_task_reg(uint64_t value)
184 {
185 	X2APIC_WRITE(APIC_TASK_REG, value);
186 }
187 
188 static void
189 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
190 {
191 	wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
192 	    (((uint64_t)cpu_id << 32) | cmd1));
193 }
194 
195 /*ARGSUSED*/
196 void
197 apic_send_EOI(uint32_t irq)
198 {
199 	apic_reg_ops->apic_write(APIC_EOI_REG, 0);
200 }
201 
202 /*
203  * Support for Directed EOI capability is available in both the xAPIC
204  * and x2APIC mode.
205  */
206 void
207 apic_send_directed_EOI(uint32_t irq)
208 {
209 	uchar_t ioapicindex;
210 	uchar_t vector;
211 	apic_irq_t *apic_irq;
212 	short intr_index;
213 
214 	/*
215 	 * Following the EOI to the local APIC unit, perform a directed
216 	 * EOI to the IOxAPIC generating the interrupt by writing to its
217 	 * EOI register.
218 	 *
219 	 * A broadcast EOI is not generated.
220 	 */
221 	apic_reg_ops->apic_write(APIC_EOI_REG, 0);
222 
223 	apic_irq = apic_irq_table[irq];
224 	while (apic_irq) {
225 		intr_index = apic_irq->airq_mps_intr_index;
226 		if (intr_index == ACPI_INDEX || intr_index >= 0) {
227 			ioapicindex = apic_irq->airq_ioapicindex;
228 			vector = apic_irq->airq_vector;
229 			ioapic_write_eoi(ioapicindex, vector);
230 		}
231 		apic_irq = apic_irq->airq_next;
232 	}
233 }
234 
235 int
236 apic_detect_x2apic(void)
237 {
238 	if (x2apic_enable == 0)
239 		return (0);
240 
241 	return (is_x86_feature(x86_featureset, X86FSET_X2APIC));
242 }
243 
244 void
245 apic_enable_x2apic(void)
246 {
247 	uint64_t apic_base_msr;
248 
249 	if (apic_local_mode() == LOCAL_X2APIC) {
250 		/* BIOS apparently has enabled X2APIC */
251 		if (apic_mode != LOCAL_X2APIC)
252 			x2apic_update_psm();
253 		return;
254 	}
255 
256 	/*
257 	 * This is the first time we are enabling X2APIC on this CPU
258 	 */
259 	apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
260 	apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
261 	wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
262 
263 	if (apic_mode != LOCAL_X2APIC)
264 		x2apic_update_psm();
265 }
266 
267 /*
268  * Determine which mode the current CPU is in. See the table above.
269  * (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
270  */
271 int
272 apic_local_mode(void)
273 {
274 	uint64_t apic_base_msr;
275 	int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) |
276 	    (0x1 << X2APIC_ENABLE_BIT));
277 
278 	apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
279 
280 	if ((apic_base_msr & bit) == bit)
281 		return (LOCAL_X2APIC);
282 	else
283 		return (LOCAL_APIC);
284 }
285 
286 void
287 apic_set_directed_EOI_handler()
288 {
289 	apic_reg_ops->apic_send_eoi = apic_send_directed_EOI;
290 }
291 
292 int
293 apic_directed_EOI_supported()
294 {
295 	uint32_t ver;
296 
297 	ver = apic_reg_ops->apic_read(APIC_VERS_REG);
298 	if (ver & APIC_DIRECTED_EOI_BIT)
299 		return (1);
300 
301 	return (0);
302 }
303 
304 /*
305  * Change apic_reg_ops depending upon the apic_mode.
306  */
307 void
308 apic_change_ops()
309 {
310 	if (apic_mode == LOCAL_APIC)
311 		apic_reg_ops = &local_apic_regs_ops;
312 	else if (apic_mode == LOCAL_X2APIC)
313 		apic_reg_ops = &x2apic_regs_ops;
314 }
315 
316 /*
317  * Generates an interprocessor interrupt to another CPU when X2APIC mode is
318  * enabled.
319  */
320 void
321 x2apic_send_ipi(int cpun, int ipl)
322 {
323 	int vector;
324 	ulong_t flag;
325 
326 	ASSERT(apic_mode == LOCAL_X2APIC);
327 
328 	/*
329 	 * With X2APIC, Intel relaxed the semantics of the
330 	 * WRMSR instruction such that references to the X2APIC
331 	 * MSR registers are no longer serializing instructions.
332 	 * The code that initiates IPIs assumes that some sort
333 	 * of memory serialization occurs. The old APIC code
334 	 * did a write to uncachable memory mapped registers.
335 	 * Any reference to uncached memory is a serializing
336 	 * operation. To mimic those semantics here, we do an
337 	 * atomic operation, which translates to a LOCK OR instruction,
338 	 * which is serializing.
339 	 */
340 	atomic_or_ulong(&flag, 1);
341 
342 	vector = apic_resv_vector[ipl];
343 
344 	flag = intr_clear();
345 
346 	/*
347 	 * According to X2APIC specification in section '2.3.5.1' of
348 	 * Interrupt Command Register Semantics, the semantics of
349 	 * programming Interrupt Command Register to dispatch an interrupt
350 	 * is simplified. A single MSR write to the 64-bit ICR is required
351 	 * for dispatching an interrupt. Specifically with the 64-bit MSR
352 	 * interface to ICR, system software is not required to check the
353 	 * status of the delivery status bit prior to writing to the ICR
354 	 * to send an IPI. With the removal of the Delivery Status bit,
355 	 * system software no longer has a reason to read the ICR. It remains
356 	 * readable only to aid in debugging.
357 	 */
358 #ifdef	DEBUG
359 	APIC_AV_PENDING_SET();
360 #endif	/* DEBUG */
361 
362 	if ((cpun == psm_get_cpu_id())) {
363 		X2APIC_WRITE(X2APIC_SELF_IPI, vector);
364 	} else {
365 		apic_reg_ops->apic_write_int_cmd(
366 		    apic_cpus[cpun].aci_local_id, vector);
367 	}
368 
369 	intr_restore(flag);
370 }
371 
372 /*
373  * Generates IPI to another CPU depending on the local APIC mode.
374  * apic_send_ipi() and x2apic_send_ipi() depends on the configured
375  * mode of the local APIC, but that may not match the actual mode
376  * early in CPU startup.
377  *
378  * Any changes made to this routine must be accompanied by similar
379  * changes to apic_send_ipi().
380  */
381 void
382 apic_common_send_ipi(int cpun, int ipl)
383 {
384 	int vector;
385 	ulong_t flag;
386 	int mode = apic_local_mode();
387 
388 	if (mode == LOCAL_X2APIC) {
389 		x2apic_send_ipi(cpun, ipl);
390 		return;
391 	}
392 
393 	ASSERT(mode == LOCAL_APIC);
394 
395 	vector = apic_resv_vector[ipl];
396 	ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
397 	flag = intr_clear();
398 	while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
399 		apic_ret();
400 	local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
401 	    vector);
402 	intr_restore(flag);
403 }
404