xref: /linux/arch/x86/kernel/apic/io_apic.c (revision 62e303e3)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *	Intel IO-APIC support for multi-Pentium hosts.
4  *
5  *	Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
6  *
7  *	Many thanks to Stig Venaas for trying out countless experimental
8  *	patches and reporting/debugging problems patiently!
9  *
10  *	(c) 1999, Multiple IO-APIC support, developed by
11  *	Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
12  *      Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
13  *	further tested and cleaned up by Zach Brown <zab@redhat.com>
14  *	and Ingo Molnar <mingo@redhat.com>
15  *
16  *	Fixes
17  *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
18  *					thanks to Eric Gilmore
19  *					and Rolf G. Tews
20  *					for testing these extensively
21  *	Paul Diefenbaugh	:	Added full ACPI support
22  *
23  * Historical information which is worth to be preserved:
24  *
25  * - SiS APIC rmw bug:
26  *
27  *	We used to have a workaround for a bug in SiS chips which
28  *	required to rewrite the index register for a read-modify-write
29  *	operation as the chip lost the index information which was
30  *	setup for the read already. We cache the data now, so that
31  *	workaround has been removed.
32  */
33 
34 #include <linux/mm.h>
35 #include <linux/interrupt.h>
36 #include <linux/irq.h>
37 #include <linux/init.h>
38 #include <linux/delay.h>
39 #include <linux/sched.h>
40 #include <linux/pci.h>
41 #include <linux/mc146818rtc.h>
42 #include <linux/compiler.h>
43 #include <linux/acpi.h>
44 #include <linux/export.h>
45 #include <linux/syscore_ops.h>
46 #include <linux/freezer.h>
47 #include <linux/kthread.h>
48 #include <linux/jiffies.h>	/* time_after() */
49 #include <linux/slab.h>
50 #include <linux/memblock.h>
51 #include <linux/msi.h>
52 
53 #include <asm/irqdomain.h>
54 #include <asm/io.h>
55 #include <asm/smp.h>
56 #include <asm/cpu.h>
57 #include <asm/desc.h>
58 #include <asm/proto.h>
59 #include <asm/acpi.h>
60 #include <asm/dma.h>
61 #include <asm/timer.h>
62 #include <asm/time.h>
63 #include <asm/i8259.h>
64 #include <asm/setup.h>
65 #include <asm/irq_remapping.h>
66 #include <asm/hw_irq.h>
67 #include <asm/apic.h>
68 #include <asm/pgtable.h>
69 #include <asm/x86_init.h>
70 
71 #define	for_each_ioapic(idx)		\
72 	for ((idx) = 0; (idx) < nr_ioapics; (idx)++)
73 #define	for_each_ioapic_reverse(idx)	\
74 	for ((idx) = nr_ioapics - 1; (idx) >= 0; (idx)--)
75 #define	for_each_pin(idx, pin)		\
76 	for ((pin) = 0; (pin) < ioapics[(idx)].nr_registers; (pin)++)
77 #define	for_each_ioapic_pin(idx, pin)	\
78 	for_each_ioapic((idx))		\
79 		for_each_pin((idx), (pin))
80 #define for_each_irq_pin(entry, head) \
81 	list_for_each_entry(entry, &head, list)
82 
83 static DEFINE_RAW_SPINLOCK(ioapic_lock);
84 static DEFINE_MUTEX(ioapic_mutex);
85 static unsigned int ioapic_dynirq_base;
86 static int ioapic_initialized;
87 
88 struct irq_pin_list {
89 	struct list_head	list;
90 	int			apic, pin;
91 };
92 
93 struct mp_chip_data {
94 	struct list_head		irq_2_pin;
95 	struct IO_APIC_route_entry	entry;
96 	bool				is_level;
97 	bool				active_low;
98 	bool				isa_irq;
99 	u32				count;
100 };
101 
102 struct mp_ioapic_gsi {
103 	u32 gsi_base;
104 	u32 gsi_end;
105 };
106 
107 static struct ioapic {
108 	/* # of IRQ routing registers */
109 	int				nr_registers;
110 	/* Saved state during suspend/resume, or while enabling intr-remap. */
111 	struct IO_APIC_route_entry	*saved_registers;
112 	/* I/O APIC config */
113 	struct mpc_ioapic		mp_config;
114 	/* IO APIC gsi routing info */
115 	struct mp_ioapic_gsi		gsi_config;
116 	struct ioapic_domain_cfg	irqdomain_cfg;
117 	struct irq_domain		*irqdomain;
118 	struct resource			*iomem_res;
119 } ioapics[MAX_IO_APICS];
120 
121 #define mpc_ioapic_ver(ioapic_idx)	ioapics[ioapic_idx].mp_config.apicver
122 
mpc_ioapic_id(int ioapic_idx)123 int mpc_ioapic_id(int ioapic_idx)
124 {
125 	return ioapics[ioapic_idx].mp_config.apicid;
126 }
127 
mpc_ioapic_addr(int ioapic_idx)128 unsigned int mpc_ioapic_addr(int ioapic_idx)
129 {
130 	return ioapics[ioapic_idx].mp_config.apicaddr;
131 }
132 
mp_ioapic_gsi_routing(int ioapic_idx)133 static inline struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx)
134 {
135 	return &ioapics[ioapic_idx].gsi_config;
136 }
137 
mp_ioapic_pin_count(int ioapic)138 static inline int mp_ioapic_pin_count(int ioapic)
139 {
140 	struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic);
141 
142 	return gsi_cfg->gsi_end - gsi_cfg->gsi_base + 1;
143 }
144 
mp_pin_to_gsi(int ioapic,int pin)145 static inline u32 mp_pin_to_gsi(int ioapic, int pin)
146 {
147 	return mp_ioapic_gsi_routing(ioapic)->gsi_base + pin;
148 }
149 
mp_is_legacy_irq(int irq)150 static inline bool mp_is_legacy_irq(int irq)
151 {
152 	return irq >= 0 && irq < nr_legacy_irqs();
153 }
154 
mp_ioapic_irqdomain(int ioapic)155 static inline struct irq_domain *mp_ioapic_irqdomain(int ioapic)
156 {
157 	return ioapics[ioapic].irqdomain;
158 }
159 
160 int nr_ioapics;
161 
162 /* The one past the highest gsi number used */
163 u32 gsi_top;
164 
165 /* MP IRQ source entries */
166 struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
167 
168 /* # of MP IRQ source entries */
169 int mp_irq_entries;
170 
171 #ifdef CONFIG_EISA
172 int mp_bus_id_to_type[MAX_MP_BUSSES];
173 #endif
174 
175 DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
176 
177 bool ioapic_is_disabled __ro_after_init;
178 
179 /**
180  * disable_ioapic_support() - disables ioapic support at runtime
181  */
disable_ioapic_support(void)182 void disable_ioapic_support(void)
183 {
184 #ifdef CONFIG_PCI
185 	noioapicquirk = 1;
186 	noioapicreroute = -1;
187 #endif
188 	ioapic_is_disabled = true;
189 }
190 
parse_noapic(char * str)191 static int __init parse_noapic(char *str)
192 {
193 	/* disable IO-APIC */
194 	disable_ioapic_support();
195 	return 0;
196 }
197 early_param("noapic", parse_noapic);
198 
199 /* Will be called in mpparse/ACPI codes for saving IRQ info */
mp_save_irq(struct mpc_intsrc * m)200 void mp_save_irq(struct mpc_intsrc *m)
201 {
202 	int i;
203 
204 	apic_pr_verbose("Int: type %d, pol %d, trig %d, bus %02x, IRQ %02x, APIC ID %x, APIC INT %02x\n",
205 			m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus,
206 			m->srcbusirq, m->dstapic, m->dstirq);
207 
208 	for (i = 0; i < mp_irq_entries; i++) {
209 		if (!memcmp(&mp_irqs[i], m, sizeof(*m)))
210 			return;
211 	}
212 
213 	memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m));
214 	if (++mp_irq_entries == MAX_IRQ_SOURCES)
215 		panic("Max # of irq sources exceeded!!\n");
216 }
217 
alloc_ioapic_saved_registers(int idx)218 static void alloc_ioapic_saved_registers(int idx)
219 {
220 	size_t size;
221 
222 	if (ioapics[idx].saved_registers)
223 		return;
224 
225 	size = sizeof(struct IO_APIC_route_entry) * ioapics[idx].nr_registers;
226 	ioapics[idx].saved_registers = kzalloc(size, GFP_KERNEL);
227 	if (!ioapics[idx].saved_registers)
228 		pr_err("IOAPIC %d: suspend/resume impossible!\n", idx);
229 }
230 
free_ioapic_saved_registers(int idx)231 static void free_ioapic_saved_registers(int idx)
232 {
233 	kfree(ioapics[idx].saved_registers);
234 	ioapics[idx].saved_registers = NULL;
235 }
236 
arch_early_ioapic_init(void)237 int __init arch_early_ioapic_init(void)
238 {
239 	int i;
240 
241 	if (!nr_legacy_irqs())
242 		io_apic_irqs = ~0UL;
243 
244 	for_each_ioapic(i)
245 		alloc_ioapic_saved_registers(i);
246 
247 	return 0;
248 }
249 
250 struct io_apic {
251 	unsigned int index;
252 	unsigned int unused[3];
253 	unsigned int data;
254 	unsigned int unused2[11];
255 	unsigned int eoi;
256 };
257 
io_apic_base(int idx)258 static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
259 {
260 	return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
261 		+ (mpc_ioapic_addr(idx) & ~PAGE_MASK);
262 }
263 
io_apic_eoi(unsigned int apic,unsigned int vector)264 static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
265 {
266 	struct io_apic __iomem *io_apic = io_apic_base(apic);
267 
268 	writel(vector, &io_apic->eoi);
269 }
270 
native_io_apic_read(unsigned int apic,unsigned int reg)271 unsigned int native_io_apic_read(unsigned int apic, unsigned int reg)
272 {
273 	struct io_apic __iomem *io_apic = io_apic_base(apic);
274 
275 	writel(reg, &io_apic->index);
276 	return readl(&io_apic->data);
277 }
278 
io_apic_write(unsigned int apic,unsigned int reg,unsigned int value)279 static void io_apic_write(unsigned int apic, unsigned int reg,
280 			  unsigned int value)
281 {
282 	struct io_apic __iomem *io_apic = io_apic_base(apic);
283 
284 	writel(reg, &io_apic->index);
285 	writel(value, &io_apic->data);
286 }
287 
__ioapic_read_entry(int apic,int pin)288 static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin)
289 {
290 	struct IO_APIC_route_entry entry;
291 
292 	entry.w1 = io_apic_read(apic, 0x10 + 2 * pin);
293 	entry.w2 = io_apic_read(apic, 0x11 + 2 * pin);
294 
295 	return entry;
296 }
297 
ioapic_read_entry(int apic,int pin)298 static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
299 {
300 	guard(raw_spinlock_irqsave)(&ioapic_lock);
301 	return __ioapic_read_entry(apic, pin);
302 }
303 
304 /*
305  * When we write a new IO APIC routing entry, we need to write the high
306  * word first! If the mask bit in the low word is clear, we will enable
307  * the interrupt, and we need to make sure the entry is fully populated
308  * before that happens.
309  */
__ioapic_write_entry(int apic,int pin,struct IO_APIC_route_entry e)310 static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
311 {
312 	io_apic_write(apic, 0x11 + 2*pin, e.w2);
313 	io_apic_write(apic, 0x10 + 2*pin, e.w1);
314 }
315 
ioapic_write_entry(int apic,int pin,struct IO_APIC_route_entry e)316 static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
317 {
318 	guard(raw_spinlock_irqsave)(&ioapic_lock);
319 	__ioapic_write_entry(apic, pin, e);
320 }
321 
322 /*
323  * When we mask an IO APIC routing entry, we need to write the low
324  * word first, in order to set the mask bit before we change the
325  * high bits!
326  */
ioapic_mask_entry(int apic,int pin)327 static void ioapic_mask_entry(int apic, int pin)
328 {
329 	struct IO_APIC_route_entry e = { .masked = true };
330 
331 	guard(raw_spinlock_irqsave)(&ioapic_lock);
332 	io_apic_write(apic, 0x10 + 2*pin, e.w1);
333 	io_apic_write(apic, 0x11 + 2*pin, e.w2);
334 }
335 
336 /*
337  * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
338  * shared ISA-space IRQs, so we have to support them. We are super
339  * fast in the common case, and fast for shared ISA-space IRQs.
340  */
add_pin_to_irq_node(struct mp_chip_data * data,int node,int apic,int pin)341 static bool add_pin_to_irq_node(struct mp_chip_data *data, int node, int apic, int pin)
342 {
343 	struct irq_pin_list *entry;
344 
345 	/* Don't allow duplicates */
346 	for_each_irq_pin(entry, data->irq_2_pin) {
347 		if (entry->apic == apic && entry->pin == pin)
348 			return true;
349 	}
350 
351 	entry = kzalloc_node(sizeof(struct irq_pin_list), GFP_ATOMIC, node);
352 	if (!entry) {
353 		pr_err("Cannot allocate irq_pin_list (%d,%d,%d)\n", node, apic, pin);
354 		return false;
355 	}
356 
357 	entry->apic = apic;
358 	entry->pin = pin;
359 	list_add_tail(&entry->list, &data->irq_2_pin);
360 	return true;
361 }
362 
__remove_pin_from_irq(struct mp_chip_data * data,int apic,int pin)363 static void __remove_pin_from_irq(struct mp_chip_data *data, int apic, int pin)
364 {
365 	struct irq_pin_list *tmp, *entry;
366 
367 	list_for_each_entry_safe(entry, tmp, &data->irq_2_pin, list) {
368 		if (entry->apic == apic && entry->pin == pin) {
369 			list_del(&entry->list);
370 			kfree(entry);
371 			return;
372 		}
373 	}
374 }
375 
io_apic_modify_irq(struct mp_chip_data * data,bool masked,void (* final)(struct irq_pin_list * entry))376 static void io_apic_modify_irq(struct mp_chip_data *data, bool masked,
377 			       void (*final)(struct irq_pin_list *entry))
378 {
379 	struct irq_pin_list *entry;
380 
381 	data->entry.masked = masked;
382 
383 	for_each_irq_pin(entry, data->irq_2_pin) {
384 		io_apic_write(entry->apic, 0x10 + 2 * entry->pin, data->entry.w1);
385 		if (final)
386 			final(entry);
387 	}
388 }
389 
390 /*
391  * Synchronize the IO-APIC and the CPU by doing a dummy read from the
392  * IO-APIC
393  */
io_apic_sync(struct irq_pin_list * entry)394 static void io_apic_sync(struct irq_pin_list *entry)
395 {
396 	struct io_apic __iomem *io_apic;
397 
398 	io_apic = io_apic_base(entry->apic);
399 	readl(&io_apic->data);
400 }
401 
mask_ioapic_irq(struct irq_data * irq_data)402 static void mask_ioapic_irq(struct irq_data *irq_data)
403 {
404 	struct mp_chip_data *data = irq_data->chip_data;
405 
406 	guard(raw_spinlock_irqsave)(&ioapic_lock);
407 	io_apic_modify_irq(data, true, &io_apic_sync);
408 }
409 
__unmask_ioapic(struct mp_chip_data * data)410 static void __unmask_ioapic(struct mp_chip_data *data)
411 {
412 	io_apic_modify_irq(data, false, NULL);
413 }
414 
unmask_ioapic_irq(struct irq_data * irq_data)415 static void unmask_ioapic_irq(struct irq_data *irq_data)
416 {
417 	struct mp_chip_data *data = irq_data->chip_data;
418 
419 	guard(raw_spinlock_irqsave)(&ioapic_lock);
420 	__unmask_ioapic(data);
421 }
422 
423 /*
424  * IO-APIC versions below 0x20 don't support EOI register.
425  * For the record, here is the information about various versions:
426  *     0Xh     82489DX
427  *     1Xh     I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant
428  *     2Xh     I/O(x)APIC which is PCI 2.2 Compliant
429  *     30h-FFh Reserved
430  *
431  * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic
432  * version as 0x2. This is an error with documentation and these ICH chips
433  * use io-apic's of version 0x20.
434  *
435  * For IO-APIC's with EOI register, we use that to do an explicit EOI.
436  * Otherwise, we simulate the EOI message manually by changing the trigger
437  * mode to edge and then back to level, with RTE being masked during this.
438  */
__eoi_ioapic_pin(int apic,int pin,int vector)439 static void __eoi_ioapic_pin(int apic, int pin, int vector)
440 {
441 	if (mpc_ioapic_ver(apic) >= 0x20) {
442 		io_apic_eoi(apic, vector);
443 	} else {
444 		struct IO_APIC_route_entry entry, entry1;
445 
446 		entry = entry1 = __ioapic_read_entry(apic, pin);
447 
448 		/* Mask the entry and change the trigger mode to edge. */
449 		entry1.masked = true;
450 		entry1.is_level = false;
451 
452 		__ioapic_write_entry(apic, pin, entry1);
453 
454 		/* Restore the previous level triggered entry. */
455 		__ioapic_write_entry(apic, pin, entry);
456 	}
457 }
458 
eoi_ioapic_pin(int vector,struct mp_chip_data * data)459 static void eoi_ioapic_pin(int vector, struct mp_chip_data *data)
460 {
461 	struct irq_pin_list *entry;
462 
463 	guard(raw_spinlock_irqsave)(&ioapic_lock);
464 	for_each_irq_pin(entry, data->irq_2_pin)
465 		__eoi_ioapic_pin(entry->apic, entry->pin, vector);
466 }
467 
clear_IO_APIC_pin(unsigned int apic,unsigned int pin)468 static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
469 {
470 	struct IO_APIC_route_entry entry;
471 
472 	/* Check delivery_mode to be sure we're not clearing an SMI pin */
473 	entry = ioapic_read_entry(apic, pin);
474 	if (entry.delivery_mode == APIC_DELIVERY_MODE_SMI)
475 		return;
476 
477 	/*
478 	 * Make sure the entry is masked and re-read the contents to check
479 	 * if it is a level triggered pin and if the remote-IRR is set.
480 	 */
481 	if (!entry.masked) {
482 		entry.masked = true;
483 		ioapic_write_entry(apic, pin, entry);
484 		entry = ioapic_read_entry(apic, pin);
485 	}
486 
487 	if (entry.irr) {
488 		/*
489 		 * Make sure the trigger mode is set to level. Explicit EOI
490 		 * doesn't clear the remote-IRR if the trigger mode is not
491 		 * set to level.
492 		 */
493 		if (!entry.is_level) {
494 			entry.is_level = true;
495 			ioapic_write_entry(apic, pin, entry);
496 		}
497 		guard(raw_spinlock_irqsave)(&ioapic_lock);
498 		__eoi_ioapic_pin(apic, pin, entry.vector);
499 	}
500 
501 	/*
502 	 * Clear the rest of the bits in the IO-APIC RTE except for the mask
503 	 * bit.
504 	 */
505 	ioapic_mask_entry(apic, pin);
506 	entry = ioapic_read_entry(apic, pin);
507 	if (entry.irr)
508 		pr_err("Unable to reset IRR for apic: %d, pin :%d\n",
509 		       mpc_ioapic_id(apic), pin);
510 }
511 
clear_IO_APIC(void)512 void clear_IO_APIC (void)
513 {
514 	int apic, pin;
515 
516 	for_each_ioapic_pin(apic, pin)
517 		clear_IO_APIC_pin(apic, pin);
518 }
519 
520 #ifdef CONFIG_X86_32
521 /*
522  * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
523  * specific CPU-side IRQs.
524  */
525 
526 #define MAX_PIRQS 8
527 static int pirq_entries[MAX_PIRQS] = {
528 	[0 ... MAX_PIRQS - 1] = -1
529 };
530 
ioapic_pirq_setup(char * str)531 static int __init ioapic_pirq_setup(char *str)
532 {
533 	int i, max, ints[MAX_PIRQS+1];
534 
535 	get_options(str, ARRAY_SIZE(ints), ints);
536 
537 	apic_pr_verbose("PIRQ redirection, working around broken MP-BIOS.\n");
538 
539 	max = MAX_PIRQS;
540 	if (ints[0] < MAX_PIRQS)
541 		max = ints[0];
542 
543 	for (i = 0; i < max; i++) {
544 		apic_pr_verbose("... PIRQ%d -> IRQ %d\n", i, ints[i + 1]);
545 		/* PIRQs are mapped upside down, usually */
546 		pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
547 	}
548 	return 1;
549 }
550 __setup("pirq=", ioapic_pirq_setup);
551 #endif /* CONFIG_X86_32 */
552 
553 /*
554  * Saves all the IO-APIC RTE's
555  */
save_ioapic_entries(void)556 int save_ioapic_entries(void)
557 {
558 	int apic, pin;
559 	int err = 0;
560 
561 	for_each_ioapic(apic) {
562 		if (!ioapics[apic].saved_registers) {
563 			err = -ENOMEM;
564 			continue;
565 		}
566 
567 		for_each_pin(apic, pin)
568 			ioapics[apic].saved_registers[pin] = ioapic_read_entry(apic, pin);
569 	}
570 
571 	return err;
572 }
573 
574 /*
575  * Mask all IO APIC entries.
576  */
mask_ioapic_entries(void)577 void mask_ioapic_entries(void)
578 {
579 	int apic, pin;
580 
581 	for_each_ioapic(apic) {
582 		if (!ioapics[apic].saved_registers)
583 			continue;
584 
585 		for_each_pin(apic, pin) {
586 			struct IO_APIC_route_entry entry;
587 
588 			entry = ioapics[apic].saved_registers[pin];
589 			if (!entry.masked) {
590 				entry.masked = true;
591 				ioapic_write_entry(apic, pin, entry);
592 			}
593 		}
594 	}
595 }
596 
597 /*
598  * Restore IO APIC entries which was saved in the ioapic structure.
599  */
restore_ioapic_entries(void)600 int restore_ioapic_entries(void)
601 {
602 	int apic, pin;
603 
604 	for_each_ioapic(apic) {
605 		if (!ioapics[apic].saved_registers)
606 			continue;
607 
608 		for_each_pin(apic, pin)
609 			ioapic_write_entry(apic, pin, ioapics[apic].saved_registers[pin]);
610 	}
611 	return 0;
612 }
613 
614 /*
615  * Find the IRQ entry number of a certain pin.
616  */
find_irq_entry(int ioapic_idx,int pin,int type)617 static int find_irq_entry(int ioapic_idx, int pin, int type)
618 {
619 	int i;
620 
621 	for (i = 0; i < mp_irq_entries; i++) {
622 		if (mp_irqs[i].irqtype == type &&
623 		    (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) ||
624 		     mp_irqs[i].dstapic == MP_APIC_ALL) &&
625 		    mp_irqs[i].dstirq == pin)
626 			return i;
627 	}
628 
629 	return -1;
630 }
631 
632 /*
633  * Find the pin to which IRQ[irq] (ISA) is connected
634  */
find_isa_irq_pin(int irq,int type)635 static int __init find_isa_irq_pin(int irq, int type)
636 {
637 	int i;
638 
639 	for (i = 0; i < mp_irq_entries; i++) {
640 		int lbus = mp_irqs[i].srcbus;
641 
642 		if (test_bit(lbus, mp_bus_not_pci) && (mp_irqs[i].irqtype == type) &&
643 		    (mp_irqs[i].srcbusirq == irq))
644 			return mp_irqs[i].dstirq;
645 	}
646 	return -1;
647 }
648 
find_isa_irq_apic(int irq,int type)649 static int __init find_isa_irq_apic(int irq, int type)
650 {
651 	int i;
652 
653 	for (i = 0; i < mp_irq_entries; i++) {
654 		int lbus = mp_irqs[i].srcbus;
655 
656 		if (test_bit(lbus, mp_bus_not_pci) && (mp_irqs[i].irqtype == type) &&
657 		    (mp_irqs[i].srcbusirq == irq))
658 			break;
659 	}
660 
661 	if (i < mp_irq_entries) {
662 		int ioapic_idx;
663 
664 		for_each_ioapic(ioapic_idx) {
665 			if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic)
666 				return ioapic_idx;
667 		}
668 	}
669 
670 	return -1;
671 }
672 
irq_active_low(int idx)673 static bool irq_active_low(int idx)
674 {
675 	int bus = mp_irqs[idx].srcbus;
676 
677 	/*
678 	 * Determine IRQ line polarity (high active or low active):
679 	 */
680 	switch (mp_irqs[idx].irqflag & MP_IRQPOL_MASK) {
681 	case MP_IRQPOL_DEFAULT:
682 		/*
683 		 * Conforms to spec, ie. bus-type dependent polarity.  PCI
684 		 * defaults to low active. [E]ISA defaults to high active.
685 		 */
686 		return !test_bit(bus, mp_bus_not_pci);
687 	case MP_IRQPOL_ACTIVE_HIGH:
688 		return false;
689 	case MP_IRQPOL_RESERVED:
690 		pr_warn("IOAPIC: Invalid polarity: 2, defaulting to low\n");
691 		fallthrough;
692 	case MP_IRQPOL_ACTIVE_LOW:
693 	default: /* Pointless default required due to do gcc stupidity */
694 		return true;
695 	}
696 }
697 
698 #ifdef CONFIG_EISA
699 /*
700  * EISA Edge/Level control register, ELCR
701  */
EISA_ELCR(unsigned int irq)702 static bool EISA_ELCR(unsigned int irq)
703 {
704 	if (irq < nr_legacy_irqs()) {
705 		unsigned int port = PIC_ELCR1 + (irq >> 3);
706 		return (inb(port) >> (irq & 7)) & 1;
707 	}
708 	apic_pr_verbose("Broken MPtable reports ISA irq %d\n", irq);
709 	return false;
710 }
711 
712 /*
713  * EISA interrupts are always active high and can be edge or level
714  * triggered depending on the ELCR value.  If an interrupt is listed as
715  * EISA conforming in the MP table, that means its trigger type must be
716  * read in from the ELCR.
717  */
eisa_irq_is_level(int idx,int bus,bool level)718 static bool eisa_irq_is_level(int idx, int bus, bool level)
719 {
720 	switch (mp_bus_id_to_type[bus]) {
721 	case MP_BUS_PCI:
722 	case MP_BUS_ISA:
723 		return level;
724 	case MP_BUS_EISA:
725 		return EISA_ELCR(mp_irqs[idx].srcbusirq);
726 	}
727 	pr_warn("IOAPIC: Invalid srcbus: %d defaulting to level\n", bus);
728 	return true;
729 }
730 #else
eisa_irq_is_level(int idx,int bus,bool level)731 static inline int eisa_irq_is_level(int idx, int bus, bool level)
732 {
733 	return level;
734 }
735 #endif
736 
irq_is_level(int idx)737 static bool irq_is_level(int idx)
738 {
739 	int bus = mp_irqs[idx].srcbus;
740 	bool level;
741 
742 	/*
743 	 * Determine IRQ trigger mode (edge or level sensitive):
744 	 */
745 	switch (mp_irqs[idx].irqflag & MP_IRQTRIG_MASK) {
746 	case MP_IRQTRIG_DEFAULT:
747 		/*
748 		 * Conforms to spec, ie. bus-type dependent trigger
749 		 * mode. PCI defaults to level, ISA to edge.
750 		 */
751 		level = !test_bit(bus, mp_bus_not_pci);
752 		/* Take EISA into account */
753 		return eisa_irq_is_level(idx, bus, level);
754 	case MP_IRQTRIG_EDGE:
755 		return false;
756 	case MP_IRQTRIG_RESERVED:
757 		pr_warn("IOAPIC: Invalid trigger mode 2 defaulting to level\n");
758 		fallthrough;
759 	case MP_IRQTRIG_LEVEL:
760 	default: /* Pointless default required due to do gcc stupidity */
761 		return true;
762 	}
763 }
764 
__acpi_get_override_irq(u32 gsi,bool * trigger,bool * polarity)765 static int __acpi_get_override_irq(u32 gsi, bool *trigger, bool *polarity)
766 {
767 	int ioapic, pin, idx;
768 
769 	if (ioapic_is_disabled)
770 		return -1;
771 
772 	ioapic = mp_find_ioapic(gsi);
773 	if (ioapic < 0)
774 		return -1;
775 
776 	pin = mp_find_ioapic_pin(ioapic, gsi);
777 	if (pin < 0)
778 		return -1;
779 
780 	idx = find_irq_entry(ioapic, pin, mp_INT);
781 	if (idx < 0)
782 		return -1;
783 
784 	*trigger = irq_is_level(idx);
785 	*polarity = irq_active_low(idx);
786 	return 0;
787 }
788 
789 #ifdef CONFIG_ACPI
acpi_get_override_irq(u32 gsi,int * is_level,int * active_low)790 int acpi_get_override_irq(u32 gsi, int *is_level, int *active_low)
791 {
792 	*is_level = *active_low = 0;
793 	return __acpi_get_override_irq(gsi, (bool *)is_level,
794 				       (bool *)active_low);
795 }
796 #endif
797 
ioapic_set_alloc_attr(struct irq_alloc_info * info,int node,int trigger,int polarity)798 void ioapic_set_alloc_attr(struct irq_alloc_info *info, int node,
799 			   int trigger, int polarity)
800 {
801 	init_irq_alloc_info(info, NULL);
802 	info->type = X86_IRQ_ALLOC_TYPE_IOAPIC;
803 	info->ioapic.node = node;
804 	info->ioapic.is_level = trigger;
805 	info->ioapic.active_low = polarity;
806 	info->ioapic.valid = 1;
807 }
808 
ioapic_copy_alloc_attr(struct irq_alloc_info * dst,struct irq_alloc_info * src,u32 gsi,int ioapic_idx,int pin)809 static void ioapic_copy_alloc_attr(struct irq_alloc_info *dst,
810 				   struct irq_alloc_info *src,
811 				   u32 gsi, int ioapic_idx, int pin)
812 {
813 	bool level, pol_low;
814 
815 	copy_irq_alloc_info(dst, src);
816 	dst->type = X86_IRQ_ALLOC_TYPE_IOAPIC;
817 	dst->devid = mpc_ioapic_id(ioapic_idx);
818 	dst->ioapic.pin = pin;
819 	dst->ioapic.valid = 1;
820 	if (src && src->ioapic.valid) {
821 		dst->ioapic.node = src->ioapic.node;
822 		dst->ioapic.is_level = src->ioapic.is_level;
823 		dst->ioapic.active_low = src->ioapic.active_low;
824 	} else {
825 		dst->ioapic.node = NUMA_NO_NODE;
826 		if (__acpi_get_override_irq(gsi, &level, &pol_low) >= 0) {
827 			dst->ioapic.is_level = level;
828 			dst->ioapic.active_low = pol_low;
829 		} else {
830 			/*
831 			 * PCI interrupts are always active low level
832 			 * triggered.
833 			 */
834 			dst->ioapic.is_level = true;
835 			dst->ioapic.active_low = true;
836 		}
837 	}
838 }
839 
ioapic_alloc_attr_node(struct irq_alloc_info * info)840 static int ioapic_alloc_attr_node(struct irq_alloc_info *info)
841 {
842 	return (info && info->ioapic.valid) ? info->ioapic.node : NUMA_NO_NODE;
843 }
844 
mp_register_handler(unsigned int irq,bool level)845 static void mp_register_handler(unsigned int irq, bool level)
846 {
847 	irq_flow_handler_t hdl;
848 	bool fasteoi;
849 
850 	if (level) {
851 		irq_set_status_flags(irq, IRQ_LEVEL);
852 		fasteoi = true;
853 	} else {
854 		irq_clear_status_flags(irq, IRQ_LEVEL);
855 		fasteoi = false;
856 	}
857 
858 	hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
859 	__irq_set_handler(irq, hdl, 0, fasteoi ? "fasteoi" : "edge");
860 }
861 
mp_check_pin_attr(int irq,struct irq_alloc_info * info)862 static bool mp_check_pin_attr(int irq, struct irq_alloc_info *info)
863 {
864 	struct mp_chip_data *data = irq_get_chip_data(irq);
865 
866 	/*
867 	 * setup_IO_APIC_irqs() programs all legacy IRQs with default trigger
868 	 * and polarity attributes. So allow the first user to reprogram the
869 	 * pin with real trigger and polarity attributes.
870 	 */
871 	if (irq < nr_legacy_irqs() && data->count == 1) {
872 		if (info->ioapic.is_level != data->is_level)
873 			mp_register_handler(irq, info->ioapic.is_level);
874 		data->entry.is_level = data->is_level = info->ioapic.is_level;
875 		data->entry.active_low = data->active_low = info->ioapic.active_low;
876 	}
877 
878 	return data->is_level == info->ioapic.is_level &&
879 	       data->active_low == info->ioapic.active_low;
880 }
881 
alloc_irq_from_domain(struct irq_domain * domain,int ioapic,u32 gsi,struct irq_alloc_info * info)882 static int alloc_irq_from_domain(struct irq_domain *domain, int ioapic, u32 gsi,
883 				 struct irq_alloc_info *info)
884 {
885 	int type = ioapics[ioapic].irqdomain_cfg.type;
886 	bool legacy = false;
887 	int irq = -1;
888 
889 	switch (type) {
890 	case IOAPIC_DOMAIN_LEGACY:
891 		/*
892 		 * Dynamically allocate IRQ number for non-ISA IRQs in the first
893 		 * 16 GSIs on some weird platforms.
894 		 */
895 		if (!ioapic_initialized || gsi >= nr_legacy_irqs())
896 			irq = gsi;
897 		legacy = mp_is_legacy_irq(irq);
898 		break;
899 	case IOAPIC_DOMAIN_STRICT:
900 		irq = gsi;
901 		break;
902 	case IOAPIC_DOMAIN_DYNAMIC:
903 		break;
904 	default:
905 		WARN(1, "ioapic: unknown irqdomain type %d\n", type);
906 		return -1;
907 	}
908 
909 	return __irq_domain_alloc_irqs(domain, irq, 1, ioapic_alloc_attr_node(info),
910 				       info, legacy, NULL);
911 }
912 
913 /*
914  * Need special handling for ISA IRQs because there may be multiple IOAPIC pins
915  * sharing the same ISA IRQ number and irqdomain only supports 1:1 mapping
916  * between IOAPIC pin and IRQ number. A typical IOAPIC has 24 pins, pin 0-15 are
917  * used for legacy IRQs and pin 16-23 are used for PCI IRQs (PIRQ A-H).
918  * When ACPI is disabled, only legacy IRQ numbers (IRQ0-15) are available, and
919  * some BIOSes may use MP Interrupt Source records to override IRQ numbers for
920  * PIRQs instead of reprogramming the interrupt routing logic. Thus there may be
921  * multiple pins sharing the same legacy IRQ number when ACPI is disabled.
922  */
alloc_isa_irq_from_domain(struct irq_domain * domain,int irq,int ioapic,int pin,struct irq_alloc_info * info)923 static int alloc_isa_irq_from_domain(struct irq_domain *domain, int irq, int ioapic, int pin,
924 				     struct irq_alloc_info *info)
925 {
926 	struct irq_data *irq_data = irq_get_irq_data(irq);
927 	int node = ioapic_alloc_attr_node(info);
928 	struct mp_chip_data *data;
929 
930 	/*
931 	 * Legacy ISA IRQ has already been allocated, just add pin to
932 	 * the pin list associated with this IRQ and program the IOAPIC
933 	 * entry.
934 	 */
935 	if (irq_data && irq_data->parent_data) {
936 		if (!mp_check_pin_attr(irq, info))
937 			return -EBUSY;
938 		if (!add_pin_to_irq_node(irq_data->chip_data, node, ioapic, info->ioapic.pin))
939 			return -ENOMEM;
940 	} else {
941 		info->flags |= X86_IRQ_ALLOC_LEGACY;
942 		irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true, NULL);
943 		if (irq >= 0) {
944 			irq_data = irq_domain_get_irq_data(domain, irq);
945 			data = irq_data->chip_data;
946 			data->isa_irq = true;
947 		}
948 	}
949 
950 	return irq;
951 }
952 
mp_map_pin_to_irq(u32 gsi,int idx,int ioapic,int pin,unsigned int flags,struct irq_alloc_info * info)953 static int mp_map_pin_to_irq(u32 gsi, int idx, int ioapic, int pin,
954 			     unsigned int flags, struct irq_alloc_info *info)
955 {
956 	struct irq_domain *domain = mp_ioapic_irqdomain(ioapic);
957 	struct irq_alloc_info tmp;
958 	struct mp_chip_data *data;
959 	bool legacy = false;
960 	int irq;
961 
962 	if (!domain)
963 		return -ENOSYS;
964 
965 	if (idx >= 0 && test_bit(mp_irqs[idx].srcbus, mp_bus_not_pci)) {
966 		irq = mp_irqs[idx].srcbusirq;
967 		legacy = mp_is_legacy_irq(irq);
968 		/*
969 		 * IRQ2 is unusable for historical reasons on systems which
970 		 * have a legacy PIC. See the comment vs. IRQ2 further down.
971 		 *
972 		 * If this gets removed at some point then the related code
973 		 * in lapic_assign_system_vectors() needs to be adjusted as
974 		 * well.
975 		 */
976 		if (legacy && irq == PIC_CASCADE_IR)
977 			return -EINVAL;
978 	}
979 
980 	guard(mutex)(&ioapic_mutex);
981 	if (!(flags & IOAPIC_MAP_ALLOC)) {
982 		if (!legacy) {
983 			irq = irq_find_mapping(domain, pin);
984 			if (irq == 0)
985 				irq = -ENOENT;
986 		}
987 	} else {
988 		ioapic_copy_alloc_attr(&tmp, info, gsi, ioapic, pin);
989 		if (legacy)
990 			irq = alloc_isa_irq_from_domain(domain, irq,
991 							ioapic, pin, &tmp);
992 		else if ((irq = irq_find_mapping(domain, pin)) == 0)
993 			irq = alloc_irq_from_domain(domain, ioapic, gsi, &tmp);
994 		else if (!mp_check_pin_attr(irq, &tmp))
995 			irq = -EBUSY;
996 		if (irq >= 0) {
997 			data = irq_get_chip_data(irq);
998 			data->count++;
999 		}
1000 	}
1001 	return irq;
1002 }
1003 
pin_2_irq(int idx,int ioapic,int pin,unsigned int flags)1004 static int pin_2_irq(int idx, int ioapic, int pin, unsigned int flags)
1005 {
1006 	u32 gsi = mp_pin_to_gsi(ioapic, pin);
1007 
1008 	/* Debugging check, we are in big trouble if this message pops up! */
1009 	if (mp_irqs[idx].dstirq != pin)
1010 		pr_err("broken BIOS or MPTABLE parser, ayiee!!\n");
1011 
1012 #ifdef CONFIG_X86_32
1013 	/* PCI IRQ command line redirection. Yes, limits are hardcoded. */
1014 	if ((pin >= 16) && (pin <= 23)) {
1015 		if (pirq_entries[pin - 16] != -1) {
1016 			if (!pirq_entries[pin - 16]) {
1017 				apic_pr_verbose("Disabling PIRQ%d\n", pin - 16);
1018 			} else {
1019 				int irq = pirq_entries[pin-16];
1020 
1021 				apic_pr_verbose("Using PIRQ%d -> IRQ %d\n", pin - 16, irq);
1022 				return irq;
1023 			}
1024 		}
1025 	}
1026 #endif
1027 
1028 	return  mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, NULL);
1029 }
1030 
mp_map_gsi_to_irq(u32 gsi,unsigned int flags,struct irq_alloc_info * info)1031 int mp_map_gsi_to_irq(u32 gsi, unsigned int flags, struct irq_alloc_info *info)
1032 {
1033 	int ioapic, pin, idx;
1034 
1035 	ioapic = mp_find_ioapic(gsi);
1036 	if (ioapic < 0)
1037 		return -ENODEV;
1038 
1039 	pin = mp_find_ioapic_pin(ioapic, gsi);
1040 	idx = find_irq_entry(ioapic, pin, mp_INT);
1041 	if ((flags & IOAPIC_MAP_CHECK) && idx < 0)
1042 		return -ENODEV;
1043 
1044 	return mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, info);
1045 }
1046 
mp_unmap_irq(int irq)1047 void mp_unmap_irq(int irq)
1048 {
1049 	struct irq_data *irq_data = irq_get_irq_data(irq);
1050 	struct mp_chip_data *data;
1051 
1052 	if (!irq_data || !irq_data->domain)
1053 		return;
1054 
1055 	data = irq_data->chip_data;
1056 	if (!data || data->isa_irq)
1057 		return;
1058 
1059 	guard(mutex)(&ioapic_mutex);
1060 	if (--data->count == 0)
1061 		irq_domain_free_irqs(irq, 1);
1062 }
1063 
1064 /*
1065  * Find a specific PCI IRQ entry.
1066  * Not an __init, possibly needed by modules
1067  */
IO_APIC_get_PCI_irq_vector(int bus,int slot,int pin)1068 int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
1069 {
1070 	int irq, i, best_ioapic = -1, best_idx = -1;
1071 
1072 	apic_pr_debug("Querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1073 		      bus, slot, pin);
1074 	if (test_bit(bus, mp_bus_not_pci)) {
1075 		apic_pr_verbose("PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1076 		return -1;
1077 	}
1078 
1079 	for (i = 0; i < mp_irq_entries; i++) {
1080 		int lbus = mp_irqs[i].srcbus;
1081 		int ioapic_idx, found = 0;
1082 
1083 		if (bus != lbus || mp_irqs[i].irqtype != mp_INT ||
1084 		    slot != ((mp_irqs[i].srcbusirq >> 2) & 0x1f))
1085 			continue;
1086 
1087 		for_each_ioapic(ioapic_idx)
1088 			if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic ||
1089 			    mp_irqs[i].dstapic == MP_APIC_ALL) {
1090 				found = 1;
1091 				break;
1092 			}
1093 		if (!found)
1094 			continue;
1095 
1096 		/* Skip ISA IRQs */
1097 		irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq, 0);
1098 		if (irq > 0 && !IO_APIC_IRQ(irq))
1099 			continue;
1100 
1101 		if (pin == (mp_irqs[i].srcbusirq & 3)) {
1102 			best_idx = i;
1103 			best_ioapic = ioapic_idx;
1104 			goto out;
1105 		}
1106 
1107 		/*
1108 		 * Use the first all-but-pin matching entry as a
1109 		 * best-guess fuzzy result for broken mptables.
1110 		 */
1111 		if (best_idx < 0) {
1112 			best_idx = i;
1113 			best_ioapic = ioapic_idx;
1114 		}
1115 	}
1116 	if (best_idx < 0)
1117 		return -1;
1118 
1119 out:
1120 	return pin_2_irq(best_idx, best_ioapic, mp_irqs[best_idx].dstirq, IOAPIC_MAP_ALLOC);
1121 }
1122 EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1123 
1124 static struct irq_chip ioapic_chip, ioapic_ir_chip;
1125 
setup_IO_APIC_irqs(void)1126 static void __init setup_IO_APIC_irqs(void)
1127 {
1128 	unsigned int ioapic, pin;
1129 	int idx;
1130 
1131 	apic_pr_verbose("Init IO_APIC IRQs\n");
1132 
1133 	for_each_ioapic_pin(ioapic, pin) {
1134 		idx = find_irq_entry(ioapic, pin, mp_INT);
1135 		if (idx < 0) {
1136 			apic_pr_verbose("apic %d pin %d not connected\n",
1137 					mpc_ioapic_id(ioapic), pin);
1138 		} else {
1139 			pin_2_irq(idx, ioapic, pin, ioapic ? 0 : IOAPIC_MAP_ALLOC);
1140 		}
1141 	}
1142 }
1143 
ioapic_zap_locks(void)1144 void ioapic_zap_locks(void)
1145 {
1146 	raw_spin_lock_init(&ioapic_lock);
1147 }
1148 
io_apic_print_entries(unsigned int apic,unsigned int nr_entries)1149 static void io_apic_print_entries(unsigned int apic, unsigned int nr_entries)
1150 {
1151 	struct IO_APIC_route_entry entry;
1152 	char buf[256];
1153 	int i;
1154 
1155 	apic_dbg("IOAPIC %d:\n", apic);
1156 	for (i = 0; i <= nr_entries; i++) {
1157 		entry = ioapic_read_entry(apic, i);
1158 		snprintf(buf, sizeof(buf), " pin%02x, %s, %s, %s, V(%02X), IRR(%1d), S(%1d)",
1159 			 i, entry.masked ? "disabled" : "enabled ",
1160 			 entry.is_level ? "level" : "edge ",
1161 			 entry.active_low ? "low " : "high",
1162 			 entry.vector, entry.irr, entry.delivery_status);
1163 		if (entry.ir_format) {
1164 			apic_dbg("%s, remapped, I(%04X),  Z(%X)\n", buf,
1165 				 (entry.ir_index_15 << 15) | entry.ir_index_0_14, entry.ir_zero);
1166 		} else {
1167 			apic_dbg("%s, %s, D(%02X%02X), M(%1d)\n", buf,
1168 				 entry.dest_mode_logical ? "logical " : "physic	al",
1169 				 entry.virt_destid_8_14, entry.destid_0_7, entry.delivery_mode);
1170 		}
1171 	}
1172 }
1173 
print_IO_APIC(int ioapic_idx)1174 static void __init print_IO_APIC(int ioapic_idx)
1175 {
1176 	union IO_APIC_reg_00 reg_00;
1177 	union IO_APIC_reg_01 reg_01;
1178 	union IO_APIC_reg_02 reg_02;
1179 	union IO_APIC_reg_03 reg_03;
1180 
1181 	scoped_guard (raw_spinlock_irqsave, &ioapic_lock) {
1182 		reg_00.raw = io_apic_read(ioapic_idx, 0);
1183 		reg_01.raw = io_apic_read(ioapic_idx, 1);
1184 		if (reg_01.bits.version >= 0x10)
1185 			reg_02.raw = io_apic_read(ioapic_idx, 2);
1186 		if (reg_01.bits.version >= 0x20)
1187 			reg_03.raw = io_apic_read(ioapic_idx, 3);
1188 	}
1189 
1190 	apic_dbg("IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx));
1191 	apic_dbg(".... register #00: %08X\n", reg_00.raw);
1192 	apic_dbg(".......    : physical APIC id: %02X\n", reg_00.bits.ID);
1193 	apic_dbg(".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
1194 	apic_dbg(".......    : LTS          : %X\n", reg_00.bits.LTS);
1195 	apic_dbg(".... register #01: %08X\n", *(int *)&reg_01);
1196 	apic_dbg(".......     : max redirection entries: %02X\n", reg_01.bits.entries);
1197 	apic_dbg(".......     : PRQ implemented: %X\n", reg_01.bits.PRQ);
1198 	apic_dbg(".......     : IO APIC version: %02X\n", reg_01.bits.version);
1199 
1200 	/*
1201 	 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1202 	 * but the value of reg_02 is read as the previous read register
1203 	 * value, so ignore it if reg_02 == reg_01.
1204 	 */
1205 	if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1206 		apic_dbg(".... register #02: %08X\n", reg_02.raw);
1207 		apic_dbg(".......     : arbitration: %02X\n", reg_02.bits.arbitration);
1208 	}
1209 
1210 	/*
1211 	 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1212 	 * or reg_03, but the value of reg_0[23] is read as the previous read
1213 	 * register value, so ignore it if reg_03 == reg_0[12].
1214 	 */
1215 	if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1216 	    reg_03.raw != reg_01.raw) {
1217 		apic_dbg(".... register #03: %08X\n", reg_03.raw);
1218 		apic_dbg(".......     : Boot DT    : %X\n", reg_03.bits.boot_DT);
1219 	}
1220 
1221 	apic_dbg(".... IRQ redirection table:\n");
1222 	io_apic_print_entries(ioapic_idx, reg_01.bits.entries);
1223 }
1224 
print_IO_APICs(void)1225 void __init print_IO_APICs(void)
1226 {
1227 	int ioapic_idx;
1228 	unsigned int irq;
1229 
1230 	apic_dbg("number of MP IRQ sources: %d.\n", mp_irq_entries);
1231 	for_each_ioapic(ioapic_idx) {
1232 		apic_dbg("number of IO-APIC #%d registers: %d.\n",
1233 			 mpc_ioapic_id(ioapic_idx), ioapics[ioapic_idx].nr_registers);
1234 	}
1235 
1236 	/*
1237 	 * We are a bit conservative about what we expect.  We have to
1238 	 * know about every hardware change ASAP.
1239 	 */
1240 	printk(KERN_INFO "testing the IO APIC.......................\n");
1241 
1242 	for_each_ioapic(ioapic_idx)
1243 		print_IO_APIC(ioapic_idx);
1244 
1245 	apic_dbg("IRQ to pin mappings:\n");
1246 	for_each_active_irq(irq) {
1247 		struct irq_pin_list *entry;
1248 		struct irq_chip *chip;
1249 		struct mp_chip_data *data;
1250 
1251 		chip = irq_get_chip(irq);
1252 		if (chip != &ioapic_chip && chip != &ioapic_ir_chip)
1253 			continue;
1254 		data = irq_get_chip_data(irq);
1255 		if (!data)
1256 			continue;
1257 		if (list_empty(&data->irq_2_pin))
1258 			continue;
1259 
1260 		apic_dbg("IRQ%d ", irq);
1261 		for_each_irq_pin(entry, data->irq_2_pin)
1262 			pr_cont("-> %d:%d", entry->apic, entry->pin);
1263 		pr_cont("\n");
1264 	}
1265 
1266 	printk(KERN_INFO ".................................... done.\n");
1267 }
1268 
1269 /* Where if anywhere is the i8259 connect in external int mode */
1270 static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1271 
enable_IO_APIC(void)1272 void __init enable_IO_APIC(void)
1273 {
1274 	int i8259_apic, i8259_pin, apic, pin;
1275 
1276 	if (ioapic_is_disabled)
1277 		nr_ioapics = 0;
1278 
1279 	if (!nr_legacy_irqs() || !nr_ioapics)
1280 		return;
1281 
1282 	for_each_ioapic_pin(apic, pin) {
1283 		/* See if any of the pins is in ExtINT mode */
1284 		struct IO_APIC_route_entry entry = ioapic_read_entry(apic, pin);
1285 
1286 		/*
1287 		 * If the interrupt line is enabled and in ExtInt mode I
1288 		 * have found the pin where the i8259 is connected.
1289 		 */
1290 		if (!entry.masked && entry.delivery_mode == APIC_DELIVERY_MODE_EXTINT) {
1291 			ioapic_i8259.apic = apic;
1292 			ioapic_i8259.pin  = pin;
1293 			break;
1294 		}
1295 	}
1296 
1297 	/*
1298 	 * Look to see what if the MP table has reported the ExtINT
1299 	 *
1300 	 * If we could not find the appropriate pin by looking at the ioapic
1301 	 * the i8259 probably is not connected the ioapic but give the
1302 	 * mptable a chance anyway.
1303 	 */
1304 	i8259_pin  = find_isa_irq_pin(0, mp_ExtINT);
1305 	i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1306 	/* Trust the MP table if nothing is setup in the hardware */
1307 	if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1308 		pr_warn("ExtINT not setup in hardware but reported by MP table\n");
1309 		ioapic_i8259.pin  = i8259_pin;
1310 		ioapic_i8259.apic = i8259_apic;
1311 	}
1312 	/* Complain if the MP table and the hardware disagree */
1313 	if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1314 	    (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1315 		pr_warn("ExtINT in hardware and MP table differ\n");
1316 
1317 	/* Do not trust the IO-APIC being empty at bootup */
1318 	clear_IO_APIC();
1319 }
1320 
native_restore_boot_irq_mode(void)1321 void native_restore_boot_irq_mode(void)
1322 {
1323 	/*
1324 	 * If the i8259 is routed through an IOAPIC Put that IOAPIC in
1325 	 * virtual wire mode so legacy interrupts can be delivered.
1326 	 */
1327 	if (ioapic_i8259.pin != -1) {
1328 		struct IO_APIC_route_entry entry;
1329 		u32 apic_id = read_apic_id();
1330 
1331 		memset(&entry, 0, sizeof(entry));
1332 		entry.masked		= false;
1333 		entry.is_level		= false;
1334 		entry.active_low	= false;
1335 		entry.dest_mode_logical	= false;
1336 		entry.delivery_mode	= APIC_DELIVERY_MODE_EXTINT;
1337 		entry.destid_0_7	= apic_id & 0xFF;
1338 		entry.virt_destid_8_14	= apic_id >> 8;
1339 
1340 		/* Add it to the IO-APIC irq-routing table */
1341 		ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1342 	}
1343 
1344 	if (boot_cpu_has(X86_FEATURE_APIC) || apic_from_smp_config())
1345 		disconnect_bsp_APIC(ioapic_i8259.pin != -1);
1346 }
1347 
restore_boot_irq_mode(void)1348 void restore_boot_irq_mode(void)
1349 {
1350 	if (!nr_legacy_irqs())
1351 		return;
1352 
1353 	x86_apic_ops.restore();
1354 }
1355 
1356 #ifdef CONFIG_X86_32
1357 /*
1358  * function to set the IO-APIC physical IDs based on the
1359  * values stored in the MPC table.
1360  *
1361  * by Matt Domsch <Matt_Domsch@dell.com>  Tue Dec 21 12:25:05 CST 1999
1362  */
setup_ioapic_ids_from_mpc_nocheck(void)1363 static void __init setup_ioapic_ids_from_mpc_nocheck(void)
1364 {
1365 	DECLARE_BITMAP(phys_id_present_map, MAX_LOCAL_APIC);
1366 	const u32 broadcast_id = 0xF;
1367 	union IO_APIC_reg_00 reg_00;
1368 	unsigned char old_id;
1369 	int ioapic_idx, i;
1370 
1371 	/*
1372 	 * This is broken; anything with a real cpu count has to
1373 	 * circumvent this idiocy regardless.
1374 	 */
1375 	copy_phys_cpu_present_map(phys_id_present_map);
1376 
1377 	/*
1378 	 * Set the IOAPIC ID to the value stored in the MPC table.
1379 	 */
1380 	for_each_ioapic(ioapic_idx) {
1381 		/* Read the register 0 value */
1382 		scoped_guard (raw_spinlock_irqsave, &ioapic_lock)
1383 			reg_00.raw = io_apic_read(ioapic_idx, 0);
1384 
1385 		old_id = mpc_ioapic_id(ioapic_idx);
1386 
1387 		if (mpc_ioapic_id(ioapic_idx) >= broadcast_id) {
1388 			pr_err(FW_BUG "IO-APIC#%d ID is %d in the MPC table!...\n",
1389 			       ioapic_idx, mpc_ioapic_id(ioapic_idx));
1390 			pr_err("... fixing up to %d. (tell your hw vendor)\n", reg_00.bits.ID);
1391 			ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID;
1392 		}
1393 
1394 		/*
1395 		 * Sanity check, is the ID really free? Every APIC in a
1396 		 * system must have a unique ID or we get lots of nice
1397 		 * 'stuck on smp_invalidate_needed IPI wait' messages.
1398 		 */
1399 		if (test_bit(mpc_ioapic_id(ioapic_idx), phys_id_present_map)) {
1400 			pr_err(FW_BUG "IO-APIC#%d ID %d is already used!...\n",
1401 			       ioapic_idx, mpc_ioapic_id(ioapic_idx));
1402 			for (i = 0; i < broadcast_id; i++)
1403 				if (!test_bit(i, phys_id_present_map))
1404 					break;
1405 			if (i >= broadcast_id)
1406 				panic("Max APIC ID exceeded!\n");
1407 			pr_err("... fixing up to %d. (tell your hw vendor)\n", i);
1408 			set_bit(i, phys_id_present_map);
1409 			ioapics[ioapic_idx].mp_config.apicid = i;
1410 		} else {
1411 			apic_pr_verbose("Setting %d in the phys_id_present_map\n",
1412 					mpc_ioapic_id(ioapic_idx));
1413 			set_bit(mpc_ioapic_id(ioapic_idx), phys_id_present_map);
1414 		}
1415 
1416 		/*
1417 		 * We need to adjust the IRQ routing table if the ID
1418 		 * changed.
1419 		 */
1420 		if (old_id != mpc_ioapic_id(ioapic_idx)) {
1421 			for (i = 0; i < mp_irq_entries; i++) {
1422 				if (mp_irqs[i].dstapic == old_id)
1423 					mp_irqs[i].dstapic = mpc_ioapic_id(ioapic_idx);
1424 			}
1425 		}
1426 
1427 		/*
1428 		 * Update the ID register according to the right value from
1429 		 * the MPC table if they are different.
1430 		 */
1431 		if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID)
1432 			continue;
1433 
1434 		apic_pr_verbose("...changing IO-APIC physical APIC ID to %d ...",
1435 				mpc_ioapic_id(ioapic_idx));
1436 
1437 		reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
1438 		scoped_guard (raw_spinlock_irqsave, &ioapic_lock) {
1439 			io_apic_write(ioapic_idx, 0, reg_00.raw);
1440 			reg_00.raw = io_apic_read(ioapic_idx, 0);
1441 		}
1442 		/* Sanity check */
1443 		if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx))
1444 			pr_cont("could not set ID!\n");
1445 		else
1446 			apic_pr_verbose(" ok.\n");
1447 	}
1448 }
1449 
setup_ioapic_ids_from_mpc(void)1450 void __init setup_ioapic_ids_from_mpc(void)
1451 {
1452 
1453 	if (acpi_ioapic)
1454 		return;
1455 	/*
1456 	 * Don't check I/O APIC IDs for xAPIC systems.  They have
1457 	 * no meaning without the serial APIC bus.
1458 	 */
1459 	if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1460 		|| APIC_XAPIC(boot_cpu_apic_version))
1461 		return;
1462 	setup_ioapic_ids_from_mpc_nocheck();
1463 }
1464 #endif
1465 
1466 int no_timer_check __initdata;
1467 
notimercheck(char * s)1468 static int __init notimercheck(char *s)
1469 {
1470 	no_timer_check = 1;
1471 	return 1;
1472 }
1473 __setup("no_timer_check", notimercheck);
1474 
delay_with_tsc(void)1475 static void __init delay_with_tsc(void)
1476 {
1477 	unsigned long long start, now;
1478 	unsigned long end = jiffies + 4;
1479 
1480 	start = rdtsc();
1481 
1482 	/*
1483 	 * We don't know the TSC frequency yet, but waiting for
1484 	 * 40000000000/HZ TSC cycles is safe:
1485 	 * 4 GHz == 10 jiffies
1486 	 * 1 GHz == 40 jiffies
1487 	 */
1488 	do {
1489 		rep_nop();
1490 		now = rdtsc();
1491 	} while ((now - start) < 40000000000ULL / HZ &&	time_before_eq(jiffies, end));
1492 }
1493 
delay_without_tsc(void)1494 static void __init delay_without_tsc(void)
1495 {
1496 	unsigned long end = jiffies + 4;
1497 	int band = 1;
1498 
1499 	/*
1500 	 * We don't know any frequency yet, but waiting for
1501 	 * 40940000000/HZ cycles is safe:
1502 	 * 4 GHz == 10 jiffies
1503 	 * 1 GHz == 40 jiffies
1504 	 * 1 << 1 + 1 << 2 +...+ 1 << 11 = 4094
1505 	 */
1506 	do {
1507 		__delay(((1U << band++) * 10000000UL) / HZ);
1508 	} while (band < 12 && time_before_eq(jiffies, end));
1509 }
1510 
1511 /*
1512  * There is a nasty bug in some older SMP boards, their mptable lies
1513  * about the timer IRQ. We do the following to work around the situation:
1514  *
1515  *	- timer IRQ defaults to IO-APIC IRQ
1516  *	- if this function detects that timer IRQs are defunct, then we fall
1517  *	  back to ISA timer IRQs
1518  */
timer_irq_works(void)1519 static int __init timer_irq_works(void)
1520 {
1521 	unsigned long t1 = jiffies;
1522 
1523 	if (no_timer_check)
1524 		return 1;
1525 
1526 	local_irq_enable();
1527 	if (boot_cpu_has(X86_FEATURE_TSC))
1528 		delay_with_tsc();
1529 	else
1530 		delay_without_tsc();
1531 
1532 	/*
1533 	 * Expect a few ticks at least, to be sure some possible
1534 	 * glue logic does not lock up after one or two first
1535 	 * ticks in a non-ExtINT mode.  Also the local APIC
1536 	 * might have cached one ExtINT interrupt.  Finally, at
1537 	 * least one tick may be lost due to delays.
1538 	 */
1539 
1540 	local_irq_disable();
1541 
1542 	/* Did jiffies advance? */
1543 	return time_after(jiffies, t1 + 4);
1544 }
1545 
1546 /*
1547  * In the SMP+IOAPIC case it might happen that there are an unspecified
1548  * number of pending IRQ events unhandled. These cases are very rare,
1549  * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
1550  * better to do it this way as thus we do not have to be aware of
1551  * 'pending' interrupts in the IRQ path, except at this point.
1552  *
1553  *
1554  * Edge triggered needs to resend any interrupt that was delayed but this
1555  * is now handled in the device independent code.
1556  *
1557  * Starting up a edge-triggered IO-APIC interrupt is nasty - we need to
1558  * make sure that we get the edge.  If it is already asserted for some
1559  * reason, we need return 1 to indicate that is was pending.
1560  *
1561  * This is not complete - we should be able to fake an edge even if it
1562  * isn't on the 8259A...
1563  */
startup_ioapic_irq(struct irq_data * data)1564 static unsigned int startup_ioapic_irq(struct irq_data *data)
1565 {
1566 	int was_pending = 0, irq = data->irq;
1567 
1568 	guard(raw_spinlock_irqsave)(&ioapic_lock);
1569 	if (irq < nr_legacy_irqs()) {
1570 		legacy_pic->mask(irq);
1571 		if (legacy_pic->irq_pending(irq))
1572 			was_pending = 1;
1573 	}
1574 	__unmask_ioapic(data->chip_data);
1575 	return was_pending;
1576 }
1577 
1578 atomic_t irq_mis_count;
1579 
1580 #ifdef CONFIG_GENERIC_PENDING_IRQ
io_apic_level_ack_pending(struct mp_chip_data * data)1581 static bool io_apic_level_ack_pending(struct mp_chip_data *data)
1582 {
1583 	struct irq_pin_list *entry;
1584 
1585 	guard(raw_spinlock_irqsave)(&ioapic_lock);
1586 	for_each_irq_pin(entry, data->irq_2_pin) {
1587 		struct IO_APIC_route_entry e;
1588 		int pin;
1589 
1590 		pin = entry->pin;
1591 		e.w1 = io_apic_read(entry->apic, 0x10 + pin*2);
1592 		/* Is the remote IRR bit set? */
1593 		if (e.irr)
1594 			return true;
1595 	}
1596 	return false;
1597 }
1598 
ioapic_prepare_move(struct irq_data * data)1599 static inline bool ioapic_prepare_move(struct irq_data *data)
1600 {
1601 	/* If we are moving the IRQ we need to mask it */
1602 	if (unlikely(irqd_is_setaffinity_pending(data))) {
1603 		if (!irqd_irq_masked(data))
1604 			mask_ioapic_irq(data);
1605 		return true;
1606 	}
1607 	return false;
1608 }
1609 
ioapic_finish_move(struct irq_data * data,bool moveit)1610 static inline void ioapic_finish_move(struct irq_data *data, bool moveit)
1611 {
1612 	if (unlikely(moveit)) {
1613 		/*
1614 		 * Only migrate the irq if the ack has been received.
1615 		 *
1616 		 * On rare occasions the broadcast level triggered ack gets
1617 		 * delayed going to ioapics, and if we reprogram the
1618 		 * vector while Remote IRR is still set the irq will never
1619 		 * fire again.
1620 		 *
1621 		 * To prevent this scenario we read the Remote IRR bit
1622 		 * of the ioapic.  This has two effects.
1623 		 * - On any sane system the read of the ioapic will
1624 		 *   flush writes (and acks) going to the ioapic from
1625 		 *   this cpu.
1626 		 * - We get to see if the ACK has actually been delivered.
1627 		 *
1628 		 * Based on failed experiments of reprogramming the
1629 		 * ioapic entry from outside of irq context starting
1630 		 * with masking the ioapic entry and then polling until
1631 		 * Remote IRR was clear before reprogramming the
1632 		 * ioapic I don't trust the Remote IRR bit to be
1633 		 * completely accurate.
1634 		 *
1635 		 * However there appears to be no other way to plug
1636 		 * this race, so if the Remote IRR bit is not
1637 		 * accurate and is causing problems then it is a hardware bug
1638 		 * and you can go talk to the chipset vendor about it.
1639 		 */
1640 		if (!io_apic_level_ack_pending(data->chip_data))
1641 			irq_move_masked_irq(data);
1642 		/* If the IRQ is masked in the core, leave it: */
1643 		if (!irqd_irq_masked(data))
1644 			unmask_ioapic_irq(data);
1645 	}
1646 }
1647 #else
ioapic_prepare_move(struct irq_data * data)1648 static inline bool ioapic_prepare_move(struct irq_data *data)
1649 {
1650 	return false;
1651 }
ioapic_finish_move(struct irq_data * data,bool moveit)1652 static inline void ioapic_finish_move(struct irq_data *data, bool moveit)
1653 {
1654 }
1655 #endif
1656 
ioapic_ack_level(struct irq_data * irq_data)1657 static void ioapic_ack_level(struct irq_data *irq_data)
1658 {
1659 	struct irq_cfg *cfg = irqd_cfg(irq_data);
1660 	unsigned long v;
1661 	bool moveit;
1662 	int i;
1663 
1664 	irq_complete_move(cfg);
1665 	moveit = ioapic_prepare_move(irq_data);
1666 
1667 	/*
1668 	 * It appears there is an erratum which affects at least version 0x11
1669 	 * of I/O APIC (that's the 82093AA and cores integrated into various
1670 	 * chipsets).  Under certain conditions a level-triggered interrupt is
1671 	 * erroneously delivered as edge-triggered one but the respective IRR
1672 	 * bit gets set nevertheless.  As a result the I/O unit expects an EOI
1673 	 * message but it will never arrive and further interrupts are blocked
1674 	 * from the source.  The exact reason is so far unknown, but the
1675 	 * phenomenon was observed when two consecutive interrupt requests
1676 	 * from a given source get delivered to the same CPU and the source is
1677 	 * temporarily disabled in between.
1678 	 *
1679 	 * A workaround is to simulate an EOI message manually.  We achieve it
1680 	 * by setting the trigger mode to edge and then to level when the edge
1681 	 * trigger mode gets detected in the TMR of a local APIC for a
1682 	 * level-triggered interrupt.  We mask the source for the time of the
1683 	 * operation to prevent an edge-triggered interrupt escaping meanwhile.
1684 	 * The idea is from Manfred Spraul.  --macro
1685 	 *
1686 	 * Also in the case when cpu goes offline, fixup_irqs() will forward
1687 	 * any unhandled interrupt on the offlined cpu to the new cpu
1688 	 * destination that is handling the corresponding interrupt. This
1689 	 * interrupt forwarding is done via IPI's. Hence, in this case also
1690 	 * level-triggered io-apic interrupt will be seen as an edge
1691 	 * interrupt in the IRR. And we can't rely on the cpu's EOI
1692 	 * to be broadcasted to the IO-APIC's which will clear the remoteIRR
1693 	 * corresponding to the level-triggered interrupt. Hence on IO-APIC's
1694 	 * supporting EOI register, we do an explicit EOI to clear the
1695 	 * remote IRR and on IO-APIC's which don't have an EOI register,
1696 	 * we use the above logic (mask+edge followed by unmask+level) from
1697 	 * Manfred Spraul to clear the remote IRR.
1698 	 */
1699 	i = cfg->vector;
1700 	v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
1701 
1702 	/*
1703 	 * We must acknowledge the irq before we move it or the acknowledge will
1704 	 * not propagate properly.
1705 	 */
1706 	apic_eoi();
1707 
1708 	/*
1709 	 * Tail end of clearing remote IRR bit (either by delivering the EOI
1710 	 * message via io-apic EOI register write or simulating it using
1711 	 * mask+edge followed by unmask+level logic) manually when the
1712 	 * level triggered interrupt is seen as the edge triggered interrupt
1713 	 * at the cpu.
1714 	 */
1715 	if (!(v & (1 << (i & 0x1f)))) {
1716 		atomic_inc(&irq_mis_count);
1717 		eoi_ioapic_pin(cfg->vector, irq_data->chip_data);
1718 	}
1719 
1720 	ioapic_finish_move(irq_data, moveit);
1721 }
1722 
ioapic_ir_ack_level(struct irq_data * irq_data)1723 static void ioapic_ir_ack_level(struct irq_data *irq_data)
1724 {
1725 	struct mp_chip_data *data = irq_data->chip_data;
1726 
1727 	/*
1728 	 * Intr-remapping uses pin number as the virtual vector
1729 	 * in the RTE. Actual vector is programmed in
1730 	 * intr-remapping table entry. Hence for the io-apic
1731 	 * EOI we use the pin number.
1732 	 */
1733 	apic_ack_irq(irq_data);
1734 	eoi_ioapic_pin(data->entry.vector, data);
1735 }
1736 
1737 /*
1738  * The I/OAPIC is just a device for generating MSI messages from legacy
1739  * interrupt pins. Various fields of the RTE translate into bits of the
1740  * resulting MSI which had a historical meaning.
1741  *
1742  * With interrupt remapping, many of those bits have different meanings
1743  * in the underlying MSI, but the way that the I/OAPIC transforms them
1744  * from its RTE to the MSI message is the same. This function allows
1745  * the parent IRQ domain to compose the MSI message, then takes the
1746  * relevant bits to put them in the appropriate places in the RTE in
1747  * order to generate that message when the IRQ happens.
1748  *
1749  * The setup here relies on a preconfigured route entry (is_level,
1750  * active_low, masked) because the parent domain is merely composing the
1751  * generic message routing information which is used for the MSI.
1752  */
ioapic_setup_msg_from_msi(struct irq_data * irq_data,struct IO_APIC_route_entry * entry)1753 static void ioapic_setup_msg_from_msi(struct irq_data *irq_data,
1754 				      struct IO_APIC_route_entry *entry)
1755 {
1756 	struct msi_msg msg;
1757 
1758 	/* Let the parent domain compose the MSI message */
1759 	irq_chip_compose_msi_msg(irq_data, &msg);
1760 
1761 	/*
1762 	 * - Real vector
1763 	 * - DMAR/IR: 8bit subhandle (ioapic.pin)
1764 	 * - AMD/IR:  8bit IRTE index
1765 	 */
1766 	entry->vector			= msg.arch_data.vector;
1767 	/* Delivery mode (for DMAR/IR all 0) */
1768 	entry->delivery_mode		= msg.arch_data.delivery_mode;
1769 	/* Destination mode or DMAR/IR index bit 15 */
1770 	entry->dest_mode_logical	= msg.arch_addr_lo.dest_mode_logical;
1771 	/* DMAR/IR: 1, 0 for all other modes */
1772 	entry->ir_format		= msg.arch_addr_lo.dmar_format;
1773 	/*
1774 	 * - DMAR/IR: index bit 0-14.
1775 	 *
1776 	 * - Virt: If the host supports x2apic without a virtualized IR
1777 	 *	   unit then bit 0-6 of dmar_index_0_14 are providing bit
1778 	 *	   8-14 of the destination id.
1779 	 *
1780 	 * All other modes have bit 0-6 of dmar_index_0_14 cleared and the
1781 	 * topmost 8 bits are destination id bit 0-7 (entry::destid_0_7).
1782 	 */
1783 	entry->ir_index_0_14		= msg.arch_addr_lo.dmar_index_0_14;
1784 }
1785 
ioapic_configure_entry(struct irq_data * irqd)1786 static void ioapic_configure_entry(struct irq_data *irqd)
1787 {
1788 	struct mp_chip_data *mpd = irqd->chip_data;
1789 	struct irq_pin_list *entry;
1790 
1791 	ioapic_setup_msg_from_msi(irqd, &mpd->entry);
1792 
1793 	for_each_irq_pin(entry, mpd->irq_2_pin)
1794 		__ioapic_write_entry(entry->apic, entry->pin, mpd->entry);
1795 }
1796 
ioapic_set_affinity(struct irq_data * irq_data,const struct cpumask * mask,bool force)1797 static int ioapic_set_affinity(struct irq_data *irq_data, const struct cpumask *mask, bool force)
1798 {
1799 	struct irq_data *parent = irq_data->parent_data;
1800 	int ret;
1801 
1802 	ret = parent->chip->irq_set_affinity(parent, mask, force);
1803 
1804 	guard(raw_spinlock_irqsave)(&ioapic_lock);
1805 	if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE)
1806 		ioapic_configure_entry(irq_data);
1807 
1808 	return ret;
1809 }
1810 
1811 /*
1812  * Interrupt shutdown masks the ioapic pin, but the interrupt might already
1813  * be in flight, but not yet serviced by the target CPU. That means
1814  * __synchronize_hardirq() would return and claim that everything is calmed
1815  * down. So free_irq() would proceed and deactivate the interrupt and free
1816  * resources.
1817  *
1818  * Once the target CPU comes around to service it it will find a cleared
1819  * vector and complain. While the spurious interrupt is harmless, the full
1820  * release of resources might prevent the interrupt from being acknowledged
1821  * which keeps the hardware in a weird state.
1822  *
1823  * Verify that the corresponding Remote-IRR bits are clear.
1824  */
ioapic_irq_get_chip_state(struct irq_data * irqd,enum irqchip_irq_state which,bool * state)1825 static int ioapic_irq_get_chip_state(struct irq_data *irqd, enum irqchip_irq_state which,
1826 				     bool *state)
1827 {
1828 	struct mp_chip_data *mcd = irqd->chip_data;
1829 	struct IO_APIC_route_entry rentry;
1830 	struct irq_pin_list *p;
1831 
1832 	if (which != IRQCHIP_STATE_ACTIVE)
1833 		return -EINVAL;
1834 
1835 	*state = false;
1836 
1837 	guard(raw_spinlock)(&ioapic_lock);
1838 	for_each_irq_pin(p, mcd->irq_2_pin) {
1839 		rentry = __ioapic_read_entry(p->apic, p->pin);
1840 		/*
1841 		 * The remote IRR is only valid in level trigger mode. It's
1842 		 * meaning is undefined for edge triggered interrupts and
1843 		 * irrelevant because the IO-APIC treats them as fire and
1844 		 * forget.
1845 		 */
1846 		if (rentry.irr && rentry.is_level) {
1847 			*state = true;
1848 			break;
1849 		}
1850 	}
1851 	return 0;
1852 }
1853 
1854 static struct irq_chip ioapic_chip __read_mostly = {
1855 	.name			= "IO-APIC",
1856 	.irq_startup		= startup_ioapic_irq,
1857 	.irq_mask		= mask_ioapic_irq,
1858 	.irq_unmask		= unmask_ioapic_irq,
1859 	.irq_ack		= irq_chip_ack_parent,
1860 	.irq_eoi		= ioapic_ack_level,
1861 	.irq_set_affinity	= ioapic_set_affinity,
1862 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
1863 	.irq_get_irqchip_state	= ioapic_irq_get_chip_state,
1864 	.flags			= IRQCHIP_SKIP_SET_WAKE |
1865 				  IRQCHIP_AFFINITY_PRE_STARTUP,
1866 };
1867 
1868 static struct irq_chip ioapic_ir_chip __read_mostly = {
1869 	.name			= "IR-IO-APIC",
1870 	.irq_startup		= startup_ioapic_irq,
1871 	.irq_mask		= mask_ioapic_irq,
1872 	.irq_unmask		= unmask_ioapic_irq,
1873 	.irq_ack		= irq_chip_ack_parent,
1874 	.irq_eoi		= ioapic_ir_ack_level,
1875 	.irq_set_affinity	= ioapic_set_affinity,
1876 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
1877 	.irq_get_irqchip_state	= ioapic_irq_get_chip_state,
1878 	.flags			= IRQCHIP_SKIP_SET_WAKE |
1879 				  IRQCHIP_AFFINITY_PRE_STARTUP,
1880 };
1881 
init_IO_APIC_traps(void)1882 static inline void init_IO_APIC_traps(void)
1883 {
1884 	struct irq_cfg *cfg;
1885 	unsigned int irq;
1886 
1887 	for_each_active_irq(irq) {
1888 		cfg = irq_cfg(irq);
1889 		if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
1890 			/*
1891 			 * Hmm.. We don't have an entry for this, so
1892 			 * default to an old-fashioned 8259 interrupt if we
1893 			 * can. Otherwise set the dummy interrupt chip.
1894 			 */
1895 			if (irq < nr_legacy_irqs())
1896 				legacy_pic->make_irq(irq);
1897 			else
1898 				irq_set_chip(irq, &no_irq_chip);
1899 		}
1900 	}
1901 }
1902 
1903 /*
1904  * The local APIC irq-chip implementation:
1905  */
mask_lapic_irq(struct irq_data * data)1906 static void mask_lapic_irq(struct irq_data *data)
1907 {
1908 	unsigned long v = apic_read(APIC_LVT0);
1909 
1910 	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1911 }
1912 
unmask_lapic_irq(struct irq_data * data)1913 static void unmask_lapic_irq(struct irq_data *data)
1914 {
1915 	unsigned long v = apic_read(APIC_LVT0);
1916 
1917 	apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
1918 }
1919 
ack_lapic_irq(struct irq_data * data)1920 static void ack_lapic_irq(struct irq_data *data)
1921 {
1922 	apic_eoi();
1923 }
1924 
1925 static struct irq_chip lapic_chip __read_mostly = {
1926 	.name		= "local-APIC",
1927 	.irq_mask	= mask_lapic_irq,
1928 	.irq_unmask	= unmask_lapic_irq,
1929 	.irq_ack	= ack_lapic_irq,
1930 };
1931 
lapic_register_intr(int irq)1932 static void lapic_register_intr(int irq)
1933 {
1934 	irq_clear_status_flags(irq, IRQ_LEVEL);
1935 	irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq, "edge");
1936 }
1937 
1938 /*
1939  * This looks a bit hackish but it's about the only one way of sending
1940  * a few INTA cycles to 8259As and any associated glue logic.  ICR does
1941  * not support the ExtINT mode, unfortunately.  We need to send these
1942  * cycles as some i82489DX-based boards have glue logic that keeps the
1943  * 8259A interrupt line asserted until INTA.  --macro
1944  */
unlock_ExtINT_logic(void)1945 static inline void __init unlock_ExtINT_logic(void)
1946 {
1947 	unsigned char save_control, save_freq_select;
1948 	struct IO_APIC_route_entry entry0, entry1;
1949 	int apic, pin, i;
1950 	u32 apic_id;
1951 
1952 	pin  = find_isa_irq_pin(8, mp_INT);
1953 	if (pin == -1) {
1954 		WARN_ON_ONCE(1);
1955 		return;
1956 	}
1957 	apic = find_isa_irq_apic(8, mp_INT);
1958 	if (apic == -1) {
1959 		WARN_ON_ONCE(1);
1960 		return;
1961 	}
1962 
1963 	entry0 = ioapic_read_entry(apic, pin);
1964 	clear_IO_APIC_pin(apic, pin);
1965 
1966 	apic_id = read_apic_id();
1967 	memset(&entry1, 0, sizeof(entry1));
1968 
1969 	entry1.dest_mode_logical	= true;
1970 	entry1.masked			= false;
1971 	entry1.destid_0_7		= apic_id & 0xFF;
1972 	entry1.virt_destid_8_14		= apic_id >> 8;
1973 	entry1.delivery_mode		= APIC_DELIVERY_MODE_EXTINT;
1974 	entry1.active_low		= entry0.active_low;
1975 	entry1.is_level			= false;
1976 	entry1.vector = 0;
1977 
1978 	ioapic_write_entry(apic, pin, entry1);
1979 
1980 	save_control = CMOS_READ(RTC_CONTROL);
1981 	save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
1982 	CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
1983 		   RTC_FREQ_SELECT);
1984 	CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
1985 
1986 	i = 100;
1987 	while (i-- > 0) {
1988 		mdelay(10);
1989 		if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
1990 			i -= 10;
1991 	}
1992 
1993 	CMOS_WRITE(save_control, RTC_CONTROL);
1994 	CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
1995 	clear_IO_APIC_pin(apic, pin);
1996 
1997 	ioapic_write_entry(apic, pin, entry0);
1998 }
1999 
2000 static int disable_timer_pin_1 __initdata;
2001 /* Actually the next is obsolete, but keep it for paranoid reasons -AK */
disable_timer_pin_setup(char * arg)2002 static int __init disable_timer_pin_setup(char *arg)
2003 {
2004 	disable_timer_pin_1 = 1;
2005 	return 0;
2006 }
2007 early_param("disable_timer_pin_1", disable_timer_pin_setup);
2008 
mp_alloc_timer_irq(int ioapic,int pin)2009 static int __init mp_alloc_timer_irq(int ioapic, int pin)
2010 {
2011 	struct irq_domain *domain = mp_ioapic_irqdomain(ioapic);
2012 	int irq = -1;
2013 
2014 	if (domain) {
2015 		struct irq_alloc_info info;
2016 
2017 		ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 0, 0);
2018 		info.devid = mpc_ioapic_id(ioapic);
2019 		info.ioapic.pin = pin;
2020 		guard(mutex)(&ioapic_mutex);
2021 		irq = alloc_isa_irq_from_domain(domain, 0, ioapic, pin, &info);
2022 	}
2023 
2024 	return irq;
2025 }
2026 
replace_pin_at_irq_node(struct mp_chip_data * data,int node,int oldapic,int oldpin,int newapic,int newpin)2027 static void __init replace_pin_at_irq_node(struct mp_chip_data *data, int node,
2028 					   int oldapic, int oldpin,
2029 					   int newapic, int newpin)
2030 {
2031 	struct irq_pin_list *entry;
2032 
2033 	for_each_irq_pin(entry, data->irq_2_pin) {
2034 		if (entry->apic == oldapic && entry->pin == oldpin) {
2035 			entry->apic = newapic;
2036 			entry->pin = newpin;
2037 			return;
2038 		}
2039 	}
2040 
2041 	/* Old apic/pin didn't exist, so just add a new one */
2042 	add_pin_to_irq_node(data, node, newapic, newpin);
2043 }
2044 
2045 /*
2046  * This code may look a bit paranoid, but it's supposed to cooperate with
2047  * a wide range of boards and BIOS bugs.  Fortunately only the timer IRQ
2048  * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
2049  * fanatically on his truly buggy board.
2050  */
check_timer(void)2051 static inline void __init check_timer(void)
2052 {
2053 	struct irq_data *irq_data = irq_get_irq_data(0);
2054 	struct mp_chip_data *data = irq_data->chip_data;
2055 	struct irq_cfg *cfg = irqd_cfg(irq_data);
2056 	int node = cpu_to_node(0);
2057 	int apic1, pin1, apic2, pin2;
2058 	int no_pin1 = 0;
2059 
2060 	if (!global_clock_event)
2061 		return;
2062 
2063 	local_irq_disable();
2064 
2065 	/*
2066 	 * get/set the timer IRQ vector:
2067 	 */
2068 	legacy_pic->mask(0);
2069 
2070 	/*
2071 	 * As IRQ0 is to be enabled in the 8259A, the virtual
2072 	 * wire has to be disabled in the local APIC.  Also
2073 	 * timer interrupts need to be acknowledged manually in
2074 	 * the 8259A for the i82489DX when using the NMI
2075 	 * watchdog as that APIC treats NMIs as level-triggered.
2076 	 * The AEOI mode will finish them in the 8259A
2077 	 * automatically.
2078 	 */
2079 	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2080 	legacy_pic->init(1);
2081 
2082 	pin1  = find_isa_irq_pin(0, mp_INT);
2083 	apic1 = find_isa_irq_apic(0, mp_INT);
2084 	pin2  = ioapic_i8259.pin;
2085 	apic2 = ioapic_i8259.apic;
2086 
2087 	pr_info("..TIMER: vector=0x%02X apic1=%d pin1=%d apic2=%d pin2=%d\n",
2088 		cfg->vector, apic1, pin1, apic2, pin2);
2089 
2090 	/*
2091 	 * Some BIOS writers are clueless and report the ExtINTA
2092 	 * I/O APIC input from the cascaded 8259A as the timer
2093 	 * interrupt input.  So just in case, if only one pin
2094 	 * was found above, try it both directly and through the
2095 	 * 8259A.
2096 	 */
2097 	if (pin1 == -1) {
2098 		panic_if_irq_remap(FW_BUG "Timer not connected to IO-APIC");
2099 		pin1 = pin2;
2100 		apic1 = apic2;
2101 		no_pin1 = 1;
2102 	} else if (pin2 == -1) {
2103 		pin2 = pin1;
2104 		apic2 = apic1;
2105 	}
2106 
2107 	if (pin1 != -1) {
2108 		/* Ok, does IRQ0 through the IOAPIC work? */
2109 		if (no_pin1) {
2110 			mp_alloc_timer_irq(apic1, pin1);
2111 		} else {
2112 			/*
2113 			 * for edge trigger, it's already unmasked,
2114 			 * so only need to unmask if it is level-trigger
2115 			 * do we really have level trigger timer?
2116 			 */
2117 			int idx = find_irq_entry(apic1, pin1, mp_INT);
2118 
2119 			if (idx != -1 && irq_is_level(idx))
2120 				unmask_ioapic_irq(irq_get_irq_data(0));
2121 		}
2122 		irq_domain_deactivate_irq(irq_data);
2123 		irq_domain_activate_irq(irq_data, false);
2124 		if (timer_irq_works()) {
2125 			if (disable_timer_pin_1 > 0)
2126 				clear_IO_APIC_pin(0, pin1);
2127 			goto out;
2128 		}
2129 		panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC");
2130 		clear_IO_APIC_pin(apic1, pin1);
2131 		if (!no_pin1)
2132 			pr_err("..MP-BIOS bug: 8254 timer not connected to IO-APIC\n");
2133 
2134 		pr_info("...trying to set up timer (IRQ0) through the 8259A ...\n");
2135 		pr_info("..... (found apic %d pin %d) ...\n", apic2, pin2);
2136 		/*
2137 		 * legacy devices should be connected to IO APIC #0
2138 		 */
2139 		replace_pin_at_irq_node(data, node, apic1, pin1, apic2, pin2);
2140 		irq_domain_deactivate_irq(irq_data);
2141 		irq_domain_activate_irq(irq_data, false);
2142 		legacy_pic->unmask(0);
2143 		if (timer_irq_works()) {
2144 			pr_info("....... works.\n");
2145 			goto out;
2146 		}
2147 		/*
2148 		 * Cleanup, just in case ...
2149 		 */
2150 		legacy_pic->mask(0);
2151 		clear_IO_APIC_pin(apic2, pin2);
2152 		pr_info("....... failed.\n");
2153 	}
2154 
2155 	pr_info("...trying to set up timer as Virtual Wire IRQ...\n");
2156 
2157 	lapic_register_intr(0);
2158 	apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);	/* Fixed mode */
2159 	legacy_pic->unmask(0);
2160 
2161 	if (timer_irq_works()) {
2162 		pr_info("..... works.\n");
2163 		goto out;
2164 	}
2165 	legacy_pic->mask(0);
2166 	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2167 	pr_info("..... failed.\n");
2168 
2169 	pr_info("...trying to set up timer as ExtINT IRQ...\n");
2170 
2171 	legacy_pic->init(0);
2172 	legacy_pic->make_irq(0);
2173 	apic_write(APIC_LVT0, APIC_DM_EXTINT);
2174 	legacy_pic->unmask(0);
2175 
2176 	unlock_ExtINT_logic();
2177 
2178 	if (timer_irq_works()) {
2179 		pr_info("..... works.\n");
2180 		goto out;
2181 	}
2182 
2183 	pr_info("..... failed :\n");
2184 	if (apic_is_x2apic_enabled()) {
2185 		pr_info("Perhaps problem with the pre-enabled x2apic mode\n"
2186 			"Try booting with x2apic and interrupt-remapping disabled in the bios.\n");
2187 	}
2188 	panic("IO-APIC + timer doesn't work!  Boot with apic=debug and send a "
2189 		"report.  Then try booting with the 'noapic' option.\n");
2190 out:
2191 	local_irq_enable();
2192 }
2193 
2194 /*
2195  * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2196  * to devices.  However there may be an I/O APIC pin available for
2197  * this interrupt regardless.  The pin may be left unconnected, but
2198  * typically it will be reused as an ExtINT cascade interrupt for
2199  * the master 8259A.  In the MPS case such a pin will normally be
2200  * reported as an ExtINT interrupt in the MP table.  With ACPI
2201  * there is no provision for ExtINT interrupts, and in the absence
2202  * of an override it would be treated as an ordinary ISA I/O APIC
2203  * interrupt, that is edge-triggered and unmasked by default.  We
2204  * used to do this, but it caused problems on some systems because
2205  * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2206  * the same ExtINT cascade interrupt to drive the local APIC of the
2207  * bootstrap processor.  Therefore we refrain from routing IRQ2 to
2208  * the I/O APIC in all cases now.  No actual device should request
2209  * it anyway.  --macro
2210  */
2211 #define PIC_IRQS	(1UL << PIC_CASCADE_IR)
2212 
mp_irqdomain_create(int ioapic)2213 static int mp_irqdomain_create(int ioapic)
2214 {
2215 	struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2216 	int hwirqs = mp_ioapic_pin_count(ioapic);
2217 	struct ioapic *ip = &ioapics[ioapic];
2218 	struct ioapic_domain_cfg *cfg = &ip->irqdomain_cfg;
2219 	struct irq_domain *parent;
2220 	struct fwnode_handle *fn;
2221 	struct irq_fwspec fwspec;
2222 
2223 	if (cfg->type == IOAPIC_DOMAIN_INVALID)
2224 		return 0;
2225 
2226 	/* Handle device tree enumerated APICs proper */
2227 	if (cfg->dev) {
2228 		fn = of_node_to_fwnode(cfg->dev);
2229 	} else {
2230 		fn = irq_domain_alloc_named_id_fwnode("IO-APIC", mpc_ioapic_id(ioapic));
2231 		if (!fn)
2232 			return -ENOMEM;
2233 	}
2234 
2235 	fwspec.fwnode = fn;
2236 	fwspec.param_count = 1;
2237 	fwspec.param[0] = mpc_ioapic_id(ioapic);
2238 
2239 	parent = irq_find_matching_fwspec(&fwspec, DOMAIN_BUS_GENERIC_MSI);
2240 	if (!parent) {
2241 		if (!cfg->dev)
2242 			irq_domain_free_fwnode(fn);
2243 		return -ENODEV;
2244 	}
2245 
2246 	ip->irqdomain = irq_domain_create_hierarchy(parent, 0, hwirqs, fn, cfg->ops,
2247 						    (void *)(long)ioapic);
2248 	if (!ip->irqdomain) {
2249 		/* Release fw handle if it was allocated above */
2250 		if (!cfg->dev)
2251 			irq_domain_free_fwnode(fn);
2252 		return -ENOMEM;
2253 	}
2254 
2255 	if (cfg->type == IOAPIC_DOMAIN_LEGACY || cfg->type == IOAPIC_DOMAIN_STRICT)
2256 		ioapic_dynirq_base = max(ioapic_dynirq_base, gsi_cfg->gsi_end + 1);
2257 
2258 	return 0;
2259 }
2260 
ioapic_destroy_irqdomain(int idx)2261 static void ioapic_destroy_irqdomain(int idx)
2262 {
2263 	struct ioapic_domain_cfg *cfg = &ioapics[idx].irqdomain_cfg;
2264 	struct fwnode_handle *fn = ioapics[idx].irqdomain->fwnode;
2265 
2266 	if (ioapics[idx].irqdomain) {
2267 		irq_domain_remove(ioapics[idx].irqdomain);
2268 		if (!cfg->dev)
2269 			irq_domain_free_fwnode(fn);
2270 		ioapics[idx].irqdomain = NULL;
2271 	}
2272 }
2273 
setup_IO_APIC(void)2274 void __init setup_IO_APIC(void)
2275 {
2276 	int ioapic;
2277 
2278 	if (ioapic_is_disabled || !nr_ioapics)
2279 		return;
2280 
2281 	io_apic_irqs = nr_legacy_irqs() ? ~PIC_IRQS : ~0UL;
2282 
2283 	apic_pr_verbose("ENABLING IO-APIC IRQs\n");
2284 	for_each_ioapic(ioapic)
2285 		BUG_ON(mp_irqdomain_create(ioapic));
2286 
2287 	/* Set up IO-APIC IRQ routing. */
2288 	x86_init.mpparse.setup_ioapic_ids();
2289 
2290 	sync_Arb_IDs();
2291 	setup_IO_APIC_irqs();
2292 	init_IO_APIC_traps();
2293 	if (nr_legacy_irqs())
2294 		check_timer();
2295 
2296 	ioapic_initialized = 1;
2297 }
2298 
resume_ioapic_id(int ioapic_idx)2299 static void resume_ioapic_id(int ioapic_idx)
2300 {
2301 	union IO_APIC_reg_00 reg_00;
2302 
2303 	guard(raw_spinlock_irqsave)(&ioapic_lock);
2304 	reg_00.raw = io_apic_read(ioapic_idx, 0);
2305 	if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) {
2306 		reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2307 		io_apic_write(ioapic_idx, 0, reg_00.raw);
2308 	}
2309 }
2310 
ioapic_resume(void)2311 static void ioapic_resume(void)
2312 {
2313 	int ioapic_idx;
2314 
2315 	for_each_ioapic_reverse(ioapic_idx)
2316 		resume_ioapic_id(ioapic_idx);
2317 
2318 	restore_ioapic_entries();
2319 }
2320 
2321 static struct syscore_ops ioapic_syscore_ops = {
2322 	.suspend	= save_ioapic_entries,
2323 	.resume		= ioapic_resume,
2324 };
2325 
ioapic_init_ops(void)2326 static int __init ioapic_init_ops(void)
2327 {
2328 	register_syscore_ops(&ioapic_syscore_ops);
2329 
2330 	return 0;
2331 }
2332 
2333 device_initcall(ioapic_init_ops);
2334 
io_apic_get_redir_entries(int ioapic)2335 static int io_apic_get_redir_entries(int ioapic)
2336 {
2337 	union IO_APIC_reg_01	reg_01;
2338 
2339 	guard(raw_spinlock_irqsave)(&ioapic_lock);
2340 	reg_01.raw = io_apic_read(ioapic, 1);
2341 
2342 	/*
2343 	 * The register returns the maximum index redir index supported,
2344 	 * which is one less than the total number of redir entries.
2345 	 */
2346 	return reg_01.bits.entries + 1;
2347 }
2348 
arch_dynirq_lower_bound(unsigned int from)2349 unsigned int arch_dynirq_lower_bound(unsigned int from)
2350 {
2351 	unsigned int ret;
2352 
2353 	/*
2354 	 * dmar_alloc_hwirq() may be called before setup_IO_APIC(), so use
2355 	 * gsi_top if ioapic_dynirq_base hasn't been initialized yet.
2356 	 */
2357 	ret = ioapic_dynirq_base ? : gsi_top;
2358 
2359 	/*
2360 	 * For DT enabled machines ioapic_dynirq_base is irrelevant and
2361 	 * always 0. gsi_top can be 0 if there is no IO/APIC registered.
2362 	 * 0 is an invalid interrupt number for dynamic allocations. Return
2363 	 * @from instead.
2364 	 */
2365 	return ret ? : from;
2366 }
2367 
2368 #ifdef CONFIG_X86_32
io_apic_get_unique_id(int ioapic,int apic_id)2369 static int io_apic_get_unique_id(int ioapic, int apic_id)
2370 {
2371 	static DECLARE_BITMAP(apic_id_map, MAX_LOCAL_APIC);
2372 	const u32 broadcast_id = 0xF;
2373 	union IO_APIC_reg_00 reg_00;
2374 	int i = 0;
2375 
2376 	/* Initialize the ID map */
2377 	if (bitmap_empty(apic_id_map, MAX_LOCAL_APIC))
2378 		copy_phys_cpu_present_map(apic_id_map);
2379 
2380 	scoped_guard (raw_spinlock_irqsave, &ioapic_lock)
2381 		reg_00.raw = io_apic_read(ioapic, 0);
2382 
2383 	if (apic_id >= broadcast_id) {
2384 		pr_warn("IOAPIC[%d]: Invalid apic_id %d, trying %d\n",
2385 			ioapic, apic_id, reg_00.bits.ID);
2386 		apic_id = reg_00.bits.ID;
2387 	}
2388 
2389 	/* Every APIC in a system must have a unique ID */
2390 	if (test_bit(apic_id, apic_id_map)) {
2391 		for (i = 0; i < broadcast_id; i++) {
2392 			if (!test_bit(i, apic_id_map))
2393 				break;
2394 		}
2395 
2396 		if (i == broadcast_id)
2397 			panic("Max apic_id exceeded!\n");
2398 
2399 		pr_warn("IOAPIC[%d]: apic_id %d already used, trying %d\n", ioapic, apic_id, i);
2400 		apic_id = i;
2401 	}
2402 
2403 	set_bit(apic_id, apic_id_map);
2404 
2405 	if (reg_00.bits.ID != apic_id) {
2406 		reg_00.bits.ID = apic_id;
2407 
2408 		scoped_guard (raw_spinlock_irqsave, &ioapic_lock) {
2409 			io_apic_write(ioapic, 0, reg_00.raw);
2410 			reg_00.raw = io_apic_read(ioapic, 0);
2411 		}
2412 
2413 		/* Sanity check */
2414 		if (reg_00.bits.ID != apic_id) {
2415 			pr_err("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
2416 			return -1;
2417 		}
2418 	}
2419 
2420 	apic_pr_verbose("IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
2421 
2422 	return apic_id;
2423 }
2424 
io_apic_unique_id(int idx,u8 id)2425 static u8 io_apic_unique_id(int idx, u8 id)
2426 {
2427 	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && !APIC_XAPIC(boot_cpu_apic_version))
2428 		return io_apic_get_unique_id(idx, id);
2429 	return id;
2430 }
2431 #else
io_apic_unique_id(int idx,u8 id)2432 static u8 io_apic_unique_id(int idx, u8 id)
2433 {
2434 	union IO_APIC_reg_00 reg_00;
2435 	DECLARE_BITMAP(used, 256);
2436 	u8 new_id;
2437 	int i;
2438 
2439 	bitmap_zero(used, 256);
2440 	for_each_ioapic(i)
2441 		__set_bit(mpc_ioapic_id(i), used);
2442 
2443 	/* Hand out the requested id if available */
2444 	if (!test_bit(id, used))
2445 		return id;
2446 
2447 	/*
2448 	 * Read the current id from the ioapic and keep it if
2449 	 * available.
2450 	 */
2451 	scoped_guard (raw_spinlock_irqsave, &ioapic_lock)
2452 		reg_00.raw = io_apic_read(idx, 0);
2453 
2454 	new_id = reg_00.bits.ID;
2455 	if (!test_bit(new_id, used)) {
2456 		apic_pr_verbose("IOAPIC[%d]: Using reg apic_id %d instead of %d\n",
2457 				idx, new_id, id);
2458 		return new_id;
2459 	}
2460 
2461 	/* Get the next free id and write it to the ioapic. */
2462 	new_id = find_first_zero_bit(used, 256);
2463 	reg_00.bits.ID = new_id;
2464 	scoped_guard (raw_spinlock_irqsave, &ioapic_lock) {
2465 		io_apic_write(idx, 0, reg_00.raw);
2466 		reg_00.raw = io_apic_read(idx, 0);
2467 	}
2468 	/* Sanity check */
2469 	BUG_ON(reg_00.bits.ID != new_id);
2470 
2471 	return new_id;
2472 }
2473 #endif
2474 
io_apic_get_version(int ioapic)2475 static int io_apic_get_version(int ioapic)
2476 {
2477 	union IO_APIC_reg_01 reg_01;
2478 
2479 	guard(raw_spinlock_irqsave)(&ioapic_lock);
2480 	reg_01.raw = io_apic_read(ioapic, 1);
2481 
2482 	return reg_01.bits.version;
2483 }
2484 
2485 /*
2486  * This function updates target affinity of IOAPIC interrupts to include
2487  * the CPUs which came online during SMP bringup.
2488  */
2489 #define IOAPIC_RESOURCE_NAME_SIZE 11
2490 
2491 static struct resource *ioapic_resources;
2492 
ioapic_setup_resources(void)2493 static struct resource * __init ioapic_setup_resources(void)
2494 {
2495 	struct resource *res;
2496 	unsigned long n;
2497 	char *mem;
2498 	int i;
2499 
2500 	if (nr_ioapics == 0)
2501 		return NULL;
2502 
2503 	n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
2504 	n *= nr_ioapics;
2505 
2506 	mem = memblock_alloc(n, SMP_CACHE_BYTES);
2507 	if (!mem)
2508 		panic("%s: Failed to allocate %lu bytes\n", __func__, n);
2509 	res = (void *)mem;
2510 
2511 	mem += sizeof(struct resource) * nr_ioapics;
2512 
2513 	for_each_ioapic(i) {
2514 		res[i].name = mem;
2515 		res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
2516 		snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
2517 		mem += IOAPIC_RESOURCE_NAME_SIZE;
2518 		ioapics[i].iomem_res = &res[i];
2519 	}
2520 
2521 	ioapic_resources = res;
2522 
2523 	return res;
2524 }
2525 
io_apic_set_fixmap(enum fixed_addresses idx,phys_addr_t phys)2526 static void io_apic_set_fixmap(enum fixed_addresses idx, phys_addr_t phys)
2527 {
2528 	pgprot_t flags = FIXMAP_PAGE_NOCACHE;
2529 
2530 	/*
2531 	 * Ensure fixmaps for IO-APIC MMIO respect memory encryption pgprot
2532 	 * bits, just like normal ioremap():
2533 	 */
2534 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
2535 		if (x86_platform.hyper.is_private_mmio(phys))
2536 			flags = pgprot_encrypted(flags);
2537 		else
2538 			flags = pgprot_decrypted(flags);
2539 	}
2540 
2541 	__set_fixmap(idx, phys, flags);
2542 }
2543 
io_apic_init_mappings(void)2544 void __init io_apic_init_mappings(void)
2545 {
2546 	unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
2547 	struct resource *ioapic_res;
2548 	int i;
2549 
2550 	ioapic_res = ioapic_setup_resources();
2551 	for_each_ioapic(i) {
2552 		if (smp_found_config) {
2553 			ioapic_phys = mpc_ioapic_addr(i);
2554 #ifdef CONFIG_X86_32
2555 			if (!ioapic_phys) {
2556 				pr_err("WARNING: bogus zero IO-APIC address found in MPTABLE, "
2557 				       "disabling IO/APIC support!\n");
2558 				smp_found_config = 0;
2559 				ioapic_is_disabled = true;
2560 				goto fake_ioapic_page;
2561 			}
2562 #endif
2563 		} else {
2564 #ifdef CONFIG_X86_32
2565 fake_ioapic_page:
2566 #endif
2567 			ioapic_phys = (unsigned long)memblock_alloc(PAGE_SIZE,
2568 								    PAGE_SIZE);
2569 			if (!ioapic_phys)
2570 				panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
2571 				      __func__, PAGE_SIZE, PAGE_SIZE);
2572 			ioapic_phys = __pa(ioapic_phys);
2573 		}
2574 		io_apic_set_fixmap(idx, ioapic_phys);
2575 		apic_pr_verbose("mapped IOAPIC to %08lx (%08lx)\n",
2576 				__fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK), ioapic_phys);
2577 		idx++;
2578 
2579 		ioapic_res->start = ioapic_phys;
2580 		ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
2581 		ioapic_res++;
2582 	}
2583 }
2584 
ioapic_insert_resources(void)2585 void __init ioapic_insert_resources(void)
2586 {
2587 	struct resource *r = ioapic_resources;
2588 	int i;
2589 
2590 	if (!r) {
2591 		if (nr_ioapics > 0)
2592 			pr_err("IO APIC resources couldn't be allocated.\n");
2593 		return;
2594 	}
2595 
2596 	for_each_ioapic(i) {
2597 		insert_resource(&iomem_resource, r);
2598 		r++;
2599 	}
2600 }
2601 
mp_find_ioapic(u32 gsi)2602 int mp_find_ioapic(u32 gsi)
2603 {
2604 	int i;
2605 
2606 	if (nr_ioapics == 0)
2607 		return -1;
2608 
2609 	/* Find the IOAPIC that manages this GSI. */
2610 	for_each_ioapic(i) {
2611 		struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
2612 
2613 		if (gsi >= gsi_cfg->gsi_base && gsi <= gsi_cfg->gsi_end)
2614 			return i;
2615 	}
2616 
2617 	pr_err("ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
2618 	return -1;
2619 }
2620 
mp_find_ioapic_pin(int ioapic,u32 gsi)2621 int mp_find_ioapic_pin(int ioapic, u32 gsi)
2622 {
2623 	struct mp_ioapic_gsi *gsi_cfg;
2624 
2625 	if (WARN_ON(ioapic < 0))
2626 		return -1;
2627 
2628 	gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2629 	if (WARN_ON(gsi > gsi_cfg->gsi_end))
2630 		return -1;
2631 
2632 	return gsi - gsi_cfg->gsi_base;
2633 }
2634 
bad_ioapic_register(int idx)2635 static int bad_ioapic_register(int idx)
2636 {
2637 	union IO_APIC_reg_00 reg_00;
2638 	union IO_APIC_reg_01 reg_01;
2639 	union IO_APIC_reg_02 reg_02;
2640 
2641 	reg_00.raw = io_apic_read(idx, 0);
2642 	reg_01.raw = io_apic_read(idx, 1);
2643 	reg_02.raw = io_apic_read(idx, 2);
2644 
2645 	if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) {
2646 		pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n",
2647 			mpc_ioapic_addr(idx));
2648 		return 1;
2649 	}
2650 
2651 	return 0;
2652 }
2653 
find_free_ioapic_entry(void)2654 static int find_free_ioapic_entry(void)
2655 {
2656 	for (int idx = 0; idx < MAX_IO_APICS; idx++) {
2657 		if (ioapics[idx].nr_registers == 0)
2658 			return idx;
2659 	}
2660 	return MAX_IO_APICS;
2661 }
2662 
2663 /**
2664  * mp_register_ioapic - Register an IOAPIC device
2665  * @id:		hardware IOAPIC ID
2666  * @address:	physical address of IOAPIC register area
2667  * @gsi_base:	base of GSI associated with the IOAPIC
2668  * @cfg:	configuration information for the IOAPIC
2669  */
mp_register_ioapic(int id,u32 address,u32 gsi_base,struct ioapic_domain_cfg * cfg)2670 int mp_register_ioapic(int id, u32 address, u32 gsi_base, struct ioapic_domain_cfg *cfg)
2671 {
2672 	bool hotplug = !!ioapic_initialized;
2673 	struct mp_ioapic_gsi *gsi_cfg;
2674 	int idx, ioapic, entries;
2675 	u32 gsi_end;
2676 
2677 	if (!address) {
2678 		pr_warn("Bogus (zero) I/O APIC address found, skipping!\n");
2679 		return -EINVAL;
2680 	}
2681 
2682 	for_each_ioapic(ioapic) {
2683 		if (ioapics[ioapic].mp_config.apicaddr == address) {
2684 			pr_warn("address 0x%x conflicts with IOAPIC%d\n", address, ioapic);
2685 			return -EEXIST;
2686 		}
2687 	}
2688 
2689 	idx = find_free_ioapic_entry();
2690 	if (idx >= MAX_IO_APICS) {
2691 		pr_warn("Max # of I/O APICs (%d) exceeded (found %d), skipping\n",
2692 			MAX_IO_APICS, idx);
2693 		return -ENOSPC;
2694 	}
2695 
2696 	ioapics[idx].mp_config.type = MP_IOAPIC;
2697 	ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
2698 	ioapics[idx].mp_config.apicaddr = address;
2699 
2700 	io_apic_set_fixmap(FIX_IO_APIC_BASE_0 + idx, address);
2701 	if (bad_ioapic_register(idx)) {
2702 		clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2703 		return -ENODEV;
2704 	}
2705 
2706 	ioapics[idx].mp_config.apicid = io_apic_unique_id(idx, id);
2707 	ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
2708 
2709 	/*
2710 	 * Build basic GSI lookup table to facilitate gsi->io_apic lookups
2711 	 * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
2712 	 */
2713 	entries = io_apic_get_redir_entries(idx);
2714 	gsi_end = gsi_base + entries - 1;
2715 	for_each_ioapic(ioapic) {
2716 		gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2717 		if ((gsi_base >= gsi_cfg->gsi_base &&
2718 		     gsi_base <= gsi_cfg->gsi_end) ||
2719 		    (gsi_end >= gsi_cfg->gsi_base &&
2720 		     gsi_end <= gsi_cfg->gsi_end)) {
2721 			pr_warn("GSI range [%u-%u] for new IOAPIC conflicts with GSI[%u-%u]\n",
2722 				gsi_base, gsi_end, gsi_cfg->gsi_base, gsi_cfg->gsi_end);
2723 			clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2724 			return -ENOSPC;
2725 		}
2726 	}
2727 	gsi_cfg = mp_ioapic_gsi_routing(idx);
2728 	gsi_cfg->gsi_base = gsi_base;
2729 	gsi_cfg->gsi_end = gsi_end;
2730 
2731 	ioapics[idx].irqdomain = NULL;
2732 	ioapics[idx].irqdomain_cfg = *cfg;
2733 
2734 	/*
2735 	 * If mp_register_ioapic() is called during early boot stage when
2736 	 * walking ACPI/DT tables, it's too early to create irqdomain,
2737 	 * we are still using bootmem allocator. So delay it to setup_IO_APIC().
2738 	 */
2739 	if (hotplug) {
2740 		if (mp_irqdomain_create(idx)) {
2741 			clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2742 			return -ENOMEM;
2743 		}
2744 		alloc_ioapic_saved_registers(idx);
2745 	}
2746 
2747 	if (gsi_cfg->gsi_end >= gsi_top)
2748 		gsi_top = gsi_cfg->gsi_end + 1;
2749 	if (nr_ioapics <= idx)
2750 		nr_ioapics = idx + 1;
2751 
2752 	/* Set nr_registers to mark entry present */
2753 	ioapics[idx].nr_registers = entries;
2754 
2755 	pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n",
2756 		idx, mpc_ioapic_id(idx), mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
2757 		gsi_cfg->gsi_base, gsi_cfg->gsi_end);
2758 
2759 	return 0;
2760 }
2761 
mp_unregister_ioapic(u32 gsi_base)2762 int mp_unregister_ioapic(u32 gsi_base)
2763 {
2764 	int ioapic, pin;
2765 	int found = 0;
2766 
2767 	for_each_ioapic(ioapic) {
2768 		if (ioapics[ioapic].gsi_config.gsi_base == gsi_base) {
2769 			found = 1;
2770 			break;
2771 		}
2772 	}
2773 
2774 	if (!found) {
2775 		pr_warn("can't find IOAPIC for GSI %d\n", gsi_base);
2776 		return -ENODEV;
2777 	}
2778 
2779 	for_each_pin(ioapic, pin) {
2780 		u32 gsi = mp_pin_to_gsi(ioapic, pin);
2781 		int irq = mp_map_gsi_to_irq(gsi, 0, NULL);
2782 		struct mp_chip_data *data;
2783 
2784 		if (irq >= 0) {
2785 			data = irq_get_chip_data(irq);
2786 			if (data && data->count) {
2787 				pr_warn("pin%d on IOAPIC%d is still in use.\n",	pin, ioapic);
2788 				return -EBUSY;
2789 			}
2790 		}
2791 	}
2792 
2793 	/* Mark entry not present */
2794 	ioapics[ioapic].nr_registers  = 0;
2795 	ioapic_destroy_irqdomain(ioapic);
2796 	free_ioapic_saved_registers(ioapic);
2797 	if (ioapics[ioapic].iomem_res)
2798 		release_resource(ioapics[ioapic].iomem_res);
2799 	clear_fixmap(FIX_IO_APIC_BASE_0 + ioapic);
2800 	memset(&ioapics[ioapic], 0, sizeof(ioapics[ioapic]));
2801 
2802 	return 0;
2803 }
2804 
mp_ioapic_registered(u32 gsi_base)2805 int mp_ioapic_registered(u32 gsi_base)
2806 {
2807 	int ioapic;
2808 
2809 	for_each_ioapic(ioapic)
2810 		if (ioapics[ioapic].gsi_config.gsi_base == gsi_base)
2811 			return 1;
2812 
2813 	return 0;
2814 }
2815 
mp_irqdomain_get_attr(u32 gsi,struct mp_chip_data * data,struct irq_alloc_info * info)2816 static void mp_irqdomain_get_attr(u32 gsi, struct mp_chip_data *data,
2817 				  struct irq_alloc_info *info)
2818 {
2819 	if (info && info->ioapic.valid) {
2820 		data->is_level = info->ioapic.is_level;
2821 		data->active_low = info->ioapic.active_low;
2822 	} else if (__acpi_get_override_irq(gsi, &data->is_level, &data->active_low) < 0) {
2823 		/* PCI interrupts are always active low level triggered. */
2824 		data->is_level = true;
2825 		data->active_low = true;
2826 	}
2827 }
2828 
2829 /*
2830  * Configure the I/O-APIC specific fields in the routing entry.
2831  *
2832  * This is important to setup the I/O-APIC specific bits (is_level,
2833  * active_low, masked) because the underlying parent domain will only
2834  * provide the routing information and is oblivious of the I/O-APIC
2835  * specific bits.
2836  *
2837  * The entry is just preconfigured at this point and not written into the
2838  * RTE. This happens later during activation which will fill in the actual
2839  * routing information.
2840  */
mp_preconfigure_entry(struct mp_chip_data * data)2841 static void mp_preconfigure_entry(struct mp_chip_data *data)
2842 {
2843 	struct IO_APIC_route_entry *entry = &data->entry;
2844 
2845 	memset(entry, 0, sizeof(*entry));
2846 	entry->is_level		 = data->is_level;
2847 	entry->active_low	 = data->active_low;
2848 	/*
2849 	 * Mask level triggered irqs. Edge triggered irqs are masked
2850 	 * by the irq core code in case they fire.
2851 	 */
2852 	entry->masked		= data->is_level;
2853 }
2854 
mp_irqdomain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)2855 int mp_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
2856 		       unsigned int nr_irqs, void *arg)
2857 {
2858 	struct irq_alloc_info *info = arg;
2859 	struct mp_chip_data *data;
2860 	struct irq_data *irq_data;
2861 	int ret, ioapic, pin;
2862 	unsigned long flags;
2863 
2864 	if (!info || nr_irqs > 1)
2865 		return -EINVAL;
2866 	irq_data = irq_domain_get_irq_data(domain, virq);
2867 	if (!irq_data)
2868 		return -EINVAL;
2869 
2870 	ioapic = mp_irqdomain_ioapic_idx(domain);
2871 	pin = info->ioapic.pin;
2872 	if (irq_find_mapping(domain, (irq_hw_number_t)pin) > 0)
2873 		return -EEXIST;
2874 
2875 	data = kzalloc(sizeof(*data), GFP_KERNEL);
2876 	if (!data)
2877 		return -ENOMEM;
2878 
2879 	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, info);
2880 	if (ret < 0)
2881 		goto free_data;
2882 
2883 	INIT_LIST_HEAD(&data->irq_2_pin);
2884 	irq_data->hwirq = info->ioapic.pin;
2885 	irq_data->chip = (domain->parent == x86_vector_domain) ?
2886 			  &ioapic_chip : &ioapic_ir_chip;
2887 	irq_data->chip_data = data;
2888 	mp_irqdomain_get_attr(mp_pin_to_gsi(ioapic, pin), data, info);
2889 
2890 	if (!add_pin_to_irq_node(data, ioapic_alloc_attr_node(info), ioapic, pin)) {
2891 		ret = -ENOMEM;
2892 		goto free_irqs;
2893 	}
2894 
2895 	mp_preconfigure_entry(data);
2896 	mp_register_handler(virq, data->is_level);
2897 
2898 	local_irq_save(flags);
2899 	if (virq < nr_legacy_irqs())
2900 		legacy_pic->mask(virq);
2901 	local_irq_restore(flags);
2902 
2903 	apic_pr_verbose("IOAPIC[%d]: Preconfigured routing entry (%d-%d -> IRQ %d Level:%i ActiveLow:%i)\n",
2904 			ioapic, mpc_ioapic_id(ioapic), pin, virq, data->is_level, data->active_low);
2905 	return 0;
2906 
2907 free_irqs:
2908 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
2909 free_data:
2910 	kfree(data);
2911 	return ret;
2912 }
2913 
mp_irqdomain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)2914 void mp_irqdomain_free(struct irq_domain *domain, unsigned int virq,
2915 		       unsigned int nr_irqs)
2916 {
2917 	struct irq_data *irq_data;
2918 	struct mp_chip_data *data;
2919 
2920 	BUG_ON(nr_irqs != 1);
2921 	irq_data = irq_domain_get_irq_data(domain, virq);
2922 	if (irq_data && irq_data->chip_data) {
2923 		data = irq_data->chip_data;
2924 		__remove_pin_from_irq(data, mp_irqdomain_ioapic_idx(domain), (int)irq_data->hwirq);
2925 		WARN_ON(!list_empty(&data->irq_2_pin));
2926 		kfree(irq_data->chip_data);
2927 	}
2928 	irq_domain_free_irqs_top(domain, virq, nr_irqs);
2929 }
2930 
mp_irqdomain_activate(struct irq_domain * domain,struct irq_data * irq_data,bool reserve)2931 int mp_irqdomain_activate(struct irq_domain *domain, struct irq_data *irq_data, bool reserve)
2932 {
2933 	guard(raw_spinlock_irqsave)(&ioapic_lock);
2934 	ioapic_configure_entry(irq_data);
2935 	return 0;
2936 }
2937 
mp_irqdomain_deactivate(struct irq_domain * domain,struct irq_data * irq_data)2938 void mp_irqdomain_deactivate(struct irq_domain *domain,
2939 			     struct irq_data *irq_data)
2940 {
2941 	/* It won't be called for IRQ with multiple IOAPIC pins associated */
2942 	ioapic_mask_entry(mp_irqdomain_ioapic_idx(domain), (int)irq_data->hwirq);
2943 }
2944 
mp_irqdomain_ioapic_idx(struct irq_domain * domain)2945 int mp_irqdomain_ioapic_idx(struct irq_domain *domain)
2946 {
2947 	return (int)(long)domain->host_data;
2948 }
2949 
2950 const struct irq_domain_ops mp_ioapic_irqdomain_ops = {
2951 	.alloc		= mp_irqdomain_alloc,
2952 	.free		= mp_irqdomain_free,
2953 	.activate	= mp_irqdomain_activate,
2954 	.deactivate	= mp_irqdomain_deactivate,
2955 };
2956