1*1e607cd8Skettenis /* $OpenBSD: intr.c,v 1.60 2024/06/15 18:01:44 kettenis Exp $ */
2f5df1827Smickey /* $NetBSD: intr.c,v 1.3 2003/03/03 22:16:20 fvdl Exp $ */
3f5df1827Smickey
4f5df1827Smickey /*
5f5df1827Smickey * Copyright 2002 (c) Wasabi Systems, Inc.
6f5df1827Smickey * All rights reserved.
7f5df1827Smickey *
8f5df1827Smickey * Written by Frank van der Linden for Wasabi Systems, Inc.
9f5df1827Smickey *
10f5df1827Smickey * Redistribution and use in source and binary forms, with or without
11f5df1827Smickey * modification, are permitted provided that the following conditions
12f5df1827Smickey * are met:
13f5df1827Smickey * 1. Redistributions of source code must retain the above copyright
14f5df1827Smickey * notice, this list of conditions and the following disclaimer.
15f5df1827Smickey * 2. Redistributions in binary form must reproduce the above copyright
16f5df1827Smickey * notice, this list of conditions and the following disclaimer in the
17f5df1827Smickey * documentation and/or other materials provided with the distribution.
18f5df1827Smickey * 3. All advertising materials mentioning features or use of this software
19f5df1827Smickey * must display the following acknowledgement:
20f5df1827Smickey * This product includes software developed for the NetBSD Project by
21f5df1827Smickey * Wasabi Systems, Inc.
22f5df1827Smickey * 4. The name of Wasabi Systems, Inc. may not be used to endorse
23f5df1827Smickey * or promote products derived from this software without specific prior
24f5df1827Smickey * written permission.
25f5df1827Smickey *
26f5df1827Smickey * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
27f5df1827Smickey * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28f5df1827Smickey * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29f5df1827Smickey * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
30f5df1827Smickey * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31f5df1827Smickey * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32f5df1827Smickey * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33f5df1827Smickey * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34f5df1827Smickey * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35f5df1827Smickey * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36f5df1827Smickey * POSSIBILITY OF SUCH DAMAGE.
37f5df1827Smickey */
38f5df1827Smickey
39f5df1827Smickey /* #define INTRDEBUG */
40f5df1827Smickey
41f5df1827Smickey #include <sys/param.h>
42f5df1827Smickey #include <sys/systm.h>
43f5df1827Smickey #include <sys/kernel.h>
44f5df1827Smickey #include <sys/syslog.h>
45f5df1827Smickey #include <sys/device.h>
46f5df1827Smickey #include <sys/malloc.h>
47f5df1827Smickey #include <sys/errno.h>
48f5df1827Smickey
49f5df1827Smickey #include <machine/atomic.h>
50f5df1827Smickey #include <machine/i8259.h>
51f5df1827Smickey #include <machine/cpu.h>
52f5df1827Smickey #include <machine/pio.h>
53a19c7295Sart #include <machine/cpufunc.h>
54f5df1827Smickey
55f5df1827Smickey #include "lapic.h"
56ad5f3503Smikeb #include "xen.h"
57218ead0bSmikeb #include "hyperv.h"
58f5df1827Smickey
59f5df1827Smickey #if NLAPIC > 0
60f5df1827Smickey #include <machine/i82489var.h>
61f5df1827Smickey #endif
62f5df1827Smickey
63f5df1827Smickey struct pic softintr_pic = {
643534b73fSjsg {0, {NULL}, NULL, 0, "softintr_pic0", NULL, 0, 0},
65f5df1827Smickey PIC_SOFT,
66f5df1827Smickey #ifdef MULTIPROCESSOR
67b5b9857bSart {},
68f5df1827Smickey #endif
69f5df1827Smickey NULL,
70f5df1827Smickey NULL,
71f5df1827Smickey NULL,
72f5df1827Smickey NULL,
73f5df1827Smickey NULL,
74f5df1827Smickey };
75f5df1827Smickey
76*1e607cd8Skettenis int intr_suspended;
77*1e607cd8Skettenis
78f5df1827Smickey /*
79f5df1827Smickey * Fill in default interrupt table (in case of spurious interrupt
80f5df1827Smickey * during configuration of kernel), setup interrupt control unit
81f5df1827Smickey */
82f5df1827Smickey void
intr_default_setup(void)83f5df1827Smickey intr_default_setup(void)
84f5df1827Smickey {
85f5df1827Smickey int i;
86f5df1827Smickey
87f5df1827Smickey /* icu vectors */
88f5df1827Smickey for (i = 0; i < NUM_LEGACY_IRQS; i++) {
89f5df1827Smickey idt_allocmap[ICU_OFFSET + i] = 1;
90f5df1827Smickey setgate(&idt[ICU_OFFSET + i],
91f5df1827Smickey i8259_stubs[i].ist_entry, 0, SDT_SYS386IGT,
92f5df1827Smickey SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
93f5df1827Smickey }
94f5df1827Smickey
95f5df1827Smickey /*
96f5df1827Smickey * Eventually might want to check if it's actually there.
97f5df1827Smickey */
98f5df1827Smickey i8259_default_setup();
99f5df1827Smickey }
100f5df1827Smickey
101f5df1827Smickey /*
102f5df1827Smickey * Handle a NMI, possibly a machine check.
103f5df1827Smickey * return true to panic system, false to ignore.
104f5df1827Smickey */
105f5df1827Smickey int
x86_nmi(void)106f5df1827Smickey x86_nmi(void)
107f5df1827Smickey {
108f5df1827Smickey log(LOG_CRIT, "NMI port 61 %x, port 70 %x\n", inb(0x61), inb(0x70));
109f5df1827Smickey return(0);
110f5df1827Smickey }
111f5df1827Smickey
112f5df1827Smickey /*
113f5df1827Smickey * Recalculate the interrupt masks from scratch.
114f5df1827Smickey */
115f5df1827Smickey void
intr_calculatemasks(struct cpu_info * ci)116f5df1827Smickey intr_calculatemasks(struct cpu_info *ci)
117f5df1827Smickey {
118092bf81aSkettenis int irq, level;
119092bf81aSkettenis u_int64_t unusedirqs, intrlevel[MAX_INTR_SOURCES];
120f5df1827Smickey struct intrhand *q;
121f5df1827Smickey
122f5df1827Smickey /* First, figure out which levels each IRQ uses. */
123092bf81aSkettenis unusedirqs = 0xffffffffffffffffUL;
124f5df1827Smickey for (irq = 0; irq < MAX_INTR_SOURCES; irq++) {
125f5df1827Smickey int levels = 0;
126f5df1827Smickey
127f5df1827Smickey if (ci->ci_isources[irq] == NULL) {
128f5df1827Smickey intrlevel[irq] = 0;
129f5df1827Smickey continue;
130f5df1827Smickey }
131f5df1827Smickey for (q = ci->ci_isources[irq]->is_handlers; q; q = q->ih_next)
132092bf81aSkettenis levels |= (1 << q->ih_level);
133f5df1827Smickey intrlevel[irq] = levels;
134f5df1827Smickey if (levels)
135092bf81aSkettenis unusedirqs &= ~(1UL << irq);
136f5df1827Smickey }
137f5df1827Smickey
138f5df1827Smickey /* Then figure out which IRQs use each level. */
139f5df1827Smickey for (level = 0; level < NIPL; level++) {
140092bf81aSkettenis u_int64_t irqs = 0;
141f5df1827Smickey for (irq = 0; irq < MAX_INTR_SOURCES; irq++)
142f5df1827Smickey if (intrlevel[irq] & (1 << level))
143092bf81aSkettenis irqs |= (1UL << irq);
144f5df1827Smickey ci->ci_imask[level] = irqs | unusedirqs;
145f5df1827Smickey }
146f5df1827Smickey
147f5df1827Smickey for (level = 0; level< (NIPL - 1); level++)
148f5df1827Smickey ci->ci_imask[level + 1] |= ci->ci_imask[level];
149f5df1827Smickey
150f5df1827Smickey for (irq = 0; irq < MAX_INTR_SOURCES; irq++) {
151f5df1827Smickey int maxlevel = IPL_NONE;
152f5df1827Smickey int minlevel = IPL_HIGH;
153f5df1827Smickey
154f5df1827Smickey if (ci->ci_isources[irq] == NULL)
155f5df1827Smickey continue;
156f5df1827Smickey for (q = ci->ci_isources[irq]->is_handlers; q;
157f5df1827Smickey q = q->ih_next) {
158f5df1827Smickey if (q->ih_level < minlevel)
159f5df1827Smickey minlevel = q->ih_level;
160f5df1827Smickey if (q->ih_level > maxlevel)
161f5df1827Smickey maxlevel = q->ih_level;
162f5df1827Smickey }
163f5df1827Smickey ci->ci_isources[irq]->is_maxlevel = maxlevel;
164f5df1827Smickey ci->ci_isources[irq]->is_minlevel = minlevel;
165f5df1827Smickey }
166f5df1827Smickey
167f5df1827Smickey for (level = 0; level < NIPL; level++)
168f5df1827Smickey ci->ci_iunmask[level] = ~ci->ci_imask[level];
169f5df1827Smickey }
170f5df1827Smickey
171f5df1827Smickey int
intr_allocate_slot_cpu(struct cpu_info * ci,struct pic * pic,int pin,int * index)172f5df1827Smickey intr_allocate_slot_cpu(struct cpu_info *ci, struct pic *pic, int pin,
173f5df1827Smickey int *index)
174f5df1827Smickey {
175f5df1827Smickey int start, slot, i;
176f5df1827Smickey struct intrsource *isp;
177f5df1827Smickey
178f5df1827Smickey start = CPU_IS_PRIMARY(ci) ? NUM_LEGACY_IRQS : 0;
179f5df1827Smickey slot = -1;
180f5df1827Smickey
18164da5045Skettenis for (i = 0; i < start; i++) {
18264da5045Skettenis isp = ci->ci_isources[i];
18364da5045Skettenis if (isp != NULL && isp->is_pic == pic && isp->is_pin == pin) {
18464da5045Skettenis slot = i;
18564da5045Skettenis start = MAX_INTR_SOURCES;
18664da5045Skettenis break;
18764da5045Skettenis }
18864da5045Skettenis }
189f5df1827Smickey for (i = start; i < MAX_INTR_SOURCES ; i++) {
190f5df1827Smickey isp = ci->ci_isources[i];
191f5df1827Smickey if (isp != NULL && isp->is_pic == pic && isp->is_pin == pin) {
192f5df1827Smickey slot = i;
193f5df1827Smickey break;
194f5df1827Smickey }
195f5df1827Smickey if (isp == NULL && slot == -1) {
196f5df1827Smickey slot = i;
197f5df1827Smickey continue;
198f5df1827Smickey }
199f5df1827Smickey }
200f5df1827Smickey if (slot == -1) {
201f5df1827Smickey return EBUSY;
202f5df1827Smickey }
203f5df1827Smickey
204f5df1827Smickey isp = ci->ci_isources[slot];
205f5df1827Smickey if (isp == NULL) {
20651394e79Schl isp = malloc(sizeof (struct intrsource), M_DEVBUF,
20751394e79Schl M_NOWAIT|M_ZERO);
208f5df1827Smickey if (isp == NULL) {
209f5df1827Smickey return ENOMEM;
210f5df1827Smickey }
211f5df1827Smickey snprintf(isp->is_evname, sizeof (isp->is_evname),
212f5df1827Smickey "pin %d", pin);
213f5df1827Smickey ci->ci_isources[slot] = isp;
214f5df1827Smickey }
215f5df1827Smickey
216f5df1827Smickey *index = slot;
217f5df1827Smickey return 0;
218f5df1827Smickey }
219f5df1827Smickey
220f5df1827Smickey /*
221f5df1827Smickey * A simple round-robin allocator to assign interrupts to CPUs.
222f5df1827Smickey */
223f5df1827Smickey int
intr_allocate_slot(struct pic * pic,int legacy_irq,int pin,int level,struct cpu_info ** cip,int * index,int * idt_slot)224f5df1827Smickey intr_allocate_slot(struct pic *pic, int legacy_irq, int pin, int level,
225f5df1827Smickey struct cpu_info **cip, int *index, int *idt_slot)
226f5df1827Smickey {
227f5df1827Smickey CPU_INFO_ITERATOR cii;
228f5df1827Smickey struct cpu_info *ci;
229f5df1827Smickey struct intrsource *isp;
230f5df1827Smickey int slot, idtvec, error;
231f5df1827Smickey
232f5df1827Smickey /*
233f5df1827Smickey * If a legacy IRQ is wanted, try to use a fixed slot pointing
234f5df1827Smickey * at the primary CPU. In the case of IO APICs, multiple pins
235f5df1827Smickey * may map to one legacy IRQ, but they should not be shared
236f5df1827Smickey * in that case, so the first one gets the legacy slot, but
237f5df1827Smickey * a subsequent allocation with a different pin will get
238f5df1827Smickey * a different slot.
239f5df1827Smickey */
240f5df1827Smickey if (legacy_irq != -1) {
241f5df1827Smickey ci = &cpu_info_primary;
242898c0ff1Sbrad /* must check for duplicate pic + pin first */
243898c0ff1Sbrad for (slot = 0 ; slot < MAX_INTR_SOURCES ; slot++) {
244898c0ff1Sbrad isp = ci->ci_isources[slot];
245898c0ff1Sbrad if (isp != NULL && isp->is_pic == pic &&
246898c0ff1Sbrad isp->is_pin == pin ) {
247898c0ff1Sbrad goto duplicate;
248898c0ff1Sbrad }
249898c0ff1Sbrad }
250f5df1827Smickey slot = legacy_irq;
251f5df1827Smickey isp = ci->ci_isources[slot];
252f5df1827Smickey if (isp == NULL) {
25351394e79Schl isp = malloc(sizeof (struct intrsource), M_DEVBUF,
25451394e79Schl M_NOWAIT|M_ZERO);
255f5df1827Smickey if (isp == NULL)
256f5df1827Smickey return ENOMEM;
257f5df1827Smickey snprintf(isp->is_evname, sizeof (isp->is_evname),
258f5df1827Smickey "pin %d", pin);
259f5df1827Smickey
260f5df1827Smickey ci->ci_isources[slot] = isp;
261f5df1827Smickey } else {
26226f38a42Sniklas if (isp->is_pic != pic || isp->is_pin != pin) {
263f5df1827Smickey if (pic == &i8259_pic)
264f5df1827Smickey return EINVAL;
265f5df1827Smickey goto other;
266f5df1827Smickey }
267f5df1827Smickey }
268898c0ff1Sbrad duplicate:
269f5df1827Smickey if (pic == &i8259_pic)
270f5df1827Smickey idtvec = ICU_OFFSET + legacy_irq;
271f5df1827Smickey else {
272f5df1827Smickey #ifdef IOAPIC_HWMASK
273f5df1827Smickey if (level > isp->is_maxlevel) {
274f5df1827Smickey #else
275f5df1827Smickey if (isp->is_minlevel == 0 || level < isp->is_minlevel) {
276f5df1827Smickey #endif
277f5df1827Smickey idtvec = idt_vec_alloc(APIC_LEVEL(level),
278f5df1827Smickey IDT_INTR_HIGH);
279f5df1827Smickey if (idtvec == 0)
280f5df1827Smickey return EBUSY;
281f5df1827Smickey } else
282f5df1827Smickey idtvec = isp->is_idtvec;
283f5df1827Smickey }
284f5df1827Smickey } else {
285f5df1827Smickey other:
286f5df1827Smickey /*
28715db3095Sdlg * Otherwise, look for a free slot elsewhere. If cip is null, it
28815db3095Sdlg * means try primary cpu but accept secondary, otherwise we need
28915db3095Sdlg * a slot on the requested cpu.
290f5df1827Smickey */
29115db3095Sdlg if (*cip == NULL)
292f5df1827Smickey ci = &cpu_info_primary;
29315db3095Sdlg else
29415db3095Sdlg ci = *cip;
29515db3095Sdlg
296f5df1827Smickey error = intr_allocate_slot_cpu(ci, pic, pin, &slot);
297f5df1827Smickey if (error == 0)
298f5df1827Smickey goto found;
29915db3095Sdlg /* Can't alloc on the requested cpu, fail. */
30015db3095Sdlg if (*cip != NULL)
30115db3095Sdlg return EBUSY;
302f5df1827Smickey
303f5df1827Smickey /*
304f5df1827Smickey * ..now try the others.
305f5df1827Smickey */
3065d5ec4d4Smiod CPU_INFO_FOREACH(cii, ci) {
307f5df1827Smickey if (CPU_IS_PRIMARY(ci))
308f5df1827Smickey continue;
309f5df1827Smickey error = intr_allocate_slot_cpu(ci, pic, pin, &slot);
310f5df1827Smickey if (error == 0)
311f5df1827Smickey goto found;
312f5df1827Smickey }
313f5df1827Smickey return EBUSY;
314f5df1827Smickey found:
315fb432fadSkettenis if (pic->pic_allocidtvec) {
316fb432fadSkettenis idtvec = pic->pic_allocidtvec(pic, pin,
317fb432fadSkettenis APIC_LEVEL(level), IDT_INTR_HIGH);
318fb432fadSkettenis } else {
319fb432fadSkettenis idtvec = idt_vec_alloc(APIC_LEVEL(level),
320fb432fadSkettenis IDT_INTR_HIGH);
321fb432fadSkettenis }
322f5df1827Smickey if (idtvec == 0) {
323fb432fadSkettenis free(ci->ci_isources[slot], M_DEVBUF,
324fb432fadSkettenis sizeof (struct intrsource));
325f5df1827Smickey ci->ci_isources[slot] = NULL;
326f5df1827Smickey return EBUSY;
327f5df1827Smickey }
328f5df1827Smickey }
329f5df1827Smickey *idt_slot = idtvec;
330f5df1827Smickey *index = slot;
331f5df1827Smickey *cip = ci;
332f5df1827Smickey return 0;
333f5df1827Smickey }
334f5df1827Smickey
335177417ccSderaadt /*
336177417ccSderaadt * True if the system has any non-level interrupts which are shared
337177417ccSderaadt * on the same pin.
338177417ccSderaadt */
339177417ccSderaadt int intr_shared_edge;
340177417ccSderaadt
341f5df1827Smickey void *
342f5df1827Smickey intr_establish(int legacy_irq, struct pic *pic, int pin, int type, int level,
34315db3095Sdlg struct cpu_info *ci, int (*handler)(void *), void *arg, const char *what)
344f5df1827Smickey {
345f5df1827Smickey struct intrhand **p, *q, *ih;
346f5df1827Smickey int slot, error, idt_vec;
347f5df1827Smickey struct intrsource *source;
348f5df1827Smickey struct intrstub *stubp;
349483e3bc5Skettenis int flags;
350f5df1827Smickey
351f5df1827Smickey #ifdef DIAGNOSTIC
352f5df1827Smickey if (legacy_irq != -1 && (legacy_irq < 0 || legacy_irq > 15))
353f5df1827Smickey panic("intr_establish: bad legacy IRQ value");
354f5df1827Smickey
355f5df1827Smickey if (legacy_irq == -1 && pic == &i8259_pic)
356f5df1827Smickey panic("intr_establish: non-legacy IRQ on i8259");
357f5df1827Smickey #endif
358f5df1827Smickey
359fc30b644Skettenis flags = level & (IPL_MPSAFE | IPL_WAKEUP);
360fc30b644Skettenis level &= ~(IPL_MPSAFE | IPL_WAKEUP);
361483e3bc5Skettenis
362a12998adSkettenis KASSERT(level <= IPL_TTY || level >= IPL_CLOCK || flags & IPL_MPSAFE);
363a12998adSkettenis
364f5df1827Smickey error = intr_allocate_slot(pic, legacy_irq, pin, level, &ci, &slot,
365f5df1827Smickey &idt_vec);
366f5df1827Smickey if (error != 0) {
367f5df1827Smickey printf("failed to allocate interrupt slot for PIC %s pin %d\n",
368f5df1827Smickey pic->pic_dev.dv_xname, pin);
369f5df1827Smickey return NULL;
370f5df1827Smickey }
371f5df1827Smickey
372f5df1827Smickey /* no point in sleeping unless someone can free memory. */
373f5df1827Smickey ih = malloc(sizeof *ih, M_DEVBUF, cold ? M_NOWAIT : M_WAITOK);
374f5df1827Smickey if (ih == NULL) {
375f5df1827Smickey printf("intr_establish: can't allocate handler info\n");
376f5df1827Smickey return NULL;
377f5df1827Smickey }
378f5df1827Smickey
379f5df1827Smickey source = ci->ci_isources[slot];
380f5df1827Smickey
381f5df1827Smickey if (source->is_handlers != NULL &&
382f5df1827Smickey source->is_pic->pic_type != pic->pic_type) {
383bae2bd50Sderaadt free(ih, M_DEVBUF, sizeof(*ih));
384f5df1827Smickey printf("intr_establish: can't share intr source between "
385f5df1827Smickey "different PIC types (legacy_irq %d pin %d slot %d)\n",
386f5df1827Smickey legacy_irq, pin, slot);
387f5df1827Smickey return NULL;
388f5df1827Smickey }
389f5df1827Smickey
390f5df1827Smickey source->is_pin = pin;
391f5df1827Smickey source->is_pic = pic;
392f5df1827Smickey
393f5df1827Smickey switch (source->is_type) {
394f5df1827Smickey case IST_NONE:
395f5df1827Smickey source->is_type = type;
396f5df1827Smickey break;
397f5df1827Smickey case IST_EDGE:
398177417ccSderaadt intr_shared_edge = 1;
399177417ccSderaadt /* FALLTHROUGH */
400f5df1827Smickey case IST_LEVEL:
401f5df1827Smickey if (source->is_type == type)
402f5df1827Smickey break;
403f5df1827Smickey case IST_PULSE:
404f5df1827Smickey if (type != IST_NONE) {
405f5df1827Smickey printf("intr_establish: pic %s pin %d: can't share "
406f5df1827Smickey "type %d with %d\n", pic->pic_name, pin,
407f5df1827Smickey source->is_type, type);
408bae2bd50Sderaadt free(ih, M_DEVBUF, sizeof(*ih));
409f5df1827Smickey return NULL;
410f5df1827Smickey }
411f5df1827Smickey break;
412f5df1827Smickey default:
41362a36cb0Sfgsch panic("intr_establish: bad intr type %d for pic %s pin %d",
414f5df1827Smickey source->is_type, pic->pic_dev.dv_xname, pin);
415f5df1827Smickey }
416f5df1827Smickey
417f5df1827Smickey if (!cold)
418f5df1827Smickey pic->pic_hwmask(pic, pin);
419f5df1827Smickey
420f5df1827Smickey /*
421f5df1827Smickey * Figure out where to put the handler.
422f5df1827Smickey * This is O(N^2), but we want to preserve the order, and N is
423f5df1827Smickey * generally small.
424f5df1827Smickey */
425f5df1827Smickey for (p = &ci->ci_isources[slot]->is_handlers;
426f5df1827Smickey (q = *p) != NULL && q->ih_level > level;
427f5df1827Smickey p = &q->ih_next)
428f5df1827Smickey ;
429f5df1827Smickey
430f5df1827Smickey ih->ih_fun = handler;
431f5df1827Smickey ih->ih_arg = arg;
432f5df1827Smickey ih->ih_next = *p;
433f5df1827Smickey ih->ih_level = level;
434483e3bc5Skettenis ih->ih_flags = flags;
435f5df1827Smickey ih->ih_pin = pin;
436f5df1827Smickey ih->ih_cpu = ci;
437f5df1827Smickey ih->ih_slot = slot;
4384667bebbSmatthew evcount_attach(&ih->ih_count, what, &source->is_idtvec);
4396816c79aSderaadt
440f5df1827Smickey *p = ih;
441f5df1827Smickey
442f5df1827Smickey intr_calculatemasks(ci);
443f5df1827Smickey
444f5df1827Smickey if (ci->ci_isources[slot]->is_resume == NULL ||
445f5df1827Smickey source->is_idtvec != idt_vec) {
446f5df1827Smickey if (source->is_idtvec != 0 && source->is_idtvec != idt_vec)
447f5df1827Smickey idt_vec_free(source->is_idtvec);
448f5df1827Smickey source->is_idtvec = idt_vec;
449f5df1827Smickey stubp = type == IST_LEVEL ?
450f5df1827Smickey &pic->pic_level_stubs[slot] : &pic->pic_edge_stubs[slot];
451f5df1827Smickey ci->ci_isources[slot]->is_resume = stubp->ist_resume;
452f5df1827Smickey ci->ci_isources[slot]->is_recurse = stubp->ist_recurse;
453f5df1827Smickey setgate(&idt[idt_vec], stubp->ist_entry, 0, SDT_SYS386IGT,
454f5df1827Smickey SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
455f5df1827Smickey }
456f5df1827Smickey
457f5df1827Smickey pic->pic_addroute(pic, ci, pin, idt_vec, type);
458f5df1827Smickey
459f5df1827Smickey if (!cold)
460f5df1827Smickey pic->pic_hwunmask(pic, pin);
461f5df1827Smickey
462f5df1827Smickey #ifdef INTRDEBUG
463f5df1827Smickey printf("allocated pic %s type %s pin %d level %d to cpu%u slot %d idt entry %d\n",
464f5df1827Smickey pic->pic_name, type == IST_EDGE ? "edge" : "level", pin, level,
465f5df1827Smickey ci->ci_apicid, slot, idt_vec);
466f5df1827Smickey #endif
467f5df1827Smickey
468f5df1827Smickey return (ih);
469f5df1827Smickey }
470f5df1827Smickey
471f5df1827Smickey /*
472f5df1827Smickey * Deregister an interrupt handler.
473f5df1827Smickey */
474f5df1827Smickey void
475f5df1827Smickey intr_disestablish(struct intrhand *ih)
476f5df1827Smickey {
477f5df1827Smickey struct intrhand **p, *q;
478f5df1827Smickey struct cpu_info *ci;
479f5df1827Smickey struct pic *pic;
480f5df1827Smickey struct intrsource *source;
481f5df1827Smickey int idtvec;
482f5df1827Smickey
483f5df1827Smickey ci = ih->ih_cpu;
484f5df1827Smickey pic = ci->ci_isources[ih->ih_slot]->is_pic;
485f5df1827Smickey source = ci->ci_isources[ih->ih_slot];
486f5df1827Smickey idtvec = source->is_idtvec;
487f5df1827Smickey
488f5df1827Smickey pic->pic_hwmask(pic, ih->ih_pin);
489092bf81aSkettenis x86_atomic_clearbits_u64(&ci->ci_ipending, (1UL << ih->ih_slot));
490f5df1827Smickey
491f5df1827Smickey /*
492f5df1827Smickey * Remove the handler from the chain.
493f5df1827Smickey */
494f5df1827Smickey for (p = &source->is_handlers; (q = *p) != NULL && q != ih;
495f5df1827Smickey p = &q->ih_next)
496f5df1827Smickey ;
497f5df1827Smickey if (q == NULL) {
498f5df1827Smickey panic("intr_disestablish: handler not registered");
499f5df1827Smickey }
500f5df1827Smickey
501f5df1827Smickey *p = q->ih_next;
502f5df1827Smickey
503f5df1827Smickey intr_calculatemasks(ci);
504590907c7Skettenis if (source->is_handlers == NULL)
505f5df1827Smickey pic->pic_delroute(pic, ci, ih->ih_pin, idtvec, source->is_type);
506590907c7Skettenis else
507f5df1827Smickey pic->pic_hwunmask(pic, ih->ih_pin);
508f5df1827Smickey
509f5df1827Smickey #ifdef INTRDEBUG
510f5df1827Smickey printf("cpu%u: remove slot %d (pic %s pin %d vec %d)\n",
511f5df1827Smickey ci->ci_apicid, ih->ih_slot, pic->pic_dev.dv_xname, ih->ih_pin,
512f5df1827Smickey idtvec);
513f5df1827Smickey #endif
514f5df1827Smickey
515f5df1827Smickey if (source->is_handlers == NULL) {
516cf647cc3Stedu free(source, M_DEVBUF, sizeof (struct intrsource));
517f5df1827Smickey ci->ci_isources[ih->ih_slot] = NULL;
518f5df1827Smickey if (pic != &i8259_pic)
519f5df1827Smickey idt_vec_free(idtvec);
520f5df1827Smickey }
521f5df1827Smickey
5226816c79aSderaadt evcount_detach(&ih->ih_count);
523bae2bd50Sderaadt free(ih, M_DEVBUF, sizeof(*ih));
524f5df1827Smickey }
525f5df1827Smickey
52658643530Sratchov int
52758643530Sratchov intr_handler(struct intrframe *frame, struct intrhand *ih)
52858643530Sratchov {
5299c9e7f95Sdlg struct cpu_info *ci = curcpu();
5309c9e7f95Sdlg int floor;
53158643530Sratchov int rc;
53258643530Sratchov #ifdef MULTIPROCESSOR
53358643530Sratchov int need_lock;
534ba5ad174Skettenis #endif
53558643530Sratchov
536ba5ad174Skettenis /*
537ba5ad174Skettenis * We may not be able to mask MSIs, so block non-wakeup
538ba5ad174Skettenis * interrupts while we're suspended.
539ba5ad174Skettenis */
540*1e607cd8Skettenis if (intr_suspended && (ih->ih_flags & IPL_WAKEUP) == 0)
541ba5ad174Skettenis return 0;
542ba5ad174Skettenis
543ba5ad174Skettenis #ifdef MULTIPROCESSOR
544483e3bc5Skettenis if (ih->ih_flags & IPL_MPSAFE)
545483e3bc5Skettenis need_lock = 0;
546483e3bc5Skettenis else
5477bf9d514Smpi need_lock = 1;
54858643530Sratchov
54958643530Sratchov if (need_lock)
55058643530Sratchov __mp_lock(&kernel_lock);
55158643530Sratchov #endif
5529c9e7f95Sdlg floor = ci->ci_handled_intr_level;
5539c9e7f95Sdlg ci->ci_handled_intr_level = ih->ih_level;
55458643530Sratchov rc = (*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : frame);
5559c9e7f95Sdlg ci->ci_handled_intr_level = floor;
55658643530Sratchov #ifdef MULTIPROCESSOR
55758643530Sratchov if (need_lock)
55858643530Sratchov __mp_unlock(&kernel_lock);
55958643530Sratchov #endif
56058643530Sratchov return rc;
56158643530Sratchov }
56258643530Sratchov
563f5df1827Smickey /*
564f5df1827Smickey * Fake interrupt handler structures for the benefit of symmetry with
565f5df1827Smickey * other interrupt sources, and the benefit of intr_calculatemasks()
566f5df1827Smickey */
567f5df1827Smickey struct intrhand fake_softclock_intrhand;
568f5df1827Smickey struct intrhand fake_softnet_intrhand;
569a2cd3cc4Skettenis struct intrhand fake_softtty_intrhand;
570f5df1827Smickey struct intrhand fake_timer_intrhand;
571f5df1827Smickey struct intrhand fake_ipi_intrhand;
572ad5f3503Smikeb #if NXEN > 0
573ad5f3503Smikeb struct intrhand fake_xen_intrhand;
574ad5f3503Smikeb #endif
575218ead0bSmikeb #if NHYPERV > 0
576218ead0bSmikeb struct intrhand fake_hyperv_intrhand;
577218ead0bSmikeb #endif
578f5df1827Smickey
579f5df1827Smickey /*
580f5df1827Smickey * Initialize all handlers that aren't dynamically allocated, and exist
581f5df1827Smickey * for each CPU.
582f5df1827Smickey */
583f5df1827Smickey void
584f5df1827Smickey cpu_intr_init(struct cpu_info *ci)
585f5df1827Smickey {
586f5df1827Smickey struct intrsource *isp;
587b5b9857bSart #if NLAPIC > 0 && defined(MULTIPROCESSOR) && 0
588f5df1827Smickey int i;
589f5df1827Smickey #endif
590f5df1827Smickey
5913cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
592f5df1827Smickey if (isp == NULL)
593f5df1827Smickey panic("can't allocate fixed interrupt source");
594f5df1827Smickey isp->is_recurse = Xsoftclock;
595f5df1827Smickey isp->is_resume = Xsoftclock;
596f5df1827Smickey fake_softclock_intrhand.ih_level = IPL_SOFTCLOCK;
597f5df1827Smickey isp->is_handlers = &fake_softclock_intrhand;
598f5df1827Smickey isp->is_pic = &softintr_pic;
599f5df1827Smickey ci->ci_isources[SIR_CLOCK] = isp;
6003cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
601f5df1827Smickey if (isp == NULL)
602f5df1827Smickey panic("can't allocate fixed interrupt source");
603f5df1827Smickey isp->is_recurse = Xsoftnet;
604f5df1827Smickey isp->is_resume = Xsoftnet;
605f5df1827Smickey fake_softnet_intrhand.ih_level = IPL_SOFTNET;
606f5df1827Smickey isp->is_handlers = &fake_softnet_intrhand;
607f5df1827Smickey isp->is_pic = &softintr_pic;
608f5df1827Smickey ci->ci_isources[SIR_NET] = isp;
6093cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
610f5df1827Smickey if (isp == NULL)
611f5df1827Smickey panic("can't allocate fixed interrupt source");
612a2cd3cc4Skettenis isp->is_recurse = Xsofttty;
613a2cd3cc4Skettenis isp->is_resume = Xsofttty;
614a2cd3cc4Skettenis fake_softtty_intrhand.ih_level = IPL_SOFTTTY;
615a2cd3cc4Skettenis isp->is_handlers = &fake_softtty_intrhand;
616f5df1827Smickey isp->is_pic = &softintr_pic;
617a2cd3cc4Skettenis ci->ci_isources[SIR_TTY] = isp;
618f5df1827Smickey #if NLAPIC > 0
6193cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
620f5df1827Smickey if (isp == NULL)
621f5df1827Smickey panic("can't allocate fixed interrupt source");
622f5df1827Smickey isp->is_recurse = Xrecurse_lapic_ltimer;
623f5df1827Smickey isp->is_resume = Xresume_lapic_ltimer;
624f5df1827Smickey fake_timer_intrhand.ih_level = IPL_CLOCK;
625f5df1827Smickey isp->is_handlers = &fake_timer_intrhand;
626f5df1827Smickey isp->is_pic = &local_pic;
627f5df1827Smickey ci->ci_isources[LIR_TIMER] = isp;
628f5df1827Smickey #ifdef MULTIPROCESSOR
6293cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
630f5df1827Smickey if (isp == NULL)
631f5df1827Smickey panic("can't allocate fixed interrupt source");
632f5df1827Smickey isp->is_recurse = Xrecurse_lapic_ipi;
633f5df1827Smickey isp->is_resume = Xresume_lapic_ipi;
634f5df1827Smickey fake_ipi_intrhand.ih_level = IPL_IPI;
635f5df1827Smickey isp->is_handlers = &fake_ipi_intrhand;
636f5df1827Smickey isp->is_pic = &local_pic;
637f5df1827Smickey ci->ci_isources[LIR_IPI] = isp;
638f5df1827Smickey #endif
639ad5f3503Smikeb #if NXEN > 0
640ad5f3503Smikeb isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
641ad5f3503Smikeb if (isp == NULL)
642ad5f3503Smikeb panic("can't allocate fixed interrupt source");
643ad5f3503Smikeb isp->is_recurse = Xrecurse_xen_upcall;
644ad5f3503Smikeb isp->is_resume = Xresume_xen_upcall;
645ad5f3503Smikeb fake_xen_intrhand.ih_level = IPL_NET;
646ad5f3503Smikeb isp->is_handlers = &fake_xen_intrhand;
647ad5f3503Smikeb isp->is_pic = &local_pic;
648ad5f3503Smikeb ci->ci_isources[LIR_XEN] = isp;
649b5b9857bSart #endif
650218ead0bSmikeb #if NHYPERV > 0
651218ead0bSmikeb isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
652218ead0bSmikeb if (isp == NULL)
653218ead0bSmikeb panic("can't allocate fixed interrupt source");
654218ead0bSmikeb isp->is_recurse = Xrecurse_hyperv_upcall;
655218ead0bSmikeb isp->is_resume = Xresume_hyperv_upcall;
656218ead0bSmikeb fake_hyperv_intrhand.ih_level = IPL_NET;
657218ead0bSmikeb isp->is_handlers = &fake_hyperv_intrhand;
658218ead0bSmikeb isp->is_pic = &local_pic;
659218ead0bSmikeb ci->ci_isources[LIR_HYPERV] = isp;
660218ead0bSmikeb #endif
661ad5f3503Smikeb #endif /* NLAPIC */
662f5df1827Smickey
663f5df1827Smickey intr_calculatemasks(ci);
664f5df1827Smickey
665f5df1827Smickey }
666f5df1827Smickey
667f5df1827Smickey void
668f5df1827Smickey intr_printconfig(void)
669f5df1827Smickey {
670f5df1827Smickey #ifdef INTRDEBUG
671f5df1827Smickey int i;
672f5df1827Smickey struct intrhand *ih;
673f5df1827Smickey struct intrsource *isp;
674f5df1827Smickey struct cpu_info *ci;
675f5df1827Smickey CPU_INFO_ITERATOR cii;
676f5df1827Smickey
6775d5ec4d4Smiod CPU_INFO_FOREACH(cii, ci) {
678f5df1827Smickey printf("cpu%d: interrupt masks:\n", ci->ci_apicid);
679f5df1827Smickey for (i = 0; i < NIPL; i++)
680f5df1827Smickey printf("IPL %d mask %lx unmask %lx\n", i,
681f5df1827Smickey (u_long)ci->ci_imask[i], (u_long)ci->ci_iunmask[i]);
682f5df1827Smickey for (i = 0; i < MAX_INTR_SOURCES; i++) {
683f5df1827Smickey isp = ci->ci_isources[i];
684f5df1827Smickey if (isp == NULL)
685f5df1827Smickey continue;
686f5df1827Smickey printf("cpu%u source %d is pin %d from pic %s maxlevel %d\n",
687f5df1827Smickey ci->ci_apicid, i, isp->is_pin,
688f5df1827Smickey isp->is_pic->pic_name, isp->is_maxlevel);
689f5df1827Smickey for (ih = isp->is_handlers; ih != NULL;
690f5df1827Smickey ih = ih->ih_next)
691f5df1827Smickey printf("\thandler %p level %d\n",
692f5df1827Smickey ih->ih_fun, ih->ih_level);
693f5df1827Smickey
694f5df1827Smickey }
695f5df1827Smickey }
696f5df1827Smickey #endif
697f5df1827Smickey }
6980b57f6e9Sart
699edc47eabSkettenis void
70025a8a282Sdlg intr_barrier(void *cookie)
701edc47eabSkettenis {
70225a8a282Sdlg struct intrhand *ih = cookie;
70325a8a282Sdlg sched_barrier(ih->ih_cpu);
704edc47eabSkettenis }
705edc47eabSkettenis
706fc30b644Skettenis #ifdef SUSPEND
707fc30b644Skettenis
708fc30b644Skettenis void
709fc30b644Skettenis intr_enable_wakeup(void)
710fc30b644Skettenis {
711fc30b644Skettenis struct cpu_info *ci = curcpu();
712fc30b644Skettenis struct pic *pic;
713fc30b644Skettenis int irq, pin;
714fc30b644Skettenis
715fc30b644Skettenis for (irq = 0; irq < MAX_INTR_SOURCES; irq++) {
716fc30b644Skettenis if (ci->ci_isources[irq] == NULL)
717fc30b644Skettenis continue;
718fc30b644Skettenis
719fc30b644Skettenis if (ci->ci_isources[irq]->is_handlers->ih_flags & IPL_WAKEUP)
720fc30b644Skettenis continue;
721fc30b644Skettenis
722fc30b644Skettenis pic = ci->ci_isources[irq]->is_pic;
723fc30b644Skettenis pin = ci->ci_isources[irq]->is_pin;
724fc30b644Skettenis if (pic->pic_hwmask)
725fc30b644Skettenis pic->pic_hwmask(pic, pin);
726fc30b644Skettenis }
727*1e607cd8Skettenis
728*1e607cd8Skettenis intr_suspended = 1;
729fc30b644Skettenis }
730fc30b644Skettenis
731fc30b644Skettenis void
732fc30b644Skettenis intr_disable_wakeup(void)
733fc30b644Skettenis {
734fc30b644Skettenis struct cpu_info *ci = curcpu();
735fc30b644Skettenis struct pic *pic;
736fc30b644Skettenis int irq, pin;
737fc30b644Skettenis
738*1e607cd8Skettenis intr_suspended = 0;
739*1e607cd8Skettenis
740fc30b644Skettenis for (irq = 0; irq < MAX_INTR_SOURCES; irq++) {
741fc30b644Skettenis if (ci->ci_isources[irq] == NULL)
742fc30b644Skettenis continue;
743fc30b644Skettenis
744fc30b644Skettenis if (ci->ci_isources[irq]->is_handlers->ih_flags & IPL_WAKEUP)
745fc30b644Skettenis continue;
746fc30b644Skettenis
747fc30b644Skettenis pic = ci->ci_isources[irq]->is_pic;
748fc30b644Skettenis pin = ci->ci_isources[irq]->is_pin;
749fc30b644Skettenis if (pic->pic_hwunmask)
750fc30b644Skettenis pic->pic_hwunmask(pic, pin);
751fc30b644Skettenis }
752fc30b644Skettenis }
753fc30b644Skettenis
754fc30b644Skettenis #endif
755fc30b644Skettenis
7560b57f6e9Sart /*
7570b57f6e9Sart * Add a mask to cpl, and return the old value of cpl.
7580b57f6e9Sart */
7590b57f6e9Sart int
7600b57f6e9Sart splraise(int nlevel)
7610b57f6e9Sart {
7620b57f6e9Sart int olevel;
7630b57f6e9Sart struct cpu_info *ci = curcpu();
7640b57f6e9Sart
7650f37d696Smpi KASSERT(nlevel >= IPL_NONE);
7660f37d696Smpi
7670b57f6e9Sart olevel = ci->ci_ilevel;
768b8befa46Sart ci->ci_ilevel = MAX(ci->ci_ilevel, nlevel);
7690b57f6e9Sart return (olevel);
7700b57f6e9Sart }
7710b57f6e9Sart
7720b57f6e9Sart /*
7730b57f6e9Sart * Restore a value to cpl (unmasking interrupts). If any unmasked
7740b57f6e9Sart * interrupts are pending, call Xspllower() to process them.
7750b57f6e9Sart */
7760b57f6e9Sart int
7770b57f6e9Sart spllower(int nlevel)
7780b57f6e9Sart {
7790b57f6e9Sart int olevel;
7800b57f6e9Sart struct cpu_info *ci = curcpu();
781092bf81aSkettenis u_int64_t imask;
78213ffdfaaSvisa u_long flags;
7830b57f6e9Sart
784a19c7295Sart imask = IUNMASK(ci, nlevel);
7850b57f6e9Sart olevel = ci->ci_ilevel;
786a19c7295Sart
78713ffdfaaSvisa flags = intr_disable();
788a19c7295Sart
789a19c7295Sart if (ci->ci_ipending & imask) {
7900b57f6e9Sart Xspllower(nlevel);
791a19c7295Sart } else {
7920b57f6e9Sart ci->ci_ilevel = nlevel;
79313ffdfaaSvisa intr_restore(flags);
794a19c7295Sart }
7950b57f6e9Sart return (olevel);
7960b57f6e9Sart }
7970b57f6e9Sart
7980b57f6e9Sart /*
7990b57f6e9Sart * Software interrupt registration
8000b57f6e9Sart *
8010b57f6e9Sart * We hand-code this to ensure that it's atomic.
8020b57f6e9Sart *
8030b57f6e9Sart * XXX always scheduled on the current CPU.
8040b57f6e9Sart */
8050b57f6e9Sart void
8060b57f6e9Sart softintr(int sir)
8070b57f6e9Sart {
8080b57f6e9Sart struct cpu_info *ci = curcpu();
8090b57f6e9Sart
8102df76cc2Sguenther __asm volatile("lock; orq %1, %0" :
811092bf81aSkettenis "=m"(ci->ci_ipending) : "ir" (1UL << sir));
8120b57f6e9Sart }
813