1*25a8a282Sdlg /* $OpenBSD: intr.c,v 1.53 2020/06/16 23:35:10 dlg Exp $ */ 2f5df1827Smickey /* $NetBSD: intr.c,v 1.3 2003/03/03 22:16:20 fvdl Exp $ */ 3f5df1827Smickey 4f5df1827Smickey /* 5f5df1827Smickey * Copyright 2002 (c) Wasabi Systems, Inc. 6f5df1827Smickey * All rights reserved. 7f5df1827Smickey * 8f5df1827Smickey * Written by Frank van der Linden for Wasabi Systems, Inc. 9f5df1827Smickey * 10f5df1827Smickey * Redistribution and use in source and binary forms, with or without 11f5df1827Smickey * modification, are permitted provided that the following conditions 12f5df1827Smickey * are met: 13f5df1827Smickey * 1. Redistributions of source code must retain the above copyright 14f5df1827Smickey * notice, this list of conditions and the following disclaimer. 15f5df1827Smickey * 2. Redistributions in binary form must reproduce the above copyright 16f5df1827Smickey * notice, this list of conditions and the following disclaimer in the 17f5df1827Smickey * documentation and/or other materials provided with the distribution. 18f5df1827Smickey * 3. All advertising materials mentioning features or use of this software 19f5df1827Smickey * must display the following acknowledgement: 20f5df1827Smickey * This product includes software developed for the NetBSD Project by 21f5df1827Smickey * Wasabi Systems, Inc. 22f5df1827Smickey * 4. The name of Wasabi Systems, Inc. may not be used to endorse 23f5df1827Smickey * or promote products derived from this software without specific prior 24f5df1827Smickey * written permission. 25f5df1827Smickey * 26f5df1827Smickey * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 27f5df1827Smickey * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28f5df1827Smickey * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29f5df1827Smickey * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 30f5df1827Smickey * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31f5df1827Smickey * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32f5df1827Smickey * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33f5df1827Smickey * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34f5df1827Smickey * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35f5df1827Smickey * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36f5df1827Smickey * POSSIBILITY OF SUCH DAMAGE. 37f5df1827Smickey */ 38f5df1827Smickey 39f5df1827Smickey /* #define INTRDEBUG */ 40f5df1827Smickey 41f5df1827Smickey #include <sys/param.h> 42f5df1827Smickey #include <sys/systm.h> 43f5df1827Smickey #include <sys/kernel.h> 44f5df1827Smickey #include <sys/syslog.h> 45f5df1827Smickey #include <sys/device.h> 46f5df1827Smickey #include <sys/malloc.h> 47f5df1827Smickey #include <sys/errno.h> 48f5df1827Smickey 49f5df1827Smickey #include <machine/atomic.h> 50f5df1827Smickey #include <machine/i8259.h> 51f5df1827Smickey #include <machine/cpu.h> 52f5df1827Smickey #include <machine/pio.h> 53a19c7295Sart #include <machine/cpufunc.h> 54f5df1827Smickey 55f5df1827Smickey #include "lapic.h" 56ad5f3503Smikeb #include "xen.h" 57218ead0bSmikeb #include "hyperv.h" 58f5df1827Smickey 59f5df1827Smickey #if NLAPIC > 0 60f5df1827Smickey #include <machine/i82489var.h> 61f5df1827Smickey #endif 62f5df1827Smickey 63f5df1827Smickey struct pic softintr_pic = { 643534b73fSjsg {0, {NULL}, NULL, 0, "softintr_pic0", NULL, 0, 0}, 65f5df1827Smickey PIC_SOFT, 66f5df1827Smickey #ifdef MULTIPROCESSOR 67b5b9857bSart {}, 68f5df1827Smickey #endif 69f5df1827Smickey NULL, 70f5df1827Smickey NULL, 71f5df1827Smickey NULL, 72f5df1827Smickey NULL, 73f5df1827Smickey NULL, 74f5df1827Smickey }; 75f5df1827Smickey 76f5df1827Smickey /* 77f5df1827Smickey * Fill in default interrupt table (in case of spurious interrupt 78f5df1827Smickey * during configuration of kernel), setup interrupt control unit 79f5df1827Smickey */ 80f5df1827Smickey void 81f5df1827Smickey intr_default_setup(void) 82f5df1827Smickey { 83f5df1827Smickey int i; 84f5df1827Smickey 85f5df1827Smickey /* icu vectors */ 86f5df1827Smickey for (i = 0; i < NUM_LEGACY_IRQS; i++) { 87f5df1827Smickey idt_allocmap[ICU_OFFSET + i] = 1; 88f5df1827Smickey setgate(&idt[ICU_OFFSET + i], 89f5df1827Smickey i8259_stubs[i].ist_entry, 0, SDT_SYS386IGT, 90f5df1827Smickey SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 91f5df1827Smickey } 92f5df1827Smickey 93f5df1827Smickey /* 94f5df1827Smickey * Eventually might want to check if it's actually there. 95f5df1827Smickey */ 96f5df1827Smickey i8259_default_setup(); 97f5df1827Smickey } 98f5df1827Smickey 99f5df1827Smickey /* 100f5df1827Smickey * Handle a NMI, possibly a machine check. 101f5df1827Smickey * return true to panic system, false to ignore. 102f5df1827Smickey */ 103f5df1827Smickey int 104f5df1827Smickey x86_nmi(void) 105f5df1827Smickey { 106f5df1827Smickey log(LOG_CRIT, "NMI port 61 %x, port 70 %x\n", inb(0x61), inb(0x70)); 107f5df1827Smickey return(0); 108f5df1827Smickey } 109f5df1827Smickey 110f5df1827Smickey /* 111f5df1827Smickey * Recalculate the interrupt masks from scratch. 112f5df1827Smickey */ 113f5df1827Smickey void 114f5df1827Smickey intr_calculatemasks(struct cpu_info *ci) 115f5df1827Smickey { 116092bf81aSkettenis int irq, level; 117092bf81aSkettenis u_int64_t unusedirqs, intrlevel[MAX_INTR_SOURCES]; 118f5df1827Smickey struct intrhand *q; 119f5df1827Smickey 120f5df1827Smickey /* First, figure out which levels each IRQ uses. */ 121092bf81aSkettenis unusedirqs = 0xffffffffffffffffUL; 122f5df1827Smickey for (irq = 0; irq < MAX_INTR_SOURCES; irq++) { 123f5df1827Smickey int levels = 0; 124f5df1827Smickey 125f5df1827Smickey if (ci->ci_isources[irq] == NULL) { 126f5df1827Smickey intrlevel[irq] = 0; 127f5df1827Smickey continue; 128f5df1827Smickey } 129f5df1827Smickey for (q = ci->ci_isources[irq]->is_handlers; q; q = q->ih_next) 130092bf81aSkettenis levels |= (1 << q->ih_level); 131f5df1827Smickey intrlevel[irq] = levels; 132f5df1827Smickey if (levels) 133092bf81aSkettenis unusedirqs &= ~(1UL << irq); 134f5df1827Smickey } 135f5df1827Smickey 136f5df1827Smickey /* Then figure out which IRQs use each level. */ 137f5df1827Smickey for (level = 0; level < NIPL; level++) { 138092bf81aSkettenis u_int64_t irqs = 0; 139f5df1827Smickey for (irq = 0; irq < MAX_INTR_SOURCES; irq++) 140f5df1827Smickey if (intrlevel[irq] & (1 << level)) 141092bf81aSkettenis irqs |= (1UL << irq); 142f5df1827Smickey ci->ci_imask[level] = irqs | unusedirqs; 143f5df1827Smickey } 144f5df1827Smickey 145f5df1827Smickey for (level = 0; level< (NIPL - 1); level++) 146f5df1827Smickey ci->ci_imask[level + 1] |= ci->ci_imask[level]; 147f5df1827Smickey 148f5df1827Smickey for (irq = 0; irq < MAX_INTR_SOURCES; irq++) { 149f5df1827Smickey int maxlevel = IPL_NONE; 150f5df1827Smickey int minlevel = IPL_HIGH; 151f5df1827Smickey 152f5df1827Smickey if (ci->ci_isources[irq] == NULL) 153f5df1827Smickey continue; 154f5df1827Smickey for (q = ci->ci_isources[irq]->is_handlers; q; 155f5df1827Smickey q = q->ih_next) { 156f5df1827Smickey if (q->ih_level < minlevel) 157f5df1827Smickey minlevel = q->ih_level; 158f5df1827Smickey if (q->ih_level > maxlevel) 159f5df1827Smickey maxlevel = q->ih_level; 160f5df1827Smickey } 161f5df1827Smickey ci->ci_isources[irq]->is_maxlevel = maxlevel; 162f5df1827Smickey ci->ci_isources[irq]->is_minlevel = minlevel; 163f5df1827Smickey } 164f5df1827Smickey 165f5df1827Smickey for (level = 0; level < NIPL; level++) 166f5df1827Smickey ci->ci_iunmask[level] = ~ci->ci_imask[level]; 167f5df1827Smickey } 168f5df1827Smickey 169f5df1827Smickey int 170f5df1827Smickey intr_allocate_slot_cpu(struct cpu_info *ci, struct pic *pic, int pin, 171f5df1827Smickey int *index) 172f5df1827Smickey { 173f5df1827Smickey int start, slot, i; 174f5df1827Smickey struct intrsource *isp; 175f5df1827Smickey 176f5df1827Smickey start = CPU_IS_PRIMARY(ci) ? NUM_LEGACY_IRQS : 0; 177f5df1827Smickey slot = -1; 178f5df1827Smickey 17964da5045Skettenis for (i = 0; i < start; i++) { 18064da5045Skettenis isp = ci->ci_isources[i]; 18164da5045Skettenis if (isp != NULL && isp->is_pic == pic && isp->is_pin == pin) { 18264da5045Skettenis slot = i; 18364da5045Skettenis start = MAX_INTR_SOURCES; 18464da5045Skettenis break; 18564da5045Skettenis } 18664da5045Skettenis } 187f5df1827Smickey for (i = start; i < MAX_INTR_SOURCES ; i++) { 188f5df1827Smickey isp = ci->ci_isources[i]; 189f5df1827Smickey if (isp != NULL && isp->is_pic == pic && isp->is_pin == pin) { 190f5df1827Smickey slot = i; 191f5df1827Smickey break; 192f5df1827Smickey } 193f5df1827Smickey if (isp == NULL && slot == -1) { 194f5df1827Smickey slot = i; 195f5df1827Smickey continue; 196f5df1827Smickey } 197f5df1827Smickey } 198f5df1827Smickey if (slot == -1) { 199f5df1827Smickey return EBUSY; 200f5df1827Smickey } 201f5df1827Smickey 202f5df1827Smickey isp = ci->ci_isources[slot]; 203f5df1827Smickey if (isp == NULL) { 20451394e79Schl isp = malloc(sizeof (struct intrsource), M_DEVBUF, 20551394e79Schl M_NOWAIT|M_ZERO); 206f5df1827Smickey if (isp == NULL) { 207f5df1827Smickey return ENOMEM; 208f5df1827Smickey } 209f5df1827Smickey snprintf(isp->is_evname, sizeof (isp->is_evname), 210f5df1827Smickey "pin %d", pin); 211f5df1827Smickey ci->ci_isources[slot] = isp; 212f5df1827Smickey } 213f5df1827Smickey 214f5df1827Smickey *index = slot; 215f5df1827Smickey return 0; 216f5df1827Smickey } 217f5df1827Smickey 218f5df1827Smickey /* 219f5df1827Smickey * A simple round-robin allocator to assign interrupts to CPUs. 220f5df1827Smickey */ 221f5df1827Smickey int 222f5df1827Smickey intr_allocate_slot(struct pic *pic, int legacy_irq, int pin, int level, 223f5df1827Smickey struct cpu_info **cip, int *index, int *idt_slot) 224f5df1827Smickey { 225f5df1827Smickey CPU_INFO_ITERATOR cii; 226f5df1827Smickey struct cpu_info *ci; 227f5df1827Smickey struct intrsource *isp; 228f5df1827Smickey int slot, idtvec, error; 229f5df1827Smickey 230f5df1827Smickey /* 231f5df1827Smickey * If a legacy IRQ is wanted, try to use a fixed slot pointing 232f5df1827Smickey * at the primary CPU. In the case of IO APICs, multiple pins 233f5df1827Smickey * may map to one legacy IRQ, but they should not be shared 234f5df1827Smickey * in that case, so the first one gets the legacy slot, but 235f5df1827Smickey * a subsequent allocation with a different pin will get 236f5df1827Smickey * a different slot. 237f5df1827Smickey */ 238f5df1827Smickey if (legacy_irq != -1) { 239f5df1827Smickey ci = &cpu_info_primary; 240898c0ff1Sbrad /* must check for duplicate pic + pin first */ 241898c0ff1Sbrad for (slot = 0 ; slot < MAX_INTR_SOURCES ; slot++) { 242898c0ff1Sbrad isp = ci->ci_isources[slot]; 243898c0ff1Sbrad if (isp != NULL && isp->is_pic == pic && 244898c0ff1Sbrad isp->is_pin == pin ) { 245898c0ff1Sbrad goto duplicate; 246898c0ff1Sbrad } 247898c0ff1Sbrad } 248f5df1827Smickey slot = legacy_irq; 249f5df1827Smickey isp = ci->ci_isources[slot]; 250f5df1827Smickey if (isp == NULL) { 25151394e79Schl isp = malloc(sizeof (struct intrsource), M_DEVBUF, 25251394e79Schl M_NOWAIT|M_ZERO); 253f5df1827Smickey if (isp == NULL) 254f5df1827Smickey return ENOMEM; 255f5df1827Smickey snprintf(isp->is_evname, sizeof (isp->is_evname), 256f5df1827Smickey "pin %d", pin); 257f5df1827Smickey 258f5df1827Smickey ci->ci_isources[slot] = isp; 259f5df1827Smickey } else { 26026f38a42Sniklas if (isp->is_pic != pic || isp->is_pin != pin) { 261f5df1827Smickey if (pic == &i8259_pic) 262f5df1827Smickey return EINVAL; 263f5df1827Smickey goto other; 264f5df1827Smickey } 265f5df1827Smickey } 266898c0ff1Sbrad duplicate: 267f5df1827Smickey if (pic == &i8259_pic) 268f5df1827Smickey idtvec = ICU_OFFSET + legacy_irq; 269f5df1827Smickey else { 270f5df1827Smickey #ifdef IOAPIC_HWMASK 271f5df1827Smickey if (level > isp->is_maxlevel) { 272f5df1827Smickey #else 273f5df1827Smickey if (isp->is_minlevel == 0 || level < isp->is_minlevel) { 274f5df1827Smickey #endif 275f5df1827Smickey idtvec = idt_vec_alloc(APIC_LEVEL(level), 276f5df1827Smickey IDT_INTR_HIGH); 277f5df1827Smickey if (idtvec == 0) 278f5df1827Smickey return EBUSY; 279f5df1827Smickey } else 280f5df1827Smickey idtvec = isp->is_idtvec; 281f5df1827Smickey } 282f5df1827Smickey } else { 283f5df1827Smickey other: 284f5df1827Smickey /* 285f5df1827Smickey * Otherwise, look for a free slot elsewhere. Do the primary 286f5df1827Smickey * CPU first. 287f5df1827Smickey */ 288f5df1827Smickey ci = &cpu_info_primary; 289f5df1827Smickey error = intr_allocate_slot_cpu(ci, pic, pin, &slot); 290f5df1827Smickey if (error == 0) 291f5df1827Smickey goto found; 292f5df1827Smickey 293f5df1827Smickey /* 294f5df1827Smickey * ..now try the others. 295f5df1827Smickey */ 2965d5ec4d4Smiod CPU_INFO_FOREACH(cii, ci) { 297f5df1827Smickey if (CPU_IS_PRIMARY(ci)) 298f5df1827Smickey continue; 299f5df1827Smickey error = intr_allocate_slot_cpu(ci, pic, pin, &slot); 300f5df1827Smickey if (error == 0) 301f5df1827Smickey goto found; 302f5df1827Smickey } 303f5df1827Smickey return EBUSY; 304f5df1827Smickey found: 305f5df1827Smickey idtvec = idt_vec_alloc(APIC_LEVEL(level), IDT_INTR_HIGH); 306f5df1827Smickey if (idtvec == 0) { 307cf647cc3Stedu free(ci->ci_isources[slot], M_DEVBUF, sizeof (struct intrsource)); 308f5df1827Smickey ci->ci_isources[slot] = NULL; 309f5df1827Smickey return EBUSY; 310f5df1827Smickey } 311f5df1827Smickey } 312f5df1827Smickey *idt_slot = idtvec; 313f5df1827Smickey *index = slot; 314f5df1827Smickey *cip = ci; 315f5df1827Smickey return 0; 316f5df1827Smickey } 317f5df1827Smickey 318177417ccSderaadt /* 319177417ccSderaadt * True if the system has any non-level interrupts which are shared 320177417ccSderaadt * on the same pin. 321177417ccSderaadt */ 322177417ccSderaadt int intr_shared_edge; 323177417ccSderaadt 324f5df1827Smickey void * 325f5df1827Smickey intr_establish(int legacy_irq, struct pic *pic, int pin, int type, int level, 326c03b1b92Smk int (*handler)(void *), void *arg, const char *what) 327f5df1827Smickey { 328f5df1827Smickey struct intrhand **p, *q, *ih; 329f5df1827Smickey struct cpu_info *ci; 330f5df1827Smickey int slot, error, idt_vec; 331f5df1827Smickey struct intrsource *source; 332f5df1827Smickey struct intrstub *stubp; 333483e3bc5Skettenis int flags; 334f5df1827Smickey 335f5df1827Smickey #ifdef DIAGNOSTIC 336f5df1827Smickey if (legacy_irq != -1 && (legacy_irq < 0 || legacy_irq > 15)) 337f5df1827Smickey panic("intr_establish: bad legacy IRQ value"); 338f5df1827Smickey 339f5df1827Smickey if (legacy_irq == -1 && pic == &i8259_pic) 340f5df1827Smickey panic("intr_establish: non-legacy IRQ on i8259"); 341f5df1827Smickey #endif 342f5df1827Smickey 343483e3bc5Skettenis flags = level & IPL_MPSAFE; 344483e3bc5Skettenis level &= ~IPL_MPSAFE; 345483e3bc5Skettenis 346a12998adSkettenis KASSERT(level <= IPL_TTY || level >= IPL_CLOCK || flags & IPL_MPSAFE); 347a12998adSkettenis 348f5df1827Smickey error = intr_allocate_slot(pic, legacy_irq, pin, level, &ci, &slot, 349f5df1827Smickey &idt_vec); 350f5df1827Smickey if (error != 0) { 351f5df1827Smickey printf("failed to allocate interrupt slot for PIC %s pin %d\n", 352f5df1827Smickey pic->pic_dev.dv_xname, pin); 353f5df1827Smickey return NULL; 354f5df1827Smickey } 355f5df1827Smickey 356f5df1827Smickey /* no point in sleeping unless someone can free memory. */ 357f5df1827Smickey ih = malloc(sizeof *ih, M_DEVBUF, cold ? M_NOWAIT : M_WAITOK); 358f5df1827Smickey if (ih == NULL) { 359f5df1827Smickey printf("intr_establish: can't allocate handler info\n"); 360f5df1827Smickey return NULL; 361f5df1827Smickey } 362f5df1827Smickey 363f5df1827Smickey source = ci->ci_isources[slot]; 364f5df1827Smickey 365f5df1827Smickey if (source->is_handlers != NULL && 366f5df1827Smickey source->is_pic->pic_type != pic->pic_type) { 367bae2bd50Sderaadt free(ih, M_DEVBUF, sizeof(*ih)); 368f5df1827Smickey printf("intr_establish: can't share intr source between " 369f5df1827Smickey "different PIC types (legacy_irq %d pin %d slot %d)\n", 370f5df1827Smickey legacy_irq, pin, slot); 371f5df1827Smickey return NULL; 372f5df1827Smickey } 373f5df1827Smickey 374f5df1827Smickey source->is_pin = pin; 375f5df1827Smickey source->is_pic = pic; 376f5df1827Smickey 377f5df1827Smickey switch (source->is_type) { 378f5df1827Smickey case IST_NONE: 379f5df1827Smickey source->is_type = type; 380f5df1827Smickey break; 381f5df1827Smickey case IST_EDGE: 382177417ccSderaadt intr_shared_edge = 1; 383177417ccSderaadt /* FALLTHROUGH */ 384f5df1827Smickey case IST_LEVEL: 385f5df1827Smickey if (source->is_type == type) 386f5df1827Smickey break; 387f5df1827Smickey case IST_PULSE: 388f5df1827Smickey if (type != IST_NONE) { 389f5df1827Smickey printf("intr_establish: pic %s pin %d: can't share " 390f5df1827Smickey "type %d with %d\n", pic->pic_name, pin, 391f5df1827Smickey source->is_type, type); 392bae2bd50Sderaadt free(ih, M_DEVBUF, sizeof(*ih)); 393f5df1827Smickey return NULL; 394f5df1827Smickey } 395f5df1827Smickey break; 396f5df1827Smickey default: 39762a36cb0Sfgsch panic("intr_establish: bad intr type %d for pic %s pin %d", 398f5df1827Smickey source->is_type, pic->pic_dev.dv_xname, pin); 399f5df1827Smickey } 400f5df1827Smickey 401f5df1827Smickey if (!cold) 402f5df1827Smickey pic->pic_hwmask(pic, pin); 403f5df1827Smickey 404f5df1827Smickey /* 405f5df1827Smickey * Figure out where to put the handler. 406f5df1827Smickey * This is O(N^2), but we want to preserve the order, and N is 407f5df1827Smickey * generally small. 408f5df1827Smickey */ 409f5df1827Smickey for (p = &ci->ci_isources[slot]->is_handlers; 410f5df1827Smickey (q = *p) != NULL && q->ih_level > level; 411f5df1827Smickey p = &q->ih_next) 412f5df1827Smickey ; 413f5df1827Smickey 414f5df1827Smickey ih->ih_fun = handler; 415f5df1827Smickey ih->ih_arg = arg; 416f5df1827Smickey ih->ih_next = *p; 417f5df1827Smickey ih->ih_level = level; 418483e3bc5Skettenis ih->ih_flags = flags; 419f5df1827Smickey ih->ih_pin = pin; 420f5df1827Smickey ih->ih_cpu = ci; 421f5df1827Smickey ih->ih_slot = slot; 4224667bebbSmatthew evcount_attach(&ih->ih_count, what, &source->is_idtvec); 4236816c79aSderaadt 424f5df1827Smickey *p = ih; 425f5df1827Smickey 426f5df1827Smickey intr_calculatemasks(ci); 427f5df1827Smickey 428f5df1827Smickey if (ci->ci_isources[slot]->is_resume == NULL || 429f5df1827Smickey source->is_idtvec != idt_vec) { 430f5df1827Smickey if (source->is_idtvec != 0 && source->is_idtvec != idt_vec) 431f5df1827Smickey idt_vec_free(source->is_idtvec); 432f5df1827Smickey source->is_idtvec = idt_vec; 433f5df1827Smickey stubp = type == IST_LEVEL ? 434f5df1827Smickey &pic->pic_level_stubs[slot] : &pic->pic_edge_stubs[slot]; 435f5df1827Smickey ci->ci_isources[slot]->is_resume = stubp->ist_resume; 436f5df1827Smickey ci->ci_isources[slot]->is_recurse = stubp->ist_recurse; 437f5df1827Smickey setgate(&idt[idt_vec], stubp->ist_entry, 0, SDT_SYS386IGT, 438f5df1827Smickey SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 439f5df1827Smickey } 440f5df1827Smickey 441f5df1827Smickey pic->pic_addroute(pic, ci, pin, idt_vec, type); 442f5df1827Smickey 443f5df1827Smickey if (!cold) 444f5df1827Smickey pic->pic_hwunmask(pic, pin); 445f5df1827Smickey 446f5df1827Smickey #ifdef INTRDEBUG 447f5df1827Smickey printf("allocated pic %s type %s pin %d level %d to cpu%u slot %d idt entry %d\n", 448f5df1827Smickey pic->pic_name, type == IST_EDGE ? "edge" : "level", pin, level, 449f5df1827Smickey ci->ci_apicid, slot, idt_vec); 450f5df1827Smickey #endif 451f5df1827Smickey 452f5df1827Smickey return (ih); 453f5df1827Smickey } 454f5df1827Smickey 455f5df1827Smickey /* 456f5df1827Smickey * Deregister an interrupt handler. 457f5df1827Smickey */ 458f5df1827Smickey void 459f5df1827Smickey intr_disestablish(struct intrhand *ih) 460f5df1827Smickey { 461f5df1827Smickey struct intrhand **p, *q; 462f5df1827Smickey struct cpu_info *ci; 463f5df1827Smickey struct pic *pic; 464f5df1827Smickey struct intrsource *source; 465f5df1827Smickey int idtvec; 466f5df1827Smickey 467f5df1827Smickey ci = ih->ih_cpu; 468f5df1827Smickey pic = ci->ci_isources[ih->ih_slot]->is_pic; 469f5df1827Smickey source = ci->ci_isources[ih->ih_slot]; 470f5df1827Smickey idtvec = source->is_idtvec; 471f5df1827Smickey 472f5df1827Smickey pic->pic_hwmask(pic, ih->ih_pin); 473092bf81aSkettenis x86_atomic_clearbits_u64(&ci->ci_ipending, (1UL << ih->ih_slot)); 474f5df1827Smickey 475f5df1827Smickey /* 476f5df1827Smickey * Remove the handler from the chain. 477f5df1827Smickey */ 478f5df1827Smickey for (p = &source->is_handlers; (q = *p) != NULL && q != ih; 479f5df1827Smickey p = &q->ih_next) 480f5df1827Smickey ; 481f5df1827Smickey if (q == NULL) { 482f5df1827Smickey panic("intr_disestablish: handler not registered"); 483f5df1827Smickey } 484f5df1827Smickey 485f5df1827Smickey *p = q->ih_next; 486f5df1827Smickey 487f5df1827Smickey intr_calculatemasks(ci); 488590907c7Skettenis if (source->is_handlers == NULL) 489f5df1827Smickey pic->pic_delroute(pic, ci, ih->ih_pin, idtvec, source->is_type); 490590907c7Skettenis else 491f5df1827Smickey pic->pic_hwunmask(pic, ih->ih_pin); 492f5df1827Smickey 493f5df1827Smickey #ifdef INTRDEBUG 494f5df1827Smickey printf("cpu%u: remove slot %d (pic %s pin %d vec %d)\n", 495f5df1827Smickey ci->ci_apicid, ih->ih_slot, pic->pic_dev.dv_xname, ih->ih_pin, 496f5df1827Smickey idtvec); 497f5df1827Smickey #endif 498f5df1827Smickey 499f5df1827Smickey if (source->is_handlers == NULL) { 500cf647cc3Stedu free(source, M_DEVBUF, sizeof (struct intrsource)); 501f5df1827Smickey ci->ci_isources[ih->ih_slot] = NULL; 502f5df1827Smickey if (pic != &i8259_pic) 503f5df1827Smickey idt_vec_free(idtvec); 504f5df1827Smickey } 505f5df1827Smickey 5066816c79aSderaadt evcount_detach(&ih->ih_count); 507bae2bd50Sderaadt free(ih, M_DEVBUF, sizeof(*ih)); 508f5df1827Smickey } 509f5df1827Smickey 51058643530Sratchov int 51158643530Sratchov intr_handler(struct intrframe *frame, struct intrhand *ih) 51258643530Sratchov { 5139c9e7f95Sdlg struct cpu_info *ci = curcpu(); 5149c9e7f95Sdlg int floor; 51558643530Sratchov int rc; 51658643530Sratchov #ifdef MULTIPROCESSOR 51758643530Sratchov int need_lock; 51858643530Sratchov 519483e3bc5Skettenis if (ih->ih_flags & IPL_MPSAFE) 520483e3bc5Skettenis need_lock = 0; 521483e3bc5Skettenis else 5227bf9d514Smpi need_lock = 1; 52358643530Sratchov 52458643530Sratchov if (need_lock) 52558643530Sratchov __mp_lock(&kernel_lock); 52658643530Sratchov #endif 5279c9e7f95Sdlg floor = ci->ci_handled_intr_level; 5289c9e7f95Sdlg ci->ci_handled_intr_level = ih->ih_level; 52958643530Sratchov rc = (*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : frame); 5309c9e7f95Sdlg ci->ci_handled_intr_level = floor; 53158643530Sratchov #ifdef MULTIPROCESSOR 53258643530Sratchov if (need_lock) 53358643530Sratchov __mp_unlock(&kernel_lock); 53458643530Sratchov #endif 53558643530Sratchov return rc; 53658643530Sratchov } 53758643530Sratchov 538f5df1827Smickey #define CONCAT(x,y) __CONCAT(x,y) 539f5df1827Smickey 540f5df1827Smickey /* 541f5df1827Smickey * Fake interrupt handler structures for the benefit of symmetry with 542f5df1827Smickey * other interrupt sources, and the benefit of intr_calculatemasks() 543f5df1827Smickey */ 544f5df1827Smickey struct intrhand fake_softclock_intrhand; 545f5df1827Smickey struct intrhand fake_softnet_intrhand; 546a2cd3cc4Skettenis struct intrhand fake_softtty_intrhand; 547f5df1827Smickey struct intrhand fake_timer_intrhand; 548f5df1827Smickey struct intrhand fake_ipi_intrhand; 549ad5f3503Smikeb #if NXEN > 0 550ad5f3503Smikeb struct intrhand fake_xen_intrhand; 551ad5f3503Smikeb #endif 552218ead0bSmikeb #if NHYPERV > 0 553218ead0bSmikeb struct intrhand fake_hyperv_intrhand; 554218ead0bSmikeb #endif 555f5df1827Smickey 556f5df1827Smickey /* 557f5df1827Smickey * Initialize all handlers that aren't dynamically allocated, and exist 558f5df1827Smickey * for each CPU. 559f5df1827Smickey */ 560f5df1827Smickey void 561f5df1827Smickey cpu_intr_init(struct cpu_info *ci) 562f5df1827Smickey { 563f5df1827Smickey struct intrsource *isp; 564b5b9857bSart #if NLAPIC > 0 && defined(MULTIPROCESSOR) && 0 565f5df1827Smickey int i; 566f5df1827Smickey #endif 567f5df1827Smickey 5683cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO); 569f5df1827Smickey if (isp == NULL) 570f5df1827Smickey panic("can't allocate fixed interrupt source"); 571f5df1827Smickey isp->is_recurse = Xsoftclock; 572f5df1827Smickey isp->is_resume = Xsoftclock; 573f5df1827Smickey fake_softclock_intrhand.ih_level = IPL_SOFTCLOCK; 574f5df1827Smickey isp->is_handlers = &fake_softclock_intrhand; 575f5df1827Smickey isp->is_pic = &softintr_pic; 576f5df1827Smickey ci->ci_isources[SIR_CLOCK] = isp; 5773cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO); 578f5df1827Smickey if (isp == NULL) 579f5df1827Smickey panic("can't allocate fixed interrupt source"); 580f5df1827Smickey isp->is_recurse = Xsoftnet; 581f5df1827Smickey isp->is_resume = Xsoftnet; 582f5df1827Smickey fake_softnet_intrhand.ih_level = IPL_SOFTNET; 583f5df1827Smickey isp->is_handlers = &fake_softnet_intrhand; 584f5df1827Smickey isp->is_pic = &softintr_pic; 585f5df1827Smickey ci->ci_isources[SIR_NET] = isp; 5863cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO); 587f5df1827Smickey if (isp == NULL) 588f5df1827Smickey panic("can't allocate fixed interrupt source"); 589a2cd3cc4Skettenis isp->is_recurse = Xsofttty; 590a2cd3cc4Skettenis isp->is_resume = Xsofttty; 591a2cd3cc4Skettenis fake_softtty_intrhand.ih_level = IPL_SOFTTTY; 592a2cd3cc4Skettenis isp->is_handlers = &fake_softtty_intrhand; 593f5df1827Smickey isp->is_pic = &softintr_pic; 594a2cd3cc4Skettenis ci->ci_isources[SIR_TTY] = isp; 595f5df1827Smickey #if NLAPIC > 0 5963cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO); 597f5df1827Smickey if (isp == NULL) 598f5df1827Smickey panic("can't allocate fixed interrupt source"); 599f5df1827Smickey isp->is_recurse = Xrecurse_lapic_ltimer; 600f5df1827Smickey isp->is_resume = Xresume_lapic_ltimer; 601f5df1827Smickey fake_timer_intrhand.ih_level = IPL_CLOCK; 602f5df1827Smickey isp->is_handlers = &fake_timer_intrhand; 603f5df1827Smickey isp->is_pic = &local_pic; 604f5df1827Smickey ci->ci_isources[LIR_TIMER] = isp; 605f5df1827Smickey #ifdef MULTIPROCESSOR 6063cd0d8ceSchl isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO); 607f5df1827Smickey if (isp == NULL) 608f5df1827Smickey panic("can't allocate fixed interrupt source"); 609f5df1827Smickey isp->is_recurse = Xrecurse_lapic_ipi; 610f5df1827Smickey isp->is_resume = Xresume_lapic_ipi; 611f5df1827Smickey fake_ipi_intrhand.ih_level = IPL_IPI; 612f5df1827Smickey isp->is_handlers = &fake_ipi_intrhand; 613f5df1827Smickey isp->is_pic = &local_pic; 614f5df1827Smickey ci->ci_isources[LIR_IPI] = isp; 615f5df1827Smickey #endif 616ad5f3503Smikeb #if NXEN > 0 617ad5f3503Smikeb isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO); 618ad5f3503Smikeb if (isp == NULL) 619ad5f3503Smikeb panic("can't allocate fixed interrupt source"); 620ad5f3503Smikeb isp->is_recurse = Xrecurse_xen_upcall; 621ad5f3503Smikeb isp->is_resume = Xresume_xen_upcall; 622ad5f3503Smikeb fake_xen_intrhand.ih_level = IPL_NET; 623ad5f3503Smikeb isp->is_handlers = &fake_xen_intrhand; 624ad5f3503Smikeb isp->is_pic = &local_pic; 625ad5f3503Smikeb ci->ci_isources[LIR_XEN] = isp; 626b5b9857bSart #endif 627218ead0bSmikeb #if NHYPERV > 0 628218ead0bSmikeb isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO); 629218ead0bSmikeb if (isp == NULL) 630218ead0bSmikeb panic("can't allocate fixed interrupt source"); 631218ead0bSmikeb isp->is_recurse = Xrecurse_hyperv_upcall; 632218ead0bSmikeb isp->is_resume = Xresume_hyperv_upcall; 633218ead0bSmikeb fake_hyperv_intrhand.ih_level = IPL_NET; 634218ead0bSmikeb isp->is_handlers = &fake_hyperv_intrhand; 635218ead0bSmikeb isp->is_pic = &local_pic; 636218ead0bSmikeb ci->ci_isources[LIR_HYPERV] = isp; 637218ead0bSmikeb #endif 638ad5f3503Smikeb #endif /* NLAPIC */ 639f5df1827Smickey 640f5df1827Smickey intr_calculatemasks(ci); 641f5df1827Smickey 642f5df1827Smickey } 643f5df1827Smickey 644f5df1827Smickey void 645f5df1827Smickey intr_printconfig(void) 646f5df1827Smickey { 647f5df1827Smickey #ifdef INTRDEBUG 648f5df1827Smickey int i; 649f5df1827Smickey struct intrhand *ih; 650f5df1827Smickey struct intrsource *isp; 651f5df1827Smickey struct cpu_info *ci; 652f5df1827Smickey CPU_INFO_ITERATOR cii; 653f5df1827Smickey 6545d5ec4d4Smiod CPU_INFO_FOREACH(cii, ci) { 655f5df1827Smickey printf("cpu%d: interrupt masks:\n", ci->ci_apicid); 656f5df1827Smickey for (i = 0; i < NIPL; i++) 657f5df1827Smickey printf("IPL %d mask %lx unmask %lx\n", i, 658f5df1827Smickey (u_long)ci->ci_imask[i], (u_long)ci->ci_iunmask[i]); 659f5df1827Smickey for (i = 0; i < MAX_INTR_SOURCES; i++) { 660f5df1827Smickey isp = ci->ci_isources[i]; 661f5df1827Smickey if (isp == NULL) 662f5df1827Smickey continue; 663f5df1827Smickey printf("cpu%u source %d is pin %d from pic %s maxlevel %d\n", 664f5df1827Smickey ci->ci_apicid, i, isp->is_pin, 665f5df1827Smickey isp->is_pic->pic_name, isp->is_maxlevel); 666f5df1827Smickey for (ih = isp->is_handlers; ih != NULL; 667f5df1827Smickey ih = ih->ih_next) 668f5df1827Smickey printf("\thandler %p level %d\n", 669f5df1827Smickey ih->ih_fun, ih->ih_level); 670f5df1827Smickey 671f5df1827Smickey } 672f5df1827Smickey } 673f5df1827Smickey #endif 674f5df1827Smickey } 6750b57f6e9Sart 676edc47eabSkettenis void 677*25a8a282Sdlg intr_barrier(void *cookie) 678edc47eabSkettenis { 679*25a8a282Sdlg struct intrhand *ih = cookie; 680*25a8a282Sdlg sched_barrier(ih->ih_cpu); 681edc47eabSkettenis } 682edc47eabSkettenis 6830b57f6e9Sart /* 6840b57f6e9Sart * Add a mask to cpl, and return the old value of cpl. 6850b57f6e9Sart */ 6860b57f6e9Sart int 6870b57f6e9Sart splraise(int nlevel) 6880b57f6e9Sart { 6890b57f6e9Sart int olevel; 6900b57f6e9Sart struct cpu_info *ci = curcpu(); 6910b57f6e9Sart 6920b57f6e9Sart olevel = ci->ci_ilevel; 693b8befa46Sart ci->ci_ilevel = MAX(ci->ci_ilevel, nlevel); 6940b57f6e9Sart return (olevel); 6950b57f6e9Sart } 6960b57f6e9Sart 6970b57f6e9Sart /* 6980b57f6e9Sart * Restore a value to cpl (unmasking interrupts). If any unmasked 6990b57f6e9Sart * interrupts are pending, call Xspllower() to process them. 7000b57f6e9Sart */ 7010b57f6e9Sart int 7020b57f6e9Sart spllower(int nlevel) 7030b57f6e9Sart { 7040b57f6e9Sart int olevel; 7050b57f6e9Sart struct cpu_info *ci = curcpu(); 706092bf81aSkettenis u_int64_t imask; 70713ffdfaaSvisa u_long flags; 7080b57f6e9Sart 709a19c7295Sart imask = IUNMASK(ci, nlevel); 7100b57f6e9Sart olevel = ci->ci_ilevel; 711a19c7295Sart 71213ffdfaaSvisa flags = intr_disable(); 713a19c7295Sart 714a19c7295Sart if (ci->ci_ipending & imask) { 7150b57f6e9Sart Xspllower(nlevel); 716a19c7295Sart } else { 7170b57f6e9Sart ci->ci_ilevel = nlevel; 71813ffdfaaSvisa intr_restore(flags); 719a19c7295Sart } 7200b57f6e9Sart return (olevel); 7210b57f6e9Sart } 7220b57f6e9Sart 7230b57f6e9Sart /* 7240b57f6e9Sart * Software interrupt registration 7250b57f6e9Sart * 7260b57f6e9Sart * We hand-code this to ensure that it's atomic. 7270b57f6e9Sart * 7280b57f6e9Sart * XXX always scheduled on the current CPU. 7290b57f6e9Sart */ 7300b57f6e9Sart void 7310b57f6e9Sart softintr(int sir) 7320b57f6e9Sart { 7330b57f6e9Sart struct cpu_info *ci = curcpu(); 7340b57f6e9Sart 7352df76cc2Sguenther __asm volatile("lock; orq %1, %0" : 736092bf81aSkettenis "=m"(ci->ci_ipending) : "ir" (1UL << sir)); 7370b57f6e9Sart } 738