1646af7c6SJohn Baldwin /*- 2ebf5747bSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3ebf5747bSPedro F. Giffuni * 4646af7c6SJohn Baldwin * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 5646af7c6SJohn Baldwin * All rights reserved. 6646af7c6SJohn Baldwin * 7646af7c6SJohn Baldwin * Redistribution and use in source and binary forms, with or without 8646af7c6SJohn Baldwin * modification, are permitted provided that the following conditions 9646af7c6SJohn Baldwin * are met: 10646af7c6SJohn Baldwin * 1. Redistributions of source code must retain the above copyright 11646af7c6SJohn Baldwin * notice, this list of conditions and the following disclaimer. 12646af7c6SJohn Baldwin * 2. Redistributions in binary form must reproduce the above copyright 13646af7c6SJohn Baldwin * notice, this list of conditions and the following disclaimer in the 14646af7c6SJohn Baldwin * documentation and/or other materials provided with the distribution. 15646af7c6SJohn Baldwin * 16646af7c6SJohn Baldwin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17646af7c6SJohn Baldwin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18646af7c6SJohn Baldwin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19646af7c6SJohn Baldwin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20646af7c6SJohn Baldwin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21646af7c6SJohn Baldwin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22646af7c6SJohn Baldwin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23646af7c6SJohn Baldwin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24646af7c6SJohn Baldwin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25646af7c6SJohn Baldwin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26646af7c6SJohn Baldwin * SUCH DAMAGE. 27646af7c6SJohn Baldwin * 28646af7c6SJohn Baldwin * $FreeBSD$ 29646af7c6SJohn Baldwin */ 30646af7c6SJohn Baldwin 31646af7c6SJohn Baldwin /* 32646af7c6SJohn Baldwin * Machine dependent interrupt code for x86. For x86, we have to 33646af7c6SJohn Baldwin * deal with different PICs. Thus, we use the passed in vector to lookup 34646af7c6SJohn Baldwin * an interrupt source associated with that vector. The interrupt source 35646af7c6SJohn Baldwin * describes which PIC the source belongs to and includes methods to handle 36646af7c6SJohn Baldwin * that source. 37646af7c6SJohn Baldwin */ 38646af7c6SJohn Baldwin 39646af7c6SJohn Baldwin #include "opt_atpic.h" 40646af7c6SJohn Baldwin #include "opt_ddb.h" 41646af7c6SJohn Baldwin 42646af7c6SJohn Baldwin #include <sys/param.h> 43646af7c6SJohn Baldwin #include <sys/bus.h> 44646af7c6SJohn Baldwin #include <sys/interrupt.h> 45646af7c6SJohn Baldwin #include <sys/ktr.h> 46646af7c6SJohn Baldwin #include <sys/kernel.h> 47646af7c6SJohn Baldwin #include <sys/lock.h> 48646af7c6SJohn Baldwin #include <sys/mutex.h> 49646af7c6SJohn Baldwin #include <sys/proc.h> 50dc6a8280SConrad Meyer #include <sys/queue.h> 51dc6a8280SConrad Meyer #include <sys/sbuf.h> 52646af7c6SJohn Baldwin #include <sys/smp.h> 53b9f62e3aSSepherosa Ziehau #include <sys/sx.h> 54dc6a8280SConrad Meyer #include <sys/sysctl.h> 55646af7c6SJohn Baldwin #include <sys/syslog.h> 56646af7c6SJohn Baldwin #include <sys/systm.h> 57dc6a8280SConrad Meyer #include <sys/taskqueue.h> 589ed01c32SGleb Smirnoff #include <sys/vmmeter.h> 59646af7c6SJohn Baldwin #include <machine/clock.h> 60646af7c6SJohn Baldwin #include <machine/intr_machdep.h> 61646af7c6SJohn Baldwin #include <machine/smp.h> 62646af7c6SJohn Baldwin #ifdef DDB 63646af7c6SJohn Baldwin #include <ddb/ddb.h> 64646af7c6SJohn Baldwin #endif 65646af7c6SJohn Baldwin 66646af7c6SJohn Baldwin #ifndef DEV_ATPIC 67646af7c6SJohn Baldwin #include <machine/segments.h> 68646af7c6SJohn Baldwin #include <machine/frame.h> 69646af7c6SJohn Baldwin #include <dev/ic/i8259.h> 70646af7c6SJohn Baldwin #include <x86/isa/icu.h> 71f79309d2SWarner Losh #include <isa/isareg.h> 72646af7c6SJohn Baldwin #endif 73646af7c6SJohn Baldwin 7427a3c9d7SJeff Roberson #include <vm/vm.h> 7527a3c9d7SJeff Roberson 76646af7c6SJohn Baldwin #define MAX_STRAY_LOG 5 77646af7c6SJohn Baldwin 78646af7c6SJohn Baldwin typedef void (*mask_fn)(void *); 79646af7c6SJohn Baldwin 80646af7c6SJohn Baldwin static int intrcnt_index; 81646af7c6SJohn Baldwin static struct intsrc *interrupt_sources[NUM_IO_INTS]; 8235d87c7eSConrad Meyer #ifdef SMP 83dc6a8280SConrad Meyer static struct intsrc *interrupt_sorted[NUM_IO_INTS]; 84dc6a8280SConrad Meyer CTASSERT(sizeof(interrupt_sources) == sizeof(interrupt_sorted)); 85dc6a8280SConrad Meyer static int intrbalance; 86dc6a8280SConrad Meyer SYSCTL_INT(_hw, OID_AUTO, intrbalance, CTLFLAG_RW, &intrbalance, 0, 87dc6a8280SConrad Meyer "Interrupt auto-balance interval (seconds). Zero disables."); 88dc6a8280SConrad Meyer static struct timeout_task intrbalance_task; 8935d87c7eSConrad Meyer #endif 90b9f62e3aSSepherosa Ziehau static struct sx intrsrc_lock; 91b9f62e3aSSepherosa Ziehau static struct mtx intrpic_lock; 92646af7c6SJohn Baldwin static struct mtx intrcnt_lock; 93548b2016SAndriy Gapon static TAILQ_HEAD(pics_head, pic) pics; 94646af7c6SJohn Baldwin 95fdce57a0SJohn Baldwin #if defined(SMP) && !defined(EARLY_AP_STARTUP) 96646af7c6SJohn Baldwin static int assign_cpu; 97646af7c6SJohn Baldwin #endif 98646af7c6SJohn Baldwin 99646af7c6SJohn Baldwin u_long intrcnt[INTRCNT_COUNT]; 100646af7c6SJohn Baldwin char intrnames[INTRCNT_COUNT * (MAXCOMLEN + 1)]; 101646af7c6SJohn Baldwin size_t sintrcnt = sizeof(intrcnt); 102646af7c6SJohn Baldwin size_t sintrnames = sizeof(intrnames); 103646af7c6SJohn Baldwin 104066da805SAdrian Chadd static int intr_assign_cpu(void *arg, int cpu); 105646af7c6SJohn Baldwin static void intr_disable_src(void *arg); 106646af7c6SJohn Baldwin static void intr_init(void *__dummy); 107646af7c6SJohn Baldwin static int intr_pic_registered(struct pic *pic); 108646af7c6SJohn Baldwin static void intrcnt_setname(const char *name, int index); 109646af7c6SJohn Baldwin static void intrcnt_updatename(struct intsrc *is); 110646af7c6SJohn Baldwin static void intrcnt_register(struct intsrc *is); 111646af7c6SJohn Baldwin 112646af7c6SJohn Baldwin static int 113646af7c6SJohn Baldwin intr_pic_registered(struct pic *pic) 114646af7c6SJohn Baldwin { 115646af7c6SJohn Baldwin struct pic *p; 116646af7c6SJohn Baldwin 117548b2016SAndriy Gapon TAILQ_FOREACH(p, &pics, pics) { 118646af7c6SJohn Baldwin if (p == pic) 119646af7c6SJohn Baldwin return (1); 120646af7c6SJohn Baldwin } 121646af7c6SJohn Baldwin return (0); 122646af7c6SJohn Baldwin } 123646af7c6SJohn Baldwin 124646af7c6SJohn Baldwin /* 125646af7c6SJohn Baldwin * Register a new interrupt controller (PIC). This is to support suspend 126646af7c6SJohn Baldwin * and resume where we suspend/resume controllers rather than individual 127646af7c6SJohn Baldwin * sources. This also allows controllers with no active sources (such as 128646af7c6SJohn Baldwin * 8259As in a system using the APICs) to participate in suspend and resume. 129646af7c6SJohn Baldwin */ 130646af7c6SJohn Baldwin int 131646af7c6SJohn Baldwin intr_register_pic(struct pic *pic) 132646af7c6SJohn Baldwin { 133646af7c6SJohn Baldwin int error; 134646af7c6SJohn Baldwin 135b9f62e3aSSepherosa Ziehau mtx_lock(&intrpic_lock); 136646af7c6SJohn Baldwin if (intr_pic_registered(pic)) 137646af7c6SJohn Baldwin error = EBUSY; 138646af7c6SJohn Baldwin else { 139548b2016SAndriy Gapon TAILQ_INSERT_TAIL(&pics, pic, pics); 140646af7c6SJohn Baldwin error = 0; 141646af7c6SJohn Baldwin } 142b9f62e3aSSepherosa Ziehau mtx_unlock(&intrpic_lock); 143646af7c6SJohn Baldwin return (error); 144646af7c6SJohn Baldwin } 145646af7c6SJohn Baldwin 146646af7c6SJohn Baldwin /* 147646af7c6SJohn Baldwin * Register a new interrupt source with the global interrupt system. 148646af7c6SJohn Baldwin * The global interrupts need to be disabled when this function is 149646af7c6SJohn Baldwin * called. 150646af7c6SJohn Baldwin */ 151646af7c6SJohn Baldwin int 152646af7c6SJohn Baldwin intr_register_source(struct intsrc *isrc) 153646af7c6SJohn Baldwin { 154646af7c6SJohn Baldwin int error, vector; 155646af7c6SJohn Baldwin 156646af7c6SJohn Baldwin KASSERT(intr_pic_registered(isrc->is_pic), ("unregistered PIC")); 157646af7c6SJohn Baldwin vector = isrc->is_pic->pic_vector(isrc); 158646af7c6SJohn Baldwin if (interrupt_sources[vector] != NULL) 159646af7c6SJohn Baldwin return (EEXIST); 160646af7c6SJohn Baldwin error = intr_event_create(&isrc->is_event, isrc, 0, vector, 161646af7c6SJohn Baldwin intr_disable_src, (mask_fn)isrc->is_pic->pic_enable_source, 162646af7c6SJohn Baldwin (mask_fn)isrc->is_pic->pic_eoi_source, intr_assign_cpu, "irq%d:", 163646af7c6SJohn Baldwin vector); 164646af7c6SJohn Baldwin if (error) 165646af7c6SJohn Baldwin return (error); 166b9f62e3aSSepherosa Ziehau sx_xlock(&intrsrc_lock); 167646af7c6SJohn Baldwin if (interrupt_sources[vector] != NULL) { 168b9f62e3aSSepherosa Ziehau sx_xunlock(&intrsrc_lock); 169646af7c6SJohn Baldwin intr_event_destroy(isrc->is_event); 170646af7c6SJohn Baldwin return (EEXIST); 171646af7c6SJohn Baldwin } 172646af7c6SJohn Baldwin intrcnt_register(isrc); 173646af7c6SJohn Baldwin interrupt_sources[vector] = isrc; 174646af7c6SJohn Baldwin isrc->is_handlers = 0; 175b9f62e3aSSepherosa Ziehau sx_xunlock(&intrsrc_lock); 176646af7c6SJohn Baldwin return (0); 177646af7c6SJohn Baldwin } 178646af7c6SJohn Baldwin 179646af7c6SJohn Baldwin struct intsrc * 180646af7c6SJohn Baldwin intr_lookup_source(int vector) 181646af7c6SJohn Baldwin { 182646af7c6SJohn Baldwin 18333099716SKonstantin Belousov if (vector < 0 || vector >= nitems(interrupt_sources)) 18433099716SKonstantin Belousov return (NULL); 185646af7c6SJohn Baldwin return (interrupt_sources[vector]); 186646af7c6SJohn Baldwin } 187646af7c6SJohn Baldwin 188646af7c6SJohn Baldwin int 189646af7c6SJohn Baldwin intr_add_handler(const char *name, int vector, driver_filter_t filter, 19027a3c9d7SJeff Roberson driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep, 19127a3c9d7SJeff Roberson int domain) 192646af7c6SJohn Baldwin { 193646af7c6SJohn Baldwin struct intsrc *isrc; 194646af7c6SJohn Baldwin int error; 195646af7c6SJohn Baldwin 196646af7c6SJohn Baldwin isrc = intr_lookup_source(vector); 197646af7c6SJohn Baldwin if (isrc == NULL) 198646af7c6SJohn Baldwin return (EINVAL); 199646af7c6SJohn Baldwin error = intr_event_add_handler(isrc->is_event, name, filter, handler, 200646af7c6SJohn Baldwin arg, intr_priority(flags), flags, cookiep); 201646af7c6SJohn Baldwin if (error == 0) { 202b9f62e3aSSepherosa Ziehau sx_xlock(&intrsrc_lock); 203646af7c6SJohn Baldwin intrcnt_updatename(isrc); 204646af7c6SJohn Baldwin isrc->is_handlers++; 205646af7c6SJohn Baldwin if (isrc->is_handlers == 1) { 20627a3c9d7SJeff Roberson isrc->is_domain = domain; 207646af7c6SJohn Baldwin isrc->is_pic->pic_enable_intr(isrc); 208646af7c6SJohn Baldwin isrc->is_pic->pic_enable_source(isrc); 209646af7c6SJohn Baldwin } 210b9f62e3aSSepherosa Ziehau sx_xunlock(&intrsrc_lock); 211646af7c6SJohn Baldwin } 212646af7c6SJohn Baldwin return (error); 213646af7c6SJohn Baldwin } 214646af7c6SJohn Baldwin 215646af7c6SJohn Baldwin int 216646af7c6SJohn Baldwin intr_remove_handler(void *cookie) 217646af7c6SJohn Baldwin { 218646af7c6SJohn Baldwin struct intsrc *isrc; 21923006680SRoger Pau Monné int error; 220646af7c6SJohn Baldwin 221646af7c6SJohn Baldwin isrc = intr_handler_source(cookie); 222646af7c6SJohn Baldwin error = intr_event_remove_handler(cookie); 223646af7c6SJohn Baldwin if (error == 0) { 224b9f62e3aSSepherosa Ziehau sx_xlock(&intrsrc_lock); 225646af7c6SJohn Baldwin isrc->is_handlers--; 226646af7c6SJohn Baldwin if (isrc->is_handlers == 0) { 227646af7c6SJohn Baldwin isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI); 228646af7c6SJohn Baldwin isrc->is_pic->pic_disable_intr(isrc); 229646af7c6SJohn Baldwin } 230646af7c6SJohn Baldwin intrcnt_updatename(isrc); 231b9f62e3aSSepherosa Ziehau sx_xunlock(&intrsrc_lock); 232646af7c6SJohn Baldwin } 233646af7c6SJohn Baldwin return (error); 234646af7c6SJohn Baldwin } 235646af7c6SJohn Baldwin 236646af7c6SJohn Baldwin int 237646af7c6SJohn Baldwin intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol) 238646af7c6SJohn Baldwin { 239646af7c6SJohn Baldwin struct intsrc *isrc; 240646af7c6SJohn Baldwin 241646af7c6SJohn Baldwin isrc = intr_lookup_source(vector); 242646af7c6SJohn Baldwin if (isrc == NULL) 243646af7c6SJohn Baldwin return (EINVAL); 244646af7c6SJohn Baldwin return (isrc->is_pic->pic_config_intr(isrc, trig, pol)); 245646af7c6SJohn Baldwin } 246646af7c6SJohn Baldwin 247646af7c6SJohn Baldwin static void 248646af7c6SJohn Baldwin intr_disable_src(void *arg) 249646af7c6SJohn Baldwin { 250646af7c6SJohn Baldwin struct intsrc *isrc; 251646af7c6SJohn Baldwin 252646af7c6SJohn Baldwin isrc = arg; 253646af7c6SJohn Baldwin isrc->is_pic->pic_disable_source(isrc, PIC_EOI); 254646af7c6SJohn Baldwin } 255646af7c6SJohn Baldwin 256646af7c6SJohn Baldwin void 257646af7c6SJohn Baldwin intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame) 258646af7c6SJohn Baldwin { 259646af7c6SJohn Baldwin struct intr_event *ie; 260646af7c6SJohn Baldwin int vector; 261646af7c6SJohn Baldwin 262646af7c6SJohn Baldwin /* 263646af7c6SJohn Baldwin * We count software interrupts when we process them. The 264646af7c6SJohn Baldwin * code here follows previous practice, but there's an 265646af7c6SJohn Baldwin * argument for counting hardware interrupts when they're 266646af7c6SJohn Baldwin * processed too. 267646af7c6SJohn Baldwin */ 268646af7c6SJohn Baldwin (*isrc->is_count)++; 26983c9dea1SGleb Smirnoff VM_CNT_INC(v_intr); 270646af7c6SJohn Baldwin 271646af7c6SJohn Baldwin ie = isrc->is_event; 272646af7c6SJohn Baldwin 273646af7c6SJohn Baldwin /* 274646af7c6SJohn Baldwin * XXX: We assume that IRQ 0 is only used for the ISA timer 275646af7c6SJohn Baldwin * device (clk). 276646af7c6SJohn Baldwin */ 277646af7c6SJohn Baldwin vector = isrc->is_pic->pic_vector(isrc); 278646af7c6SJohn Baldwin if (vector == 0) 279646af7c6SJohn Baldwin clkintr_pending = 1; 280646af7c6SJohn Baldwin 281646af7c6SJohn Baldwin /* 282646af7c6SJohn Baldwin * For stray interrupts, mask and EOI the source, bump the 283646af7c6SJohn Baldwin * stray count, and log the condition. 284646af7c6SJohn Baldwin */ 285646af7c6SJohn Baldwin if (intr_event_handle(ie, frame) != 0) { 286646af7c6SJohn Baldwin isrc->is_pic->pic_disable_source(isrc, PIC_EOI); 287646af7c6SJohn Baldwin (*isrc->is_straycount)++; 288646af7c6SJohn Baldwin if (*isrc->is_straycount < MAX_STRAY_LOG) 289646af7c6SJohn Baldwin log(LOG_ERR, "stray irq%d\n", vector); 290646af7c6SJohn Baldwin else if (*isrc->is_straycount == MAX_STRAY_LOG) 291646af7c6SJohn Baldwin log(LOG_CRIT, 292646af7c6SJohn Baldwin "too many stray irq %d's: not logging anymore\n", 293646af7c6SJohn Baldwin vector); 294646af7c6SJohn Baldwin } 295646af7c6SJohn Baldwin } 296646af7c6SJohn Baldwin 297646af7c6SJohn Baldwin void 298428b7ca2SJustin T. Gibbs intr_resume(bool suspend_cancelled) 299646af7c6SJohn Baldwin { 300646af7c6SJohn Baldwin struct pic *pic; 301646af7c6SJohn Baldwin 302646af7c6SJohn Baldwin #ifndef DEV_ATPIC 303646af7c6SJohn Baldwin atpic_reset(); 304646af7c6SJohn Baldwin #endif 305b9f62e3aSSepherosa Ziehau mtx_lock(&intrpic_lock); 306548b2016SAndriy Gapon TAILQ_FOREACH(pic, &pics, pics) { 307646af7c6SJohn Baldwin if (pic->pic_resume != NULL) 308428b7ca2SJustin T. Gibbs pic->pic_resume(pic, suspend_cancelled); 309646af7c6SJohn Baldwin } 310b9f62e3aSSepherosa Ziehau mtx_unlock(&intrpic_lock); 311646af7c6SJohn Baldwin } 312646af7c6SJohn Baldwin 313646af7c6SJohn Baldwin void 314646af7c6SJohn Baldwin intr_suspend(void) 315646af7c6SJohn Baldwin { 316646af7c6SJohn Baldwin struct pic *pic; 317646af7c6SJohn Baldwin 318b9f62e3aSSepherosa Ziehau mtx_lock(&intrpic_lock); 319548b2016SAndriy Gapon TAILQ_FOREACH_REVERSE(pic, &pics, pics_head, pics) { 320646af7c6SJohn Baldwin if (pic->pic_suspend != NULL) 321646af7c6SJohn Baldwin pic->pic_suspend(pic); 322646af7c6SJohn Baldwin } 323b9f62e3aSSepherosa Ziehau mtx_unlock(&intrpic_lock); 324646af7c6SJohn Baldwin } 325646af7c6SJohn Baldwin 326646af7c6SJohn Baldwin static int 327066da805SAdrian Chadd intr_assign_cpu(void *arg, int cpu) 328646af7c6SJohn Baldwin { 329646af7c6SJohn Baldwin #ifdef SMP 330646af7c6SJohn Baldwin struct intsrc *isrc; 331646af7c6SJohn Baldwin int error; 332646af7c6SJohn Baldwin 333fdce57a0SJohn Baldwin #ifdef EARLY_AP_STARTUP 334fdce57a0SJohn Baldwin MPASS(mp_ncpus == 1 || smp_started); 335fecabb72SJohn Baldwin 336fecabb72SJohn Baldwin /* Nothing to do if there is only a single CPU. */ 337fecabb72SJohn Baldwin if (mp_ncpus > 1 && cpu != NOCPU) { 338fdce57a0SJohn Baldwin #else 339646af7c6SJohn Baldwin /* 340646af7c6SJohn Baldwin * Don't do anything during early boot. We will pick up the 341646af7c6SJohn Baldwin * assignment once the APs are started. 342646af7c6SJohn Baldwin */ 343646af7c6SJohn Baldwin if (assign_cpu && cpu != NOCPU) { 344fdce57a0SJohn Baldwin #endif 345646af7c6SJohn Baldwin isrc = arg; 346b9f62e3aSSepherosa Ziehau sx_xlock(&intrsrc_lock); 347646af7c6SJohn Baldwin error = isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); 348dc6a8280SConrad Meyer if (error == 0) 349dc6a8280SConrad Meyer isrc->is_cpu = cpu; 350b9f62e3aSSepherosa Ziehau sx_xunlock(&intrsrc_lock); 351646af7c6SJohn Baldwin } else 352646af7c6SJohn Baldwin error = 0; 353646af7c6SJohn Baldwin return (error); 354646af7c6SJohn Baldwin #else 355646af7c6SJohn Baldwin return (EOPNOTSUPP); 356646af7c6SJohn Baldwin #endif 357646af7c6SJohn Baldwin } 358646af7c6SJohn Baldwin 359646af7c6SJohn Baldwin static void 360646af7c6SJohn Baldwin intrcnt_setname(const char *name, int index) 361646af7c6SJohn Baldwin { 362646af7c6SJohn Baldwin 363646af7c6SJohn Baldwin snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", 364646af7c6SJohn Baldwin MAXCOMLEN, name); 365646af7c6SJohn Baldwin } 366646af7c6SJohn Baldwin 367646af7c6SJohn Baldwin static void 368646af7c6SJohn Baldwin intrcnt_updatename(struct intsrc *is) 369646af7c6SJohn Baldwin { 370646af7c6SJohn Baldwin 371646af7c6SJohn Baldwin intrcnt_setname(is->is_event->ie_fullname, is->is_index); 372646af7c6SJohn Baldwin } 373646af7c6SJohn Baldwin 374646af7c6SJohn Baldwin static void 375646af7c6SJohn Baldwin intrcnt_register(struct intsrc *is) 376646af7c6SJohn Baldwin { 377646af7c6SJohn Baldwin char straystr[MAXCOMLEN + 1]; 378646af7c6SJohn Baldwin 379646af7c6SJohn Baldwin KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__)); 380646af7c6SJohn Baldwin mtx_lock_spin(&intrcnt_lock); 381646af7c6SJohn Baldwin is->is_index = intrcnt_index; 382646af7c6SJohn Baldwin intrcnt_index += 2; 383646af7c6SJohn Baldwin snprintf(straystr, MAXCOMLEN + 1, "stray irq%d", 384646af7c6SJohn Baldwin is->is_pic->pic_vector(is)); 385646af7c6SJohn Baldwin intrcnt_updatename(is); 386646af7c6SJohn Baldwin is->is_count = &intrcnt[is->is_index]; 387646af7c6SJohn Baldwin intrcnt_setname(straystr, is->is_index + 1); 388646af7c6SJohn Baldwin is->is_straycount = &intrcnt[is->is_index + 1]; 389646af7c6SJohn Baldwin mtx_unlock_spin(&intrcnt_lock); 390646af7c6SJohn Baldwin } 391646af7c6SJohn Baldwin 392646af7c6SJohn Baldwin void 393646af7c6SJohn Baldwin intrcnt_add(const char *name, u_long **countp) 394646af7c6SJohn Baldwin { 395646af7c6SJohn Baldwin 396646af7c6SJohn Baldwin mtx_lock_spin(&intrcnt_lock); 397646af7c6SJohn Baldwin *countp = &intrcnt[intrcnt_index]; 398646af7c6SJohn Baldwin intrcnt_setname(name, intrcnt_index); 399646af7c6SJohn Baldwin intrcnt_index++; 400646af7c6SJohn Baldwin mtx_unlock_spin(&intrcnt_lock); 401646af7c6SJohn Baldwin } 402646af7c6SJohn Baldwin 403646af7c6SJohn Baldwin static void 404646af7c6SJohn Baldwin intr_init(void *dummy __unused) 405646af7c6SJohn Baldwin { 406646af7c6SJohn Baldwin 407646af7c6SJohn Baldwin intrcnt_setname("???", 0); 408646af7c6SJohn Baldwin intrcnt_index = 1; 409548b2016SAndriy Gapon TAILQ_INIT(&pics); 410b9f62e3aSSepherosa Ziehau mtx_init(&intrpic_lock, "intrpic", NULL, MTX_DEF); 411b9f62e3aSSepherosa Ziehau sx_init(&intrsrc_lock, "intrsrc"); 412646af7c6SJohn Baldwin mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); 413646af7c6SJohn Baldwin } 414646af7c6SJohn Baldwin SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); 415646af7c6SJohn Baldwin 4167a2c1d8cSJohn Baldwin static void 4177a2c1d8cSJohn Baldwin intr_init_final(void *dummy __unused) 4187a2c1d8cSJohn Baldwin { 4197a2c1d8cSJohn Baldwin 4207a2c1d8cSJohn Baldwin /* 4217a2c1d8cSJohn Baldwin * Enable interrupts on the BSP after all of the interrupt 4227a2c1d8cSJohn Baldwin * controllers are initialized. Device interrupts are still 4237a2c1d8cSJohn Baldwin * disabled in the interrupt controllers until interrupt 4247a2c1d8cSJohn Baldwin * handlers are registered. Interrupts are enabled on each AP 4257a2c1d8cSJohn Baldwin * after their first context switch. 4267a2c1d8cSJohn Baldwin */ 4277a2c1d8cSJohn Baldwin enable_intr(); 4287a2c1d8cSJohn Baldwin } 4297a2c1d8cSJohn Baldwin SYSINIT(intr_init_final, SI_SUB_INTR, SI_ORDER_ANY, intr_init_final, NULL); 4307a2c1d8cSJohn Baldwin 431646af7c6SJohn Baldwin #ifndef DEV_ATPIC 432646af7c6SJohn Baldwin /* Initialize the two 8259A's to a known-good shutdown state. */ 433646af7c6SJohn Baldwin void 434646af7c6SJohn Baldwin atpic_reset(void) 435646af7c6SJohn Baldwin { 436646af7c6SJohn Baldwin 437646af7c6SJohn Baldwin outb(IO_ICU1, ICW1_RESET | ICW1_IC4); 438646af7c6SJohn Baldwin outb(IO_ICU1 + ICU_IMR_OFFSET, IDT_IO_INTS); 439dff207f8SYoshihiro Takahashi outb(IO_ICU1 + ICU_IMR_OFFSET, IRQ_MASK(ICU_SLAVEID)); 440dff207f8SYoshihiro Takahashi outb(IO_ICU1 + ICU_IMR_OFFSET, MASTER_MODE); 441646af7c6SJohn Baldwin outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff); 442646af7c6SJohn Baldwin outb(IO_ICU1, OCW3_SEL | OCW3_RR); 443646af7c6SJohn Baldwin 444646af7c6SJohn Baldwin outb(IO_ICU2, ICW1_RESET | ICW1_IC4); 445646af7c6SJohn Baldwin outb(IO_ICU2 + ICU_IMR_OFFSET, IDT_IO_INTS + 8); 446dff207f8SYoshihiro Takahashi outb(IO_ICU2 + ICU_IMR_OFFSET, ICU_SLAVEID); 447dff207f8SYoshihiro Takahashi outb(IO_ICU2 + ICU_IMR_OFFSET, SLAVE_MODE); 448646af7c6SJohn Baldwin outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff); 449646af7c6SJohn Baldwin outb(IO_ICU2, OCW3_SEL | OCW3_RR); 450646af7c6SJohn Baldwin } 451646af7c6SJohn Baldwin #endif 452646af7c6SJohn Baldwin 453646af7c6SJohn Baldwin /* Add a description to an active interrupt handler. */ 454646af7c6SJohn Baldwin int 455646af7c6SJohn Baldwin intr_describe(u_int vector, void *ih, const char *descr) 456646af7c6SJohn Baldwin { 457646af7c6SJohn Baldwin struct intsrc *isrc; 458646af7c6SJohn Baldwin int error; 459646af7c6SJohn Baldwin 460646af7c6SJohn Baldwin isrc = intr_lookup_source(vector); 461646af7c6SJohn Baldwin if (isrc == NULL) 462646af7c6SJohn Baldwin return (EINVAL); 463646af7c6SJohn Baldwin error = intr_event_describe_handler(isrc->is_event, ih, descr); 464646af7c6SJohn Baldwin if (error) 465646af7c6SJohn Baldwin return (error); 466646af7c6SJohn Baldwin intrcnt_updatename(isrc); 467646af7c6SJohn Baldwin return (0); 468646af7c6SJohn Baldwin } 469646af7c6SJohn Baldwin 4700a110d5bSKonstantin Belousov void 4710a110d5bSKonstantin Belousov intr_reprogram(void) 4720a110d5bSKonstantin Belousov { 4730a110d5bSKonstantin Belousov struct intsrc *is; 4740a110d5bSKonstantin Belousov int v; 4750a110d5bSKonstantin Belousov 476b9f62e3aSSepherosa Ziehau sx_xlock(&intrsrc_lock); 4770a110d5bSKonstantin Belousov for (v = 0; v < NUM_IO_INTS; v++) { 4780a110d5bSKonstantin Belousov is = interrupt_sources[v]; 4790a110d5bSKonstantin Belousov if (is == NULL) 4800a110d5bSKonstantin Belousov continue; 4810a110d5bSKonstantin Belousov if (is->is_pic->pic_reprogram_pin != NULL) 4820a110d5bSKonstantin Belousov is->is_pic->pic_reprogram_pin(is); 4830a110d5bSKonstantin Belousov } 484b9f62e3aSSepherosa Ziehau sx_xunlock(&intrsrc_lock); 4850a110d5bSKonstantin Belousov } 4860a110d5bSKonstantin Belousov 487646af7c6SJohn Baldwin #ifdef DDB 488646af7c6SJohn Baldwin /* 489646af7c6SJohn Baldwin * Dump data about interrupt handlers 490646af7c6SJohn Baldwin */ 491646af7c6SJohn Baldwin DB_SHOW_COMMAND(irqs, db_show_irqs) 492646af7c6SJohn Baldwin { 493646af7c6SJohn Baldwin struct intsrc **isrc; 494646af7c6SJohn Baldwin int i, verbose; 495646af7c6SJohn Baldwin 496646af7c6SJohn Baldwin if (strcmp(modif, "v") == 0) 497646af7c6SJohn Baldwin verbose = 1; 498646af7c6SJohn Baldwin else 499646af7c6SJohn Baldwin verbose = 0; 500646af7c6SJohn Baldwin isrc = interrupt_sources; 501646af7c6SJohn Baldwin for (i = 0; i < NUM_IO_INTS && !db_pager_quit; i++, isrc++) 502646af7c6SJohn Baldwin if (*isrc != NULL) 503646af7c6SJohn Baldwin db_dump_intr_event((*isrc)->is_event, verbose); 504646af7c6SJohn Baldwin } 505646af7c6SJohn Baldwin #endif 506646af7c6SJohn Baldwin 507646af7c6SJohn Baldwin #ifdef SMP 508646af7c6SJohn Baldwin /* 509646af7c6SJohn Baldwin * Support for balancing interrupt sources across CPUs. For now we just 510646af7c6SJohn Baldwin * allocate CPUs round-robin. 511646af7c6SJohn Baldwin */ 512646af7c6SJohn Baldwin 5138d791e5aSJohn Baldwin cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1); 51427a3c9d7SJeff Roberson static int current_cpu[MAXMEMDOM]; 51527a3c9d7SJeff Roberson 51627a3c9d7SJeff Roberson static void 51727a3c9d7SJeff Roberson intr_init_cpus(void) 51827a3c9d7SJeff Roberson { 51927a3c9d7SJeff Roberson int i; 52027a3c9d7SJeff Roberson 52127a3c9d7SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 52227a3c9d7SJeff Roberson current_cpu[i] = 0; 52327a3c9d7SJeff Roberson if (!CPU_ISSET(current_cpu[i], &intr_cpus) || 52427a3c9d7SJeff Roberson !CPU_ISSET(current_cpu[i], &cpuset_domain[i])) 52527a3c9d7SJeff Roberson intr_next_cpu(i); 52627a3c9d7SJeff Roberson } 52727a3c9d7SJeff Roberson } 528646af7c6SJohn Baldwin 529646af7c6SJohn Baldwin /* 530646af7c6SJohn Baldwin * Return the CPU that the next interrupt source should use. For now 531646af7c6SJohn Baldwin * this just returns the next local APIC according to round-robin. 532646af7c6SJohn Baldwin */ 533646af7c6SJohn Baldwin u_int 53427a3c9d7SJeff Roberson intr_next_cpu(int domain) 535646af7c6SJohn Baldwin { 536646af7c6SJohn Baldwin u_int apic_id; 537646af7c6SJohn Baldwin 538fdce57a0SJohn Baldwin #ifdef EARLY_AP_STARTUP 539fdce57a0SJohn Baldwin MPASS(mp_ncpus == 1 || smp_started); 540fecabb72SJohn Baldwin if (mp_ncpus == 1) 541fecabb72SJohn Baldwin return (PCPU_GET(apic_id)); 542fdce57a0SJohn Baldwin #else 543646af7c6SJohn Baldwin /* Leave all interrupts on the BSP during boot. */ 544646af7c6SJohn Baldwin if (!assign_cpu) 545646af7c6SJohn Baldwin return (PCPU_GET(apic_id)); 546fdce57a0SJohn Baldwin #endif 547646af7c6SJohn Baldwin 548646af7c6SJohn Baldwin mtx_lock_spin(&icu_lock); 54927a3c9d7SJeff Roberson apic_id = cpu_apic_ids[current_cpu[domain]]; 550646af7c6SJohn Baldwin do { 55127a3c9d7SJeff Roberson current_cpu[domain]++; 55227a3c9d7SJeff Roberson if (current_cpu[domain] > mp_maxid) 55327a3c9d7SJeff Roberson current_cpu[domain] = 0; 55427a3c9d7SJeff Roberson } while (!CPU_ISSET(current_cpu[domain], &intr_cpus) || 55527a3c9d7SJeff Roberson !CPU_ISSET(current_cpu[domain], &cpuset_domain[domain])); 556646af7c6SJohn Baldwin mtx_unlock_spin(&icu_lock); 557646af7c6SJohn Baldwin return (apic_id); 558646af7c6SJohn Baldwin } 559646af7c6SJohn Baldwin 560646af7c6SJohn Baldwin /* Attempt to bind the specified IRQ to the specified CPU. */ 561646af7c6SJohn Baldwin int 562646af7c6SJohn Baldwin intr_bind(u_int vector, u_char cpu) 563646af7c6SJohn Baldwin { 564646af7c6SJohn Baldwin struct intsrc *isrc; 565646af7c6SJohn Baldwin 566646af7c6SJohn Baldwin isrc = intr_lookup_source(vector); 567646af7c6SJohn Baldwin if (isrc == NULL) 568646af7c6SJohn Baldwin return (EINVAL); 569646af7c6SJohn Baldwin return (intr_event_bind(isrc->is_event, cpu)); 570646af7c6SJohn Baldwin } 571646af7c6SJohn Baldwin 572646af7c6SJohn Baldwin /* 573646af7c6SJohn Baldwin * Add a CPU to our mask of valid CPUs that can be destinations of 574646af7c6SJohn Baldwin * interrupts. 575646af7c6SJohn Baldwin */ 576646af7c6SJohn Baldwin void 577646af7c6SJohn Baldwin intr_add_cpu(u_int cpu) 578646af7c6SJohn Baldwin { 579646af7c6SJohn Baldwin 580646af7c6SJohn Baldwin if (cpu >= MAXCPU) 581646af7c6SJohn Baldwin panic("%s: Invalid CPU ID", __func__); 582646af7c6SJohn Baldwin if (bootverbose) 583646af7c6SJohn Baldwin printf("INTR: Adding local APIC %d as a target\n", 584646af7c6SJohn Baldwin cpu_apic_ids[cpu]); 585646af7c6SJohn Baldwin 586646af7c6SJohn Baldwin CPU_SET(cpu, &intr_cpus); 587646af7c6SJohn Baldwin } 588646af7c6SJohn Baldwin 58927a3c9d7SJeff Roberson #ifdef EARLY_AP_STARTUP 59027a3c9d7SJeff Roberson static void 59127a3c9d7SJeff Roberson intr_smp_startup(void *arg __unused) 59227a3c9d7SJeff Roberson { 59327a3c9d7SJeff Roberson 59427a3c9d7SJeff Roberson intr_init_cpus(); 59527a3c9d7SJeff Roberson return; 59627a3c9d7SJeff Roberson } 59727a3c9d7SJeff Roberson SYSINIT(intr_smp_startup, SI_SUB_SMP, SI_ORDER_SECOND, intr_smp_startup, 59827a3c9d7SJeff Roberson NULL); 59927a3c9d7SJeff Roberson 60027a3c9d7SJeff Roberson #else 601646af7c6SJohn Baldwin /* 602646af7c6SJohn Baldwin * Distribute all the interrupt sources among the available CPUs once the 603646af7c6SJohn Baldwin * AP's have been launched. 604646af7c6SJohn Baldwin */ 605646af7c6SJohn Baldwin static void 606646af7c6SJohn Baldwin intr_shuffle_irqs(void *arg __unused) 607646af7c6SJohn Baldwin { 608646af7c6SJohn Baldwin struct intsrc *isrc; 609dc6a8280SConrad Meyer u_int cpu; 610646af7c6SJohn Baldwin int i; 611646af7c6SJohn Baldwin 61227a3c9d7SJeff Roberson intr_init_cpus(); 613646af7c6SJohn Baldwin /* Don't bother on UP. */ 614646af7c6SJohn Baldwin if (mp_ncpus == 1) 615646af7c6SJohn Baldwin return; 616646af7c6SJohn Baldwin 617646af7c6SJohn Baldwin /* Round-robin assign a CPU to each enabled source. */ 618b9f62e3aSSepherosa Ziehau sx_xlock(&intrsrc_lock); 619646af7c6SJohn Baldwin assign_cpu = 1; 620646af7c6SJohn Baldwin for (i = 0; i < NUM_IO_INTS; i++) { 621646af7c6SJohn Baldwin isrc = interrupt_sources[i]; 622646af7c6SJohn Baldwin if (isrc != NULL && isrc->is_handlers > 0) { 623646af7c6SJohn Baldwin /* 624646af7c6SJohn Baldwin * If this event is already bound to a CPU, 625646af7c6SJohn Baldwin * then assign the source to that CPU instead 626646af7c6SJohn Baldwin * of picking one via round-robin. Note that 627646af7c6SJohn Baldwin * this is careful to only advance the 628646af7c6SJohn Baldwin * round-robin if the CPU assignment succeeds. 629646af7c6SJohn Baldwin */ 630dc6a8280SConrad Meyer cpu = isrc->is_event->ie_cpu; 631dc6a8280SConrad Meyer if (cpu == NOCPU) 63227a3c9d7SJeff Roberson cpu = current_cpu[isrc->is_domain]; 633dc6a8280SConrad Meyer if (isrc->is_pic->pic_assign_cpu(isrc, 634dc6a8280SConrad Meyer cpu_apic_ids[cpu]) == 0) { 635dc6a8280SConrad Meyer isrc->is_cpu = cpu; 636dc6a8280SConrad Meyer if (isrc->is_event->ie_cpu == NOCPU) 63727a3c9d7SJeff Roberson intr_next_cpu(isrc->is_domain); 638dc6a8280SConrad Meyer } 639646af7c6SJohn Baldwin } 640646af7c6SJohn Baldwin } 641b9f62e3aSSepherosa Ziehau sx_xunlock(&intrsrc_lock); 642646af7c6SJohn Baldwin } 643646af7c6SJohn Baldwin SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, 644646af7c6SJohn Baldwin NULL); 645fdce57a0SJohn Baldwin #endif 646dc6a8280SConrad Meyer 647dc6a8280SConrad Meyer /* 648dc6a8280SConrad Meyer * TODO: Export this information in a non-MD fashion, integrate with vmstat -i. 649dc6a8280SConrad Meyer */ 650dc6a8280SConrad Meyer static int 651dc6a8280SConrad Meyer sysctl_hw_intrs(SYSCTL_HANDLER_ARGS) 652dc6a8280SConrad Meyer { 653dc6a8280SConrad Meyer struct sbuf sbuf; 654dc6a8280SConrad Meyer struct intsrc *isrc; 655dc6a8280SConrad Meyer int error; 656dc6a8280SConrad Meyer int i; 657dc6a8280SConrad Meyer 658dc6a8280SConrad Meyer error = sysctl_wire_old_buffer(req, 0); 659dc6a8280SConrad Meyer if (error != 0) 660dc6a8280SConrad Meyer return (error); 661dc6a8280SConrad Meyer 662dc6a8280SConrad Meyer sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 663dc6a8280SConrad Meyer sx_slock(&intrsrc_lock); 664dc6a8280SConrad Meyer for (i = 0; i < NUM_IO_INTS; i++) { 665dc6a8280SConrad Meyer isrc = interrupt_sources[i]; 666dc6a8280SConrad Meyer if (isrc == NULL) 667dc6a8280SConrad Meyer continue; 66827a3c9d7SJeff Roberson sbuf_printf(&sbuf, "%s:%d @cpu%d(domain%d): %ld\n", 669dc6a8280SConrad Meyer isrc->is_event->ie_fullname, 670dc6a8280SConrad Meyer isrc->is_index, 671dc6a8280SConrad Meyer isrc->is_cpu, 67227a3c9d7SJeff Roberson isrc->is_domain, 673dc6a8280SConrad Meyer *isrc->is_count); 674dc6a8280SConrad Meyer } 675dc6a8280SConrad Meyer 676dc6a8280SConrad Meyer sx_sunlock(&intrsrc_lock); 677dc6a8280SConrad Meyer error = sbuf_finish(&sbuf); 678dc6a8280SConrad Meyer sbuf_delete(&sbuf); 679dc6a8280SConrad Meyer return (error); 680dc6a8280SConrad Meyer } 681dc6a8280SConrad Meyer SYSCTL_PROC(_hw, OID_AUTO, intrs, CTLTYPE_STRING | CTLFLAG_RW, 682dc6a8280SConrad Meyer 0, 0, sysctl_hw_intrs, "A", "interrupt:number @cpu: count"); 683dc6a8280SConrad Meyer 684dc6a8280SConrad Meyer /* 685dc6a8280SConrad Meyer * Compare two, possibly NULL, entries in the interrupt source array 686dc6a8280SConrad Meyer * by load. 687dc6a8280SConrad Meyer */ 688dc6a8280SConrad Meyer static int 689dc6a8280SConrad Meyer intrcmp(const void *one, const void *two) 690dc6a8280SConrad Meyer { 691dc6a8280SConrad Meyer const struct intsrc *i1, *i2; 692dc6a8280SConrad Meyer 693dc6a8280SConrad Meyer i1 = *(const struct intsrc * const *)one; 694dc6a8280SConrad Meyer i2 = *(const struct intsrc * const *)two; 695dc6a8280SConrad Meyer if (i1 != NULL && i2 != NULL) 696dc6a8280SConrad Meyer return (*i1->is_count - *i2->is_count); 697dc6a8280SConrad Meyer if (i1 != NULL) 698dc6a8280SConrad Meyer return (1); 699dc6a8280SConrad Meyer if (i2 != NULL) 700dc6a8280SConrad Meyer return (-1); 701dc6a8280SConrad Meyer return (0); 702dc6a8280SConrad Meyer } 703dc6a8280SConrad Meyer 704dc6a8280SConrad Meyer /* 705dc6a8280SConrad Meyer * Balance IRQs across available CPUs according to load. 706dc6a8280SConrad Meyer */ 707dc6a8280SConrad Meyer static void 708dc6a8280SConrad Meyer intr_balance(void *dummy __unused, int pending __unused) 709dc6a8280SConrad Meyer { 710dc6a8280SConrad Meyer struct intsrc *isrc; 711dc6a8280SConrad Meyer int interval; 712dc6a8280SConrad Meyer u_int cpu; 713dc6a8280SConrad Meyer int i; 714dc6a8280SConrad Meyer 715dc6a8280SConrad Meyer interval = intrbalance; 716dc6a8280SConrad Meyer if (interval == 0) 717dc6a8280SConrad Meyer goto out; 718dc6a8280SConrad Meyer 719dc6a8280SConrad Meyer /* 720dc6a8280SConrad Meyer * Sort interrupts according to count. 721dc6a8280SConrad Meyer */ 722dc6a8280SConrad Meyer sx_xlock(&intrsrc_lock); 723dc6a8280SConrad Meyer memcpy(interrupt_sorted, interrupt_sources, sizeof(interrupt_sorted)); 724dc6a8280SConrad Meyer qsort(interrupt_sorted, NUM_IO_INTS, sizeof(interrupt_sorted[0]), 725dc6a8280SConrad Meyer intrcmp); 726dc6a8280SConrad Meyer 727dc6a8280SConrad Meyer /* 728dc6a8280SConrad Meyer * Restart the scan from the same location to avoid moving in the 729dc6a8280SConrad Meyer * common case. 730dc6a8280SConrad Meyer */ 73127a3c9d7SJeff Roberson intr_init_cpus(); 732dc6a8280SConrad Meyer 733dc6a8280SConrad Meyer /* 734dc6a8280SConrad Meyer * Assign round-robin from most loaded to least. 735dc6a8280SConrad Meyer */ 736dc6a8280SConrad Meyer for (i = NUM_IO_INTS - 1; i >= 0; i--) { 737dc6a8280SConrad Meyer isrc = interrupt_sorted[i]; 738dc6a8280SConrad Meyer if (isrc == NULL || isrc->is_event->ie_cpu != NOCPU) 739dc6a8280SConrad Meyer continue; 74027a3c9d7SJeff Roberson cpu = current_cpu[isrc->is_domain]; 74127a3c9d7SJeff Roberson intr_next_cpu(isrc->is_domain); 742dc6a8280SConrad Meyer if (isrc->is_cpu != cpu && 743dc6a8280SConrad Meyer isrc->is_pic->pic_assign_cpu(isrc, 744dc6a8280SConrad Meyer cpu_apic_ids[cpu]) == 0) 745dc6a8280SConrad Meyer isrc->is_cpu = cpu; 746dc6a8280SConrad Meyer } 747dc6a8280SConrad Meyer sx_xunlock(&intrsrc_lock); 748dc6a8280SConrad Meyer out: 749dc6a8280SConrad Meyer taskqueue_enqueue_timeout(taskqueue_thread, &intrbalance_task, 750dc6a8280SConrad Meyer interval ? hz * interval : hz * 60); 751dc6a8280SConrad Meyer 752dc6a8280SConrad Meyer } 753dc6a8280SConrad Meyer 754dc6a8280SConrad Meyer static void 755dc6a8280SConrad Meyer intr_balance_init(void *dummy __unused) 756dc6a8280SConrad Meyer { 757dc6a8280SConrad Meyer 758dc6a8280SConrad Meyer TIMEOUT_TASK_INIT(taskqueue_thread, &intrbalance_task, 0, intr_balance, 759dc6a8280SConrad Meyer NULL); 760dc6a8280SConrad Meyer taskqueue_enqueue_timeout(taskqueue_thread, &intrbalance_task, hz); 761dc6a8280SConrad Meyer } 762dc6a8280SConrad Meyer SYSINIT(intr_balance_init, SI_SUB_SMP, SI_ORDER_ANY, intr_balance_init, NULL); 763dc6a8280SConrad Meyer 764646af7c6SJohn Baldwin #else 765646af7c6SJohn Baldwin /* 766646af7c6SJohn Baldwin * Always route interrupts to the current processor in the UP case. 767646af7c6SJohn Baldwin */ 768646af7c6SJohn Baldwin u_int 76927a3c9d7SJeff Roberson intr_next_cpu(int domain) 770646af7c6SJohn Baldwin { 771646af7c6SJohn Baldwin 772646af7c6SJohn Baldwin return (PCPU_GET(apic_id)); 773646af7c6SJohn Baldwin } 774646af7c6SJohn Baldwin #endif 775