xref: /freebsd/sys/x86/x86/intr_machdep.c (revision 38069501) 
1 /*-
2  * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD$
27  */
28 
29 /*
30  * Machine dependent interrupt code for x86.  For x86, we have to
31  * deal with different PICs.  Thus, we use the passed in vector to lookup
32  * an interrupt source associated with that vector.  The interrupt source
33  * describes which PIC the source belongs to and includes methods to handle
34  * that source.
35  */
36 
37 #include "opt_atpic.h"
38 #include "opt_ddb.h"
39 
40 #include <sys/param.h>
41 #include <sys/bus.h>
42 #include <sys/interrupt.h>
43 #include <sys/ktr.h>
44 #include <sys/kernel.h>
45 #include <sys/lock.h>
46 #include <sys/mutex.h>
47 #include <sys/proc.h>
48 #include <sys/queue.h>
49 #include <sys/sbuf.h>
50 #include <sys/smp.h>
51 #include <sys/sx.h>
52 #include <sys/sysctl.h>
53 #include <sys/syslog.h>
54 #include <sys/systm.h>
55 #include <sys/taskqueue.h>
56 #include <sys/vmmeter.h>
57 #include <machine/clock.h>
58 #include <machine/intr_machdep.h>
59 #include <machine/smp.h>
60 #ifdef DDB
61 #include <ddb/ddb.h>
62 #endif
63 
64 #ifndef DEV_ATPIC
65 #include <machine/segments.h>
66 #include <machine/frame.h>
67 #include <dev/ic/i8259.h>
68 #include <x86/isa/icu.h>
69 #include <isa/isareg.h>
70 #endif
71 
72 #define	MAX_STRAY_LOG	5
73 
74 typedef void (*mask_fn)(void *);
75 
76 static int intrcnt_index;
77 static struct intsrc *interrupt_sources[NUM_IO_INTS];
78 #ifdef SMP
79 static struct intsrc *interrupt_sorted[NUM_IO_INTS];
80 CTASSERT(sizeof(interrupt_sources) == sizeof(interrupt_sorted));
81 static int intrbalance;
82 SYSCTL_INT(_hw, OID_AUTO, intrbalance, CTLFLAG_RW, &intrbalance, 0,
83     "Interrupt auto-balance interval (seconds).  Zero disables.");
84 static struct timeout_task intrbalance_task;
85 #endif
86 static struct sx intrsrc_lock;
87 static struct mtx intrpic_lock;
88 static struct mtx intrcnt_lock;
89 static TAILQ_HEAD(pics_head, pic) pics;
90 
91 #if defined(SMP) && !defined(EARLY_AP_STARTUP)
92 static int assign_cpu;
93 #endif
94 
95 u_long intrcnt[INTRCNT_COUNT];
96 char intrnames[INTRCNT_COUNT * (MAXCOMLEN + 1)];
97 size_t sintrcnt = sizeof(intrcnt);
98 size_t sintrnames = sizeof(intrnames);
99 
100 static int	intr_assign_cpu(void *arg, int cpu);
101 static void	intr_disable_src(void *arg);
102 static void	intr_init(void *__dummy);
103 static int	intr_pic_registered(struct pic *pic);
104 static void	intrcnt_setname(const char *name, int index);
105 static void	intrcnt_updatename(struct intsrc *is);
106 static void	intrcnt_register(struct intsrc *is);
107 
108 static int
109 intr_pic_registered(struct pic *pic)
110 {
111 	struct pic *p;
112 
113 	TAILQ_FOREACH(p, &pics, pics) {
114 		if (p == pic)
115 			return (1);
116 	}
117 	return (0);
118 }
119 
120 /*
121  * Register a new interrupt controller (PIC).  This is to support suspend
122  * and resume where we suspend/resume controllers rather than individual
123  * sources.  This also allows controllers with no active sources (such as
124  * 8259As in a system using the APICs) to participate in suspend and resume.
125  */
126 int
127 intr_register_pic(struct pic *pic)
128 {
129 	int error;
130 
131 	mtx_lock(&intrpic_lock);
132 	if (intr_pic_registered(pic))
133 		error = EBUSY;
134 	else {
135 		TAILQ_INSERT_TAIL(&pics, pic, pics);
136 		error = 0;
137 	}
138 	mtx_unlock(&intrpic_lock);
139 	return (error);
140 }
141 
142 /*
143  * Register a new interrupt source with the global interrupt system.
144  * The global interrupts need to be disabled when this function is
145  * called.
146  */
147 int
148 intr_register_source(struct intsrc *isrc)
149 {
150 	int error, vector;
151 
152 	KASSERT(intr_pic_registered(isrc->is_pic), ("unregistered PIC"));
153 	vector = isrc->is_pic->pic_vector(isrc);
154 	if (interrupt_sources[vector] != NULL)
155 		return (EEXIST);
156 	error = intr_event_create(&isrc->is_event, isrc, 0, vector,
157 	    intr_disable_src, (mask_fn)isrc->is_pic->pic_enable_source,
158 	    (mask_fn)isrc->is_pic->pic_eoi_source, intr_assign_cpu, "irq%d:",
159 	    vector);
160 	if (error)
161 		return (error);
162 	sx_xlock(&intrsrc_lock);
163 	if (interrupt_sources[vector] != NULL) {
164 		sx_xunlock(&intrsrc_lock);
165 		intr_event_destroy(isrc->is_event);
166 		return (EEXIST);
167 	}
168 	intrcnt_register(isrc);
169 	interrupt_sources[vector] = isrc;
170 	isrc->is_handlers = 0;
171 	sx_xunlock(&intrsrc_lock);
172 	return (0);
173 }
174 
175 struct intsrc *
176 intr_lookup_source(int vector)
177 {
178 
179 	return (interrupt_sources[vector]);
180 }
181 
182 int
183 intr_add_handler(const char *name, int vector, driver_filter_t filter,
184     driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)
185 {
186 	struct intsrc *isrc;
187 	int error;
188 
189 	isrc = intr_lookup_source(vector);
190 	if (isrc == NULL)
191 		return (EINVAL);
192 	error = intr_event_add_handler(isrc->is_event, name, filter, handler,
193 	    arg, intr_priority(flags), flags, cookiep);
194 	if (error == 0) {
195 		sx_xlock(&intrsrc_lock);
196 		intrcnt_updatename(isrc);
197 		isrc->is_handlers++;
198 		if (isrc->is_handlers == 1) {
199 			isrc->is_pic->pic_enable_intr(isrc);
200 			isrc->is_pic->pic_enable_source(isrc);
201 		}
202 		sx_xunlock(&intrsrc_lock);
203 	}
204 	return (error);
205 }
206 
207 int
208 intr_remove_handler(void *cookie)
209 {
210 	struct intsrc *isrc;
211 	int error;
212 
213 	isrc = intr_handler_source(cookie);
214 	error = intr_event_remove_handler(cookie);
215 	if (error == 0) {
216 		sx_xlock(&intrsrc_lock);
217 		isrc->is_handlers--;
218 		if (isrc->is_handlers == 0) {
219 			isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI);
220 			isrc->is_pic->pic_disable_intr(isrc);
221 		}
222 		intrcnt_updatename(isrc);
223 		sx_xunlock(&intrsrc_lock);
224 	}
225 	return (error);
226 }
227 
228 int
229 intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol)
230 {
231 	struct intsrc *isrc;
232 
233 	isrc = intr_lookup_source(vector);
234 	if (isrc == NULL)
235 		return (EINVAL);
236 	return (isrc->is_pic->pic_config_intr(isrc, trig, pol));
237 }
238 
239 static void
240 intr_disable_src(void *arg)
241 {
242 	struct intsrc *isrc;
243 
244 	isrc = arg;
245 	isrc->is_pic->pic_disable_source(isrc, PIC_EOI);
246 }
247 
248 void
249 intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame)
250 {
251 	struct intr_event *ie;
252 	int vector;
253 
254 	/*
255 	 * We count software interrupts when we process them.  The
256 	 * code here follows previous practice, but there's an
257 	 * argument for counting hardware interrupts when they're
258 	 * processed too.
259 	 */
260 	(*isrc->is_count)++;
261 	VM_CNT_INC(v_intr);
262 
263 	ie = isrc->is_event;
264 
265 	/*
266 	 * XXX: We assume that IRQ 0 is only used for the ISA timer
267 	 * device (clk).
268 	 */
269 	vector = isrc->is_pic->pic_vector(isrc);
270 	if (vector == 0)
271 		clkintr_pending = 1;
272 
273 	/*
274 	 * For stray interrupts, mask and EOI the source, bump the
275 	 * stray count, and log the condition.
276 	 */
277 	if (intr_event_handle(ie, frame) != 0) {
278 		isrc->is_pic->pic_disable_source(isrc, PIC_EOI);
279 		(*isrc->is_straycount)++;
280 		if (*isrc->is_straycount < MAX_STRAY_LOG)
281 			log(LOG_ERR, "stray irq%d\n", vector);
282 		else if (*isrc->is_straycount == MAX_STRAY_LOG)
283 			log(LOG_CRIT,
284 			    "too many stray irq %d's: not logging anymore\n",
285 			    vector);
286 	}
287 }
288 
289 void
290 intr_resume(bool suspend_cancelled)
291 {
292 	struct pic *pic;
293 
294 #ifndef DEV_ATPIC
295 	atpic_reset();
296 #endif
297 	mtx_lock(&intrpic_lock);
298 	TAILQ_FOREACH(pic, &pics, pics) {
299 		if (pic->pic_resume != NULL)
300 			pic->pic_resume(pic, suspend_cancelled);
301 	}
302 	mtx_unlock(&intrpic_lock);
303 }
304 
305 void
306 intr_suspend(void)
307 {
308 	struct pic *pic;
309 
310 	mtx_lock(&intrpic_lock);
311 	TAILQ_FOREACH_REVERSE(pic, &pics, pics_head, pics) {
312 		if (pic->pic_suspend != NULL)
313 			pic->pic_suspend(pic);
314 	}
315 	mtx_unlock(&intrpic_lock);
316 }
317 
318 static int
319 intr_assign_cpu(void *arg, int cpu)
320 {
321 #ifdef SMP
322 	struct intsrc *isrc;
323 	int error;
324 
325 #ifdef EARLY_AP_STARTUP
326 	MPASS(mp_ncpus == 1 || smp_started);
327 
328 	/* Nothing to do if there is only a single CPU. */
329 	if (mp_ncpus > 1 && cpu != NOCPU) {
330 #else
331 	/*
332 	 * Don't do anything during early boot.  We will pick up the
333 	 * assignment once the APs are started.
334 	 */
335 	if (assign_cpu && cpu != NOCPU) {
336 #endif
337 		isrc = arg;
338 		sx_xlock(&intrsrc_lock);
339 		error = isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]);
340 		if (error == 0)
341 			isrc->is_cpu = cpu;
342 		sx_xunlock(&intrsrc_lock);
343 	} else
344 		error = 0;
345 	return (error);
346 #else
347 	return (EOPNOTSUPP);
348 #endif
349 }
350 
351 static void
352 intrcnt_setname(const char *name, int index)
353 {
354 
355 	snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s",
356 	    MAXCOMLEN, name);
357 }
358 
359 static void
360 intrcnt_updatename(struct intsrc *is)
361 {
362 
363 	intrcnt_setname(is->is_event->ie_fullname, is->is_index);
364 }
365 
366 static void
367 intrcnt_register(struct intsrc *is)
368 {
369 	char straystr[MAXCOMLEN + 1];
370 
371 	KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__));
372 	mtx_lock_spin(&intrcnt_lock);
373 	is->is_index = intrcnt_index;
374 	intrcnt_index += 2;
375 	snprintf(straystr, MAXCOMLEN + 1, "stray irq%d",
376 	    is->is_pic->pic_vector(is));
377 	intrcnt_updatename(is);
378 	is->is_count = &intrcnt[is->is_index];
379 	intrcnt_setname(straystr, is->is_index + 1);
380 	is->is_straycount = &intrcnt[is->is_index + 1];
381 	mtx_unlock_spin(&intrcnt_lock);
382 }
383 
384 void
385 intrcnt_add(const char *name, u_long **countp)
386 {
387 
388 	mtx_lock_spin(&intrcnt_lock);
389 	*countp = &intrcnt[intrcnt_index];
390 	intrcnt_setname(name, intrcnt_index);
391 	intrcnt_index++;
392 	mtx_unlock_spin(&intrcnt_lock);
393 }
394 
395 static void
396 intr_init(void *dummy __unused)
397 {
398 
399 	intrcnt_setname("???", 0);
400 	intrcnt_index = 1;
401 	TAILQ_INIT(&pics);
402 	mtx_init(&intrpic_lock, "intrpic", NULL, MTX_DEF);
403 	sx_init(&intrsrc_lock, "intrsrc");
404 	mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
405 }
406 SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL);
407 
408 static void
409 intr_init_final(void *dummy __unused)
410 {
411 
412 	/*
413 	 * Enable interrupts on the BSP after all of the interrupt
414 	 * controllers are initialized.  Device interrupts are still
415 	 * disabled in the interrupt controllers until interrupt
416 	 * handlers are registered.  Interrupts are enabled on each AP
417 	 * after their first context switch.
418 	 */
419 	enable_intr();
420 }
421 SYSINIT(intr_init_final, SI_SUB_INTR, SI_ORDER_ANY, intr_init_final, NULL);
422 
423 #ifndef DEV_ATPIC
424 /* Initialize the two 8259A's to a known-good shutdown state. */
425 void
426 atpic_reset(void)
427 {
428 
429 	outb(IO_ICU1, ICW1_RESET | ICW1_IC4);
430 	outb(IO_ICU1 + ICU_IMR_OFFSET, IDT_IO_INTS);
431 	outb(IO_ICU1 + ICU_IMR_OFFSET, IRQ_MASK(ICU_SLAVEID));
432 	outb(IO_ICU1 + ICU_IMR_OFFSET, MASTER_MODE);
433 	outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff);
434 	outb(IO_ICU1, OCW3_SEL | OCW3_RR);
435 
436 	outb(IO_ICU2, ICW1_RESET | ICW1_IC4);
437 	outb(IO_ICU2 + ICU_IMR_OFFSET, IDT_IO_INTS + 8);
438 	outb(IO_ICU2 + ICU_IMR_OFFSET, ICU_SLAVEID);
439 	outb(IO_ICU2 + ICU_IMR_OFFSET, SLAVE_MODE);
440 	outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff);
441 	outb(IO_ICU2, OCW3_SEL | OCW3_RR);
442 }
443 #endif
444 
445 /* Add a description to an active interrupt handler. */
446 int
447 intr_describe(u_int vector, void *ih, const char *descr)
448 {
449 	struct intsrc *isrc;
450 	int error;
451 
452 	isrc = intr_lookup_source(vector);
453 	if (isrc == NULL)
454 		return (EINVAL);
455 	error = intr_event_describe_handler(isrc->is_event, ih, descr);
456 	if (error)
457 		return (error);
458 	intrcnt_updatename(isrc);
459 	return (0);
460 }
461 
462 void
463 intr_reprogram(void)
464 {
465 	struct intsrc *is;
466 	int v;
467 
468 	sx_xlock(&intrsrc_lock);
469 	for (v = 0; v < NUM_IO_INTS; v++) {
470 		is = interrupt_sources[v];
471 		if (is == NULL)
472 			continue;
473 		if (is->is_pic->pic_reprogram_pin != NULL)
474 			is->is_pic->pic_reprogram_pin(is);
475 	}
476 	sx_xunlock(&intrsrc_lock);
477 }
478 
479 #ifdef DDB
480 /*
481  * Dump data about interrupt handlers
482  */
483 DB_SHOW_COMMAND(irqs, db_show_irqs)
484 {
485 	struct intsrc **isrc;
486 	int i, verbose;
487 
488 	if (strcmp(modif, "v") == 0)
489 		verbose = 1;
490 	else
491 		verbose = 0;
492 	isrc = interrupt_sources;
493 	for (i = 0; i < NUM_IO_INTS && !db_pager_quit; i++, isrc++)
494 		if (*isrc != NULL)
495 			db_dump_intr_event((*isrc)->is_event, verbose);
496 }
497 #endif
498 
499 #ifdef SMP
500 /*
501  * Support for balancing interrupt sources across CPUs.  For now we just
502  * allocate CPUs round-robin.
503  */
504 
505 cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1);
506 static int current_cpu;
507 
508 /*
509  * Return the CPU that the next interrupt source should use.  For now
510  * this just returns the next local APIC according to round-robin.
511  */
512 u_int
513 intr_next_cpu(void)
514 {
515 	u_int apic_id;
516 
517 #ifdef EARLY_AP_STARTUP
518 	MPASS(mp_ncpus == 1 || smp_started);
519 	if (mp_ncpus == 1)
520 		return (PCPU_GET(apic_id));
521 #else
522 	/* Leave all interrupts on the BSP during boot. */
523 	if (!assign_cpu)
524 		return (PCPU_GET(apic_id));
525 #endif
526 
527 	mtx_lock_spin(&icu_lock);
528 	apic_id = cpu_apic_ids[current_cpu];
529 	do {
530 		current_cpu++;
531 		if (current_cpu > mp_maxid)
532 			current_cpu = 0;
533 	} while (!CPU_ISSET(current_cpu, &intr_cpus));
534 	mtx_unlock_spin(&icu_lock);
535 	return (apic_id);
536 }
537 
538 /* Attempt to bind the specified IRQ to the specified CPU. */
539 int
540 intr_bind(u_int vector, u_char cpu)
541 {
542 	struct intsrc *isrc;
543 
544 	isrc = intr_lookup_source(vector);
545 	if (isrc == NULL)
546 		return (EINVAL);
547 	return (intr_event_bind(isrc->is_event, cpu));
548 }
549 
550 /*
551  * Add a CPU to our mask of valid CPUs that can be destinations of
552  * interrupts.
553  */
554 void
555 intr_add_cpu(u_int cpu)
556 {
557 
558 	if (cpu >= MAXCPU)
559 		panic("%s: Invalid CPU ID", __func__);
560 	if (bootverbose)
561 		printf("INTR: Adding local APIC %d as a target\n",
562 		    cpu_apic_ids[cpu]);
563 
564 	CPU_SET(cpu, &intr_cpus);
565 }
566 
567 #ifndef EARLY_AP_STARTUP
568 /*
569  * Distribute all the interrupt sources among the available CPUs once the
570  * AP's have been launched.
571  */
572 static void
573 intr_shuffle_irqs(void *arg __unused)
574 {
575 	struct intsrc *isrc;
576 	u_int cpu;
577 	int i;
578 
579 	/* Don't bother on UP. */
580 	if (mp_ncpus == 1)
581 		return;
582 
583 	/* Round-robin assign a CPU to each enabled source. */
584 	sx_xlock(&intrsrc_lock);
585 	assign_cpu = 1;
586 	for (i = 0; i < NUM_IO_INTS; i++) {
587 		isrc = interrupt_sources[i];
588 		if (isrc != NULL && isrc->is_handlers > 0) {
589 			/*
590 			 * If this event is already bound to a CPU,
591 			 * then assign the source to that CPU instead
592 			 * of picking one via round-robin.  Note that
593 			 * this is careful to only advance the
594 			 * round-robin if the CPU assignment succeeds.
595 			 */
596 			cpu = isrc->is_event->ie_cpu;
597 			if (cpu == NOCPU)
598 				cpu = current_cpu;
599 			if (isrc->is_pic->pic_assign_cpu(isrc,
600 			    cpu_apic_ids[cpu]) == 0) {
601 				isrc->is_cpu = cpu;
602 				if (isrc->is_event->ie_cpu == NOCPU)
603 					intr_next_cpu();
604 			}
605 		}
606 	}
607 	sx_xunlock(&intrsrc_lock);
608 }
609 SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs,
610     NULL);
611 #endif
612 
613 /*
614  * TODO: Export this information in a non-MD fashion, integrate with vmstat -i.
615  */
616 static int
617 sysctl_hw_intrs(SYSCTL_HANDLER_ARGS)
618 {
619 	struct sbuf sbuf;
620 	struct intsrc *isrc;
621 	int error;
622 	int i;
623 
624 	error = sysctl_wire_old_buffer(req, 0);
625 	if (error != 0)
626 		return (error);
627 
628 	sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
629 	sx_slock(&intrsrc_lock);
630 	for (i = 0; i < NUM_IO_INTS; i++) {
631 		isrc = interrupt_sources[i];
632 		if (isrc == NULL)
633 			continue;
634 		sbuf_printf(&sbuf, "%s:%d @%d: %ld\n",
635 		    isrc->is_event->ie_fullname,
636 		    isrc->is_index,
637 		    isrc->is_cpu,
638 		    *isrc->is_count);
639 	}
640 
641 	sx_sunlock(&intrsrc_lock);
642 	error = sbuf_finish(&sbuf);
643 	sbuf_delete(&sbuf);
644 	return (error);
645 }
646 SYSCTL_PROC(_hw, OID_AUTO, intrs, CTLTYPE_STRING | CTLFLAG_RW,
647     0, 0, sysctl_hw_intrs, "A", "interrupt:number @cpu: count");
648 
649 /*
650  * Compare two, possibly NULL, entries in the interrupt source array
651  * by load.
652  */
653 static int
654 intrcmp(const void *one, const void *two)
655 {
656 	const struct intsrc *i1, *i2;
657 
658 	i1 = *(const struct intsrc * const *)one;
659 	i2 = *(const struct intsrc * const *)two;
660 	if (i1 != NULL && i2 != NULL)
661 		return (*i1->is_count - *i2->is_count);
662 	if (i1 != NULL)
663 		return (1);
664 	if (i2 != NULL)
665 		return (-1);
666 	return (0);
667 }
668 
669 /*
670  * Balance IRQs across available CPUs according to load.
671  */
672 static void
673 intr_balance(void *dummy __unused, int pending __unused)
674 {
675 	struct intsrc *isrc;
676 	int interval;
677 	u_int cpu;
678 	int i;
679 
680 	interval = intrbalance;
681 	if (interval == 0)
682 		goto out;
683 
684 	/*
685 	 * Sort interrupts according to count.
686 	 */
687 	sx_xlock(&intrsrc_lock);
688 	memcpy(interrupt_sorted, interrupt_sources, sizeof(interrupt_sorted));
689 	qsort(interrupt_sorted, NUM_IO_INTS, sizeof(interrupt_sorted[0]),
690 	    intrcmp);
691 
692 	/*
693 	 * Restart the scan from the same location to avoid moving in the
694 	 * common case.
695 	 */
696 	current_cpu = 0;
697 
698 	/*
699 	 * Assign round-robin from most loaded to least.
700 	 */
701 	for (i = NUM_IO_INTS - 1; i >= 0; i--) {
702 		isrc = interrupt_sorted[i];
703 		if (isrc == NULL  || isrc->is_event->ie_cpu != NOCPU)
704 			continue;
705 		cpu = current_cpu;
706 		intr_next_cpu();
707 		if (isrc->is_cpu != cpu &&
708 		    isrc->is_pic->pic_assign_cpu(isrc,
709 		    cpu_apic_ids[cpu]) == 0)
710 			isrc->is_cpu = cpu;
711 	}
712 	sx_xunlock(&intrsrc_lock);
713 out:
714 	taskqueue_enqueue_timeout(taskqueue_thread, &intrbalance_task,
715 	    interval ? hz * interval : hz * 60);
716 
717 }
718 
719 static void
720 intr_balance_init(void *dummy __unused)
721 {
722 
723 	TIMEOUT_TASK_INIT(taskqueue_thread, &intrbalance_task, 0, intr_balance,
724 	    NULL);
725 	taskqueue_enqueue_timeout(taskqueue_thread, &intrbalance_task, hz);
726 }
727 SYSINIT(intr_balance_init, SI_SUB_SMP, SI_ORDER_ANY, intr_balance_init, NULL);
728 
729 #else
730 /*
731  * Always route interrupts to the current processor in the UP case.
732  */
733 u_int
734 intr_next_cpu(void)
735 {
736 
737 	return (PCPU_GET(apic_id));
738 }
739 #endif
740