1 /******************************************************************************
2 * xen_intr.c
3 *
4 * Xen event and interrupt services for x86 HVM guests.
5 *
6 * Copyright (c) 2002-2005, K A Fraser
7 * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com>
8 * Copyright (c) 2012, Spectra Logic Corporation
9 * Copyright © 2021-2023, Elliott Mitchell
10 *
11 * This file may be distributed separately from the Linux kernel, or
12 * incorporated into other software packages, subject to the following license:
13 *
14 * Permission is hereby granted, free of charge, to any person obtaining a copy
15 * of this source file (the "Software"), to deal in the Software without
16 * restriction, including without limitation the rights to use, copy, modify,
17 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
18 * and to permit persons to whom the Software is furnished to do so, subject to
19 * the following conditions:
20 *
21 * The above copyright notice and this permission notice shall be included in
22 * all copies or substantial portions of the Software.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
30 * IN THE SOFTWARE.
31 */
32
33 #include <sys/cdefs.h>
34 #include "opt_ddb.h"
35
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/bus.h>
39 #include <sys/kernel.h>
40 #include <sys/limits.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/interrupt.h>
44 #include <sys/pcpu.h>
45 #include <sys/proc.h>
46 #include <sys/smp.h>
47 #include <sys/refcount.h>
48
49 #include <vm/vm.h>
50 #include <vm/pmap.h>
51
52 #include <machine/smp.h>
53 #include <machine/stdarg.h>
54
55 #include <xen/xen-os.h>
56 #include <xen/hypervisor.h>
57 #include <xen/xen_intr.h>
58 #include <xen/evtchn/evtchnvar.h>
59
60 #include <machine/xen/arch-intr.h>
61
62 #ifdef DDB
63 #include <ddb/ddb.h>
64 #endif
65
66 /**
67 * Per-cpu event channel processing state.
68 */
69 struct xen_intr_pcpu_data {
70 /**
71 * The last event channel bitmap section (level one bit) processed.
72 * This is used to ensure we scan all ports before
73 * servicing an already servied port again.
74 */
75 u_int last_processed_l1i;
76
77 /**
78 * The last event channel processed within the event channel
79 * bitmap being scanned.
80 */
81 u_int last_processed_l2i;
82
83 /**
84 * A bitmap of ports that can be serviced from this CPU.
85 * A set bit means interrupt handling is enabled.
86 */
87 xen_ulong_t evtchn_enabled[sizeof(xen_ulong_t) * 8];
88 };
89
90 /*
91 * Start the scan at port 0 by initializing the last scanned
92 * location as the highest numbered event channel port.
93 */
94 DPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = {
95 .last_processed_l1i = LONG_BIT - 1,
96 .last_processed_l2i = LONG_BIT - 1
97 };
98
99 DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
100
101 #define INVALID_EVTCHN (~(evtchn_port_t)0) /* Invalid event channel */
102 #define is_valid_evtchn(x) ((uintmax_t)(x) < NR_EVENT_CHANNELS)
103
104 /*
105 * Lock for interrupt core data.
106 *
107 * Modifying xen_intr_port_to_isrc[], or isrc->xi_port (implies the former)
108 * requires this lock be held. Any time this lock is not held, the condition
109 * `!xen_intr_port_to_isrc[i] || (xen_intr_port_to_isrc[i]->ix_port == i)`
110 * MUST be true for all values of i which are valid indicies of the array.
111 *
112 * Acquire/release operations for isrc->xi_refcount require this lock be held.
113 */
114 static struct mtx xen_intr_isrc_lock;
115 static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS];
116
117 /*------------------------- Private Functions --------------------------------*/
118
119 /**
120 * Retrieve a handle for a Xen interrupt source.
121 *
122 * \param isrc A valid Xen interrupt source structure.
123 *
124 * \returns A handle suitable for use with xen_intr_isrc_from_handle()
125 * to retrieve the original Xen interrupt source structure.
126 */
127
128 static inline xen_intr_handle_t
xen_intr_handle_from_isrc(struct xenisrc * isrc)129 xen_intr_handle_from_isrc(struct xenisrc *isrc)
130 {
131 return (isrc);
132 }
133
134 /**
135 * Lookup a Xen interrupt source object given an interrupt binding handle.
136 *
137 * \param handle A handle initialized by a previous call to
138 * xen_intr_bind_isrc().
139 *
140 * \returns A pointer to the Xen interrupt source object associated
141 * with the given interrupt handle. NULL if no association
142 * currently exists.
143 */
144 static inline struct xenisrc *
xen_intr_isrc_from_handle(xen_intr_handle_t handle)145 xen_intr_isrc_from_handle(xen_intr_handle_t handle)
146 {
147 return ((struct xenisrc *)handle);
148 }
149
150 /**
151 * Disable signal delivery for an event channel port on the
152 * specified CPU.
153 *
154 * \param port The event channel port to mask.
155 *
156 * This API is used to manage the port<=>CPU binding of event
157 * channel handlers.
158 *
159 * \note This operation does not preclude reception of an event
160 * for this event channel on another CPU. To mask the
161 * event channel globally, use evtchn_mask().
162 */
163 static inline void
evtchn_cpu_mask_port(u_int cpu,evtchn_port_t port)164 evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port)
165 {
166 struct xen_intr_pcpu_data *pcpu;
167
168 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
169 KASSERT(is_valid_evtchn(port), ("Invalid event channel port"));
170 xen_clear_bit(port, pcpu->evtchn_enabled);
171 }
172
173 /**
174 * Enable signal delivery for an event channel port on the
175 * specified CPU.
176 *
177 * \param port The event channel port to unmask.
178 *
179 * This API is used to manage the port<=>CPU binding of event
180 * channel handlers.
181 *
182 * \note This operation does not guarantee that event delivery
183 * is enabled for this event channel port. The port must
184 * also be globally enabled. See evtchn_unmask().
185 */
186 static inline void
evtchn_cpu_unmask_port(u_int cpu,evtchn_port_t port)187 evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port)
188 {
189 struct xen_intr_pcpu_data *pcpu;
190
191 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
192 KASSERT(is_valid_evtchn(port), ("Invalid event channel port"));
193 xen_set_bit(port, pcpu->evtchn_enabled);
194 }
195
196 /**
197 * Attempt to free an active Xen interrupt source object.
198 *
199 * \param isrc The interrupt source object to release.
200 *
201 * \returns EBUSY if the source is still in use, otherwise 0.
202 */
203 static int
xen_intr_release_isrc(struct xenisrc * isrc)204 xen_intr_release_isrc(struct xenisrc *isrc)
205 {
206
207 mtx_lock(&xen_intr_isrc_lock);
208 if (is_valid_evtchn(isrc->xi_port)) {
209 evtchn_mask_port(isrc->xi_port);
210 evtchn_clear_port(isrc->xi_port);
211
212 /* Rebind port to CPU 0. */
213 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
214 evtchn_cpu_unmask_port(0, isrc->xi_port);
215
216 if (isrc->xi_close != 0) {
217 struct evtchn_close close = { .port = isrc->xi_port };
218
219 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
220 panic("EVTCHNOP_close failed");
221 }
222
223 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
224 }
225 /* not reachable from xen_intr_port_to_isrc[], unlock */
226 mtx_unlock(&xen_intr_isrc_lock);
227
228 xen_arch_intr_release(isrc);
229 return (0);
230 }
231
232 /**
233 * Associate an interrupt handler with an already allocated local Xen
234 * event channel port.
235 *
236 * \param isrcp The returned Xen interrupt object associated with
237 * the specified local port.
238 * \param local_port The event channel to bind.
239 * \param type The event channel type of local_port.
240 * \param intr_owner The device making this bind request.
241 * \param filter An interrupt filter handler. Specify NULL
242 * to always dispatch to the ithread handler.
243 * \param handler An interrupt ithread handler. Optional (can
244 * specify NULL) if all necessary event actions
245 * are performed by filter.
246 * \param arg Argument to present to both filter and handler.
247 * \param irqflags Interrupt handler flags. See sys/bus.h.
248 * \param handlep Pointer to an opaque handle used to manage this
249 * registration.
250 *
251 * \returns 0 on success, otherwise an errno.
252 */
253 static int
xen_intr_bind_isrc(struct xenisrc ** isrcp,evtchn_port_t local_port,enum evtchn_type type,const char * intr_owner,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * const port_handlep)254 xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port,
255 enum evtchn_type type, const char *intr_owner, driver_filter_t filter,
256 driver_intr_t handler, void *arg, enum intr_type flags,
257 xen_intr_handle_t *const port_handlep)
258 {
259 struct xenisrc *isrc;
260 int error;
261
262 *isrcp = NULL;
263 if (port_handlep == NULL) {
264 printf("%s: %s: Bad event handle\n", intr_owner, __func__);
265 return (EINVAL);
266 }
267 *port_handlep = NULL;
268
269 isrc = xen_arch_intr_alloc();
270 if (isrc == NULL)
271 return (ENOSPC);
272
273 isrc->xi_cookie = NULL;
274 isrc->xi_type = type;
275 isrc->xi_port = local_port;
276 isrc->xi_close = false;
277 isrc->xi_cpu = 0;
278 refcount_init(&isrc->xi_refcount, 1);
279 mtx_lock(&xen_intr_isrc_lock);
280 xen_intr_port_to_isrc[isrc->xi_port] = isrc;
281 mtx_unlock(&xen_intr_isrc_lock);
282
283 #ifdef SMP
284 if (type == EVTCHN_TYPE_PORT) {
285 /*
286 * By default all interrupts are assigned to vCPU#0
287 * unless specified otherwise, so shuffle them to balance
288 * the interrupt load.
289 */
290 xen_intr_assign_cpu(isrc, xen_arch_intr_next_cpu(isrc));
291 }
292 #endif
293
294 /*
295 * If a filter or handler function is provided, add it to the event.
296 * Otherwise the event channel is left masked and without a handler,
297 * the caller is in charge of setting that up.
298 */
299 if (filter != NULL || handler != NULL) {
300 error = xen_intr_add_handler(intr_owner, filter, handler, arg,
301 flags, xen_intr_handle_from_isrc(isrc));
302 if (error != 0) {
303 xen_intr_release_isrc(isrc);
304 return (error);
305 }
306 }
307
308 *isrcp = isrc;
309 /* Assign the opaque handler */
310 *port_handlep = xen_intr_handle_from_isrc(isrc);
311 return (0);
312 }
313
314 /**
315 * Determine the event channel ports at the given section of the
316 * event port bitmap which have pending events for the given cpu.
317 *
318 * \param pcpu The Xen interrupt pcpu data for the cpu being queried.
319 * \param sh The Xen shared info area.
320 * \param idx The index of the section of the event channel bitmap to
321 * inspect.
322 *
323 * \returns A u_long with bits set for every event channel with pending
324 * events.
325 */
326 static inline u_long
xen_intr_active_ports(const struct xen_intr_pcpu_data * const pcpu,const u_int idx)327 xen_intr_active_ports(const struct xen_intr_pcpu_data *const pcpu,
328 const u_int idx)
329 {
330 volatile const shared_info_t *const sh = HYPERVISOR_shared_info;
331
332 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0]));
333 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0]));
334 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending));
335 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled));
336 return (sh->evtchn_pending[idx]
337 & ~sh->evtchn_mask[idx]
338 & pcpu->evtchn_enabled[idx]);
339 }
340
341 /**
342 * Interrupt handler for processing all Xen event channel events.
343 *
344 * \param unused
345 */
346 int
xen_intr_handle_upcall(void * unused __unused)347 xen_intr_handle_upcall(void *unused __unused)
348 {
349 struct trapframe *trap_frame = curthread->td_intr_frame;
350 u_int l1i, l2i, port, cpu __diagused;
351 u_long masked_l1, masked_l2;
352 struct xenisrc *isrc;
353 vcpu_info_t *v;
354 struct xen_intr_pcpu_data *pc;
355 u_long l1, l2;
356
357 /*
358 * The upcall handler is an interrupt handler itself (that calls other
359 * interrupt handlers), hence the caller has the responsibility to
360 * increase td_intr_nesting_level ahead of dispatching the upcall
361 * handler.
362 */
363 KASSERT(curthread->td_intr_nesting_level > 0,
364 ("Unexpected thread context"));
365
366 /* We must remain on the same vCPU during this function */
367 CRITICAL_ASSERT(curthread);
368
369 cpu = PCPU_GET(cpuid);
370 pc = DPCPU_PTR(xen_intr_pcpu);
371 v = DPCPU_GET(vcpu_info);
372
373 if (!xen_has_percpu_evtchn()) {
374 KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU"));
375 }
376
377 v->evtchn_upcall_pending = 0;
378 /* No need for a barrier on x86 -- XCHG is a barrier on x86. */
379 #if !defined(__amd64__) && !defined(__i386__)
380 /* Clear master flag /before/ clearing selector flag. */
381 wmb();
382 #endif
383 l1 = atomic_readandclear_xen_ulong(&v->evtchn_pending_sel);
384
385 l1i = pc->last_processed_l1i;
386 l2i = pc->last_processed_l2i;
387
388 while (l1 != 0) {
389 l1i = (l1i + 1) % LONG_BIT;
390 masked_l1 = l1 & ((~0UL) << l1i);
391
392 if (masked_l1 == 0) {
393 /*
394 * if we masked out all events, wrap around
395 * to the beginning.
396 */
397 l1i = LONG_BIT - 1;
398 l2i = LONG_BIT - 1;
399 continue;
400 }
401 l1i = ffsl(masked_l1) - 1;
402
403 do {
404 l2 = xen_intr_active_ports(pc, l1i);
405
406 l2i = (l2i + 1) % LONG_BIT;
407 masked_l2 = l2 & ((~0UL) << l2i);
408
409 if (masked_l2 == 0) {
410 /* if we masked out all events, move on */
411 l2i = LONG_BIT - 1;
412 break;
413 }
414 l2i = ffsl(masked_l2) - 1;
415
416 /* process port */
417 port = (l1i * LONG_BIT) + l2i;
418 evtchn_clear_port(port);
419
420 isrc = xen_intr_port_to_isrc[port];
421 if (__predict_false(isrc == NULL))
422 continue;
423
424 /* Make sure we are firing on the right vCPU */
425 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)),
426 ("Received unexpected event on vCPU#%u, event bound to vCPU#%u",
427 PCPU_GET(cpuid), isrc->xi_cpu));
428
429 /*
430 * Reduce interrupt nesting level ahead of calling the
431 * per-arch interrupt dispatch helper. This is
432 * required because the per-arch dispatcher will also
433 * increase td_intr_nesting_level, and then handlers
434 * would wrongly see td_intr_nesting_level = 2 when
435 * there's no nesting at all.
436 */
437 curthread->td_intr_nesting_level--;
438 xen_arch_intr_execute_handlers(isrc, trap_frame);
439 curthread->td_intr_nesting_level++;
440
441 /*
442 * If this is the final port processed,
443 * we'll pick up here+1 next time.
444 */
445 pc->last_processed_l1i = l1i;
446 pc->last_processed_l2i = l2i;
447
448 } while (l2i != LONG_BIT - 1);
449
450 l2 = xen_intr_active_ports(pc, l1i);
451 if (l2 == 0) {
452 /*
453 * We handled all ports, so we can clear the
454 * selector bit.
455 */
456 l1 &= ~(1UL << l1i);
457 }
458 }
459
460 return (FILTER_HANDLED);
461 }
462
463 static int
xen_intr_init(void * dummy __unused)464 xen_intr_init(void *dummy __unused)
465 {
466 shared_info_t *s = HYPERVISOR_shared_info;
467 struct xen_intr_pcpu_data *pcpu;
468 int i;
469
470 if (!xen_domain())
471 return (0);
472
473 _Static_assert(is_valid_evtchn(0),
474 "is_valid_evtchn(0) fails (unused by Xen, but valid by interface");
475 _Static_assert(is_valid_evtchn(NR_EVENT_CHANNELS - 1),
476 "is_valid_evtchn(max) fails (is a valid channel)");
477 _Static_assert(!is_valid_evtchn(NR_EVENT_CHANNELS),
478 "is_valid_evtchn(>max) fails (NOT a valid channel)");
479 _Static_assert(!is_valid_evtchn(~(evtchn_port_t)0),
480 "is_valid_evtchn(maxint) fails (overflow?)");
481 _Static_assert(!is_valid_evtchn(INVALID_EVTCHN),
482 "is_valid_evtchn(INVALID_EVTCHN) fails (must be invalid!)");
483 _Static_assert(!is_valid_evtchn(-1),
484 "is_valid_evtchn(-1) fails (negative are invalid)");
485
486 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF);
487
488 /*
489 * Set the per-cpu mask of CPU#0 to enable all, since by default all
490 * event channels are bound to CPU#0.
491 */
492 CPU_FOREACH(i) {
493 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
494 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
495 sizeof(pcpu->evtchn_enabled));
496 }
497
498 for (i = 0; i < nitems(s->evtchn_mask); i++)
499 atomic_store_rel_xen_ulong(&s->evtchn_mask[i], ~0);
500
501 xen_arch_intr_init();
502
503 if (bootverbose)
504 printf("Xen interrupt system initialized\n");
505
506 return (0);
507 }
508 SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL);
509
510 /*--------------------------- Common PIC Functions ---------------------------*/
511
512 static void
xen_rebind_ipi(struct xenisrc * isrc)513 xen_rebind_ipi(struct xenisrc *isrc)
514 {
515 #ifdef SMP
516 u_int cpu = isrc->xi_cpu;
517 u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu);
518 int error;
519 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
520
521 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
522 &bind_ipi);
523 if (error != 0)
524 panic("unable to rebind xen IPI: %d", error);
525
526 isrc->xi_port = bind_ipi.port;
527 #else
528 panic("Resume IPI event channel on UP");
529 #endif
530 }
531
532 static void
xen_rebind_virq(struct xenisrc * isrc)533 xen_rebind_virq(struct xenisrc *isrc)
534 {
535 u_int cpu = isrc->xi_cpu;
536 u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu);
537 int error;
538 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq,
539 .vcpu = vcpu_id };
540
541 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
542 &bind_virq);
543 if (error != 0)
544 panic("unable to rebind xen VIRQ#%u: %d", isrc->xi_virq, error);
545
546 isrc->xi_port = bind_virq.port;
547 }
548
549 static struct xenisrc *
xen_intr_rebind_isrc(struct xenisrc * isrc)550 xen_intr_rebind_isrc(struct xenisrc *isrc)
551 {
552 #ifdef SMP
553 u_int cpu = isrc->xi_cpu;
554 int error;
555 #endif
556 struct xenisrc *prev;
557
558 switch (isrc->xi_type) {
559 case EVTCHN_TYPE_IPI:
560 xen_rebind_ipi(isrc);
561 break;
562 case EVTCHN_TYPE_VIRQ:
563 xen_rebind_virq(isrc);
564 break;
565 default:
566 return (NULL);
567 }
568
569 prev = xen_intr_port_to_isrc[isrc->xi_port];
570 xen_intr_port_to_isrc[isrc->xi_port] = isrc;
571
572 #ifdef SMP
573 isrc->xi_cpu = 0;
574 error = xen_intr_assign_cpu(isrc, cpu);
575 if (error)
576 panic("%s(): unable to rebind Xen channel %u to vCPU%u: %d",
577 __func__, isrc->xi_port, cpu, error);
578 #endif
579
580 evtchn_unmask_port(isrc->xi_port);
581
582 return (prev);
583 }
584
585 /**
586 * Return this PIC to service after being suspended.
587 */
588 void
xen_intr_resume(void)589 xen_intr_resume(void)
590 {
591 shared_info_t *s = HYPERVISOR_shared_info;
592 u_int isrc_idx;
593 int i;
594
595 /* Reset the per-CPU masks */
596 CPU_FOREACH(i) {
597 struct xen_intr_pcpu_data *pcpu;
598
599 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
600 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
601 sizeof(pcpu->evtchn_enabled));
602 }
603
604 /* Mask all event channels. */
605 for (i = 0; i < nitems(s->evtchn_mask); i++)
606 atomic_store_rel_xen_ulong(&s->evtchn_mask[i], ~0);
607
608 /* Clear existing port mappings */
609 for (isrc_idx = 0; isrc_idx < NR_EVENT_CHANNELS; ++isrc_idx)
610 if (xen_intr_port_to_isrc[isrc_idx] != NULL)
611 xen_intr_port_to_isrc[isrc_idx]->xi_port =
612 INVALID_EVTCHN;
613
614 /* Remap in-use isrcs, using xen_intr_port_to_isrc as listing */
615 for (isrc_idx = 0; isrc_idx < NR_EVENT_CHANNELS; ++isrc_idx) {
616 struct xenisrc *cur = xen_intr_port_to_isrc[isrc_idx];
617
618 /* empty or entry already taken care of */
619 if (cur == NULL || cur->xi_port == isrc_idx)
620 continue;
621
622 xen_intr_port_to_isrc[isrc_idx] = NULL;
623
624 do {
625 KASSERT(!is_valid_evtchn(cur->xi_port),
626 ("%s(): Multiple channels on single intr?",
627 __func__));
628
629 cur = xen_intr_rebind_isrc(cur);
630 } while (cur != NULL);
631 }
632 }
633
634 /**
635 * Disable a Xen interrupt source.
636 *
637 * \param isrc The interrupt source to disable.
638 */
639 void
xen_intr_disable_intr(struct xenisrc * isrc)640 xen_intr_disable_intr(struct xenisrc *isrc)
641 {
642
643 if (__predict_true(is_valid_evtchn(isrc->xi_port)))
644 evtchn_mask_port(isrc->xi_port);
645 }
646
647 /**
648 * Configure CPU affinity for interrupt source event delivery.
649 *
650 * \param isrc The interrupt source to configure.
651 * \param to_cpu The id of the CPU for handling future events.
652 *
653 * \returns 0 if successful, otherwise an errno.
654 */
655 int
xen_intr_assign_cpu(struct xenisrc * isrc,u_int to_cpu)656 xen_intr_assign_cpu(struct xenisrc *isrc, u_int to_cpu)
657 {
658 #ifdef SMP
659 struct evtchn_bind_vcpu bind_vcpu;
660 u_int vcpu_id = XEN_CPUID_TO_VCPUID(to_cpu);
661 int error, masked;
662
663 if (!xen_has_percpu_evtchn())
664 return (EOPNOTSUPP);
665
666 mtx_lock(&xen_intr_isrc_lock);
667 if (!is_valid_evtchn(isrc->xi_port)) {
668 mtx_unlock(&xen_intr_isrc_lock);
669 return (EINVAL);
670 }
671
672 /*
673 * Mask the event channel while binding it to prevent interrupt
674 * delivery with an inconsistent state in isrc->xi_cpu.
675 */
676 masked = evtchn_test_and_set_mask(isrc->xi_port);
677 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) ||
678 (isrc->xi_type == EVTCHN_TYPE_IPI)) {
679 /*
680 * Virtual IRQs are associated with a cpu by
681 * the Hypervisor at evtchn_bind_virq time, so
682 * all we need to do is update the per-CPU masks.
683 */
684 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
685 isrc->xi_cpu = to_cpu;
686 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
687 goto out;
688 }
689
690 bind_vcpu.port = isrc->xi_port;
691 bind_vcpu.vcpu = vcpu_id;
692
693 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu);
694 if (isrc->xi_cpu != to_cpu) {
695 if (error == 0) {
696 /* Commit to new binding by removing the old one. */
697 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
698 isrc->xi_cpu = to_cpu;
699 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
700 }
701 }
702
703 out:
704 if (masked == 0)
705 evtchn_unmask_port(isrc->xi_port);
706 mtx_unlock(&xen_intr_isrc_lock);
707 return (0);
708 #else
709 return (EOPNOTSUPP);
710 #endif
711 }
712
713 /*------------------- Virtual Interrupt Source PIC Functions -----------------*/
714 /*
715 * Mask a level triggered interrupt source.
716 *
717 * \param isrc The interrupt source to mask (if necessary).
718 */
719 void
xen_intr_disable_source(struct xenisrc * isrc)720 xen_intr_disable_source(struct xenisrc *isrc)
721 {
722
723 /*
724 * NB: checking if the event channel is already masked is
725 * needed because the event channel user-space device
726 * masks event channels on its filter as part of its
727 * normal operation, and those shouldn't be automatically
728 * unmasked by the generic interrupt code. The event channel
729 * device will unmask them when needed.
730 */
731 if (__predict_true(is_valid_evtchn(isrc->xi_port)))
732 isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port);
733 }
734
735 /*
736 * Unmask a level triggered interrupt source.
737 *
738 * \param isrc The interrupt source to unmask (if necessary).
739 */
740 void
xen_intr_enable_source(struct xenisrc * isrc)741 xen_intr_enable_source(struct xenisrc *isrc)
742 {
743
744 if (isrc->xi_masked == 0)
745 evtchn_unmask_port(isrc->xi_port);
746 }
747
748 /*
749 * Enable and unmask the interrupt source.
750 *
751 * \param isrc The interrupt source to enable.
752 */
753 void
xen_intr_enable_intr(struct xenisrc * isrc)754 xen_intr_enable_intr(struct xenisrc *isrc)
755 {
756
757 evtchn_unmask_port(isrc->xi_port);
758 }
759
760 /*--------------------------- Public Functions -------------------------------*/
761 /*------- API comments for these methods can be found in xen/xenintr.h -------*/
762 int
xen_intr_bind_local_port(device_t dev,evtchn_port_t local_port,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)763 xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port,
764 driver_filter_t filter, driver_intr_t handler, void *arg,
765 enum intr_type flags, xen_intr_handle_t *port_handlep)
766 {
767 struct xenisrc *isrc;
768 int error;
769
770 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT,
771 device_get_nameunit(dev), filter, handler, arg, flags,
772 port_handlep);
773 if (error != 0)
774 return (error);
775
776 /*
777 * The Event Channel API didn't open this port, so it is not
778 * responsible for closing it automatically on unbind.
779 */
780 isrc->xi_close = 0;
781 return (0);
782 }
783
784 int
xen_intr_alloc_and_bind_local_port(device_t dev,u_int remote_domain,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)785 xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain,
786 driver_filter_t filter, driver_intr_t handler, void *arg,
787 enum intr_type flags, xen_intr_handle_t *port_handlep)
788 {
789 struct xenisrc *isrc;
790 struct evtchn_alloc_unbound alloc_unbound;
791 int error;
792
793 alloc_unbound.dom = DOMID_SELF;
794 alloc_unbound.remote_dom = remote_domain;
795 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
796 &alloc_unbound);
797 if (error != 0) {
798 /*
799 * XXX Trap Hypercall error code Linuxisms in
800 * the HYPERCALL layer.
801 */
802 return (-error);
803 }
804
805 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT,
806 device_get_nameunit(dev), filter, handler, arg, flags,
807 port_handlep);
808 if (error != 0) {
809 evtchn_close_t close = { .port = alloc_unbound.port };
810 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
811 panic("EVTCHNOP_close failed");
812 return (error);
813 }
814
815 isrc->xi_close = 1;
816 return (0);
817 }
818
819 int
xen_intr_bind_remote_port(device_t dev,u_int remote_domain,u_int remote_port,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)820 xen_intr_bind_remote_port(device_t dev, u_int remote_domain,
821 u_int remote_port, driver_filter_t filter, driver_intr_t handler,
822 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep)
823 {
824 struct xenisrc *isrc;
825 struct evtchn_bind_interdomain bind_interdomain;
826 int error;
827
828 bind_interdomain.remote_dom = remote_domain;
829 bind_interdomain.remote_port = remote_port;
830 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
831 &bind_interdomain);
832 if (error != 0) {
833 /*
834 * XXX Trap Hypercall error code Linuxisms in
835 * the HYPERCALL layer.
836 */
837 return (-error);
838 }
839
840 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port,
841 EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg,
842 flags, port_handlep);
843 if (error) {
844 evtchn_close_t close = { .port = bind_interdomain.local_port };
845 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
846 panic("EVTCHNOP_close failed");
847 return (error);
848 }
849
850 /*
851 * The Event Channel API opened this port, so it is
852 * responsible for closing it automatically on unbind.
853 */
854 isrc->xi_close = 1;
855 return (0);
856 }
857
858 int
xen_intr_bind_virq(device_t dev,u_int virq,u_int cpu,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)859 xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu,
860 driver_filter_t filter, driver_intr_t handler, void *arg,
861 enum intr_type flags, xen_intr_handle_t *port_handlep)
862 {
863 u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu);
864 struct xenisrc *isrc;
865 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id };
866 int error;
867
868 isrc = NULL;
869 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
870 if (error != 0) {
871 /*
872 * XXX Trap Hypercall error code Linuxisms in
873 * the HYPERCALL layer.
874 */
875 return (-error);
876 }
877
878 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ,
879 device_get_nameunit(dev), filter, handler, arg, flags,
880 port_handlep);
881
882 #ifdef SMP
883 if (error == 0)
884 error = xen_arch_intr_event_bind(isrc, cpu);
885 #endif
886
887 if (error != 0) {
888 evtchn_close_t close = { .port = bind_virq.port };
889
890 xen_intr_unbind(port_handlep);
891 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
892 panic("EVTCHNOP_close failed");
893 return (error);
894 }
895
896 #ifdef SMP
897 if (isrc->xi_cpu != cpu) {
898 /*
899 * Too early in the boot process for the generic interrupt
900 * code to perform the binding. Update our event channel
901 * masks manually so events can't fire on the wrong cpu
902 * during AP startup.
903 */
904 xen_intr_assign_cpu(isrc, cpu);
905 }
906 #endif
907
908 /*
909 * The Event Channel API opened this port, so it is
910 * responsible for closing it automatically on unbind.
911 */
912 isrc->xi_close = 1;
913 isrc->xi_virq = virq;
914
915 return (0);
916 }
917
918 int
xen_intr_alloc_and_bind_ipi(u_int cpu,driver_filter_t filter,enum intr_type flags,xen_intr_handle_t * port_handlep)919 xen_intr_alloc_and_bind_ipi(u_int cpu, driver_filter_t filter,
920 enum intr_type flags, xen_intr_handle_t *port_handlep)
921 {
922 #ifdef SMP
923 u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu);
924 struct xenisrc *isrc;
925 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
926 /* Same size as the one used by intr_handler->ih_name. */
927 char name[MAXCOMLEN + 1];
928 int error;
929
930 isrc = NULL;
931 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
932 if (error != 0) {
933 /*
934 * XXX Trap Hypercall error code Linuxisms in
935 * the HYPERCALL layer.
936 */
937 return (-error);
938 }
939
940 snprintf(name, sizeof(name), "cpu%u", cpu);
941
942 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI,
943 name, filter, NULL, NULL, flags, port_handlep);
944 if (error != 0) {
945 evtchn_close_t close = { .port = bind_ipi.port };
946
947 xen_intr_unbind(port_handlep);
948 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
949 panic("EVTCHNOP_close failed");
950 return (error);
951 }
952
953 if (isrc->xi_cpu != cpu) {
954 /*
955 * Too early in the boot process for the generic interrupt
956 * code to perform the binding. Update our event channel
957 * masks manually so events can't fire on the wrong cpu
958 * during AP startup.
959 */
960 xen_intr_assign_cpu(isrc, cpu);
961 }
962
963 /*
964 * The Event Channel API opened this port, so it is
965 * responsible for closing it automatically on unbind.
966 */
967 isrc->xi_close = 1;
968 return (0);
969 #else
970 return (EOPNOTSUPP);
971 #endif
972 }
973
974 int
xen_intr_describe(xen_intr_handle_t port_handle,const char * fmt,...)975 xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...)
976 {
977 char descr[MAXCOMLEN + 1];
978 struct xenisrc *isrc;
979 va_list ap;
980
981 isrc = xen_intr_isrc_from_handle(port_handle);
982 if (isrc == NULL)
983 return (EINVAL);
984
985 va_start(ap, fmt);
986 vsnprintf(descr, sizeof(descr), fmt, ap);
987 va_end(ap);
988 return (xen_arch_intr_describe(isrc, isrc->xi_cookie, descr));
989 }
990
991 void
xen_intr_unbind(xen_intr_handle_t * port_handlep)992 xen_intr_unbind(xen_intr_handle_t *port_handlep)
993 {
994 struct xenisrc *isrc;
995
996 KASSERT(port_handlep != NULL,
997 ("NULL xen_intr_handle_t passed to %s", __func__));
998
999 isrc = xen_intr_isrc_from_handle(*port_handlep);
1000 *port_handlep = NULL;
1001 if (isrc == NULL)
1002 return;
1003
1004 mtx_lock(&xen_intr_isrc_lock);
1005 if (refcount_release(&isrc->xi_refcount) == 0) {
1006 mtx_unlock(&xen_intr_isrc_lock);
1007 return;
1008 }
1009 mtx_unlock(&xen_intr_isrc_lock);
1010
1011 if (isrc->xi_cookie != NULL)
1012 xen_arch_intr_remove_handler(isrc, isrc->xi_cookie);
1013 xen_intr_release_isrc(isrc);
1014 }
1015
1016 void
xen_intr_signal(xen_intr_handle_t handle)1017 xen_intr_signal(xen_intr_handle_t handle)
1018 {
1019 struct xenisrc *isrc;
1020
1021 isrc = xen_intr_isrc_from_handle(handle);
1022 if (isrc != NULL) {
1023 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT ||
1024 isrc->xi_type == EVTCHN_TYPE_IPI,
1025 ("evtchn_signal on something other than a local port"));
1026 struct evtchn_send send = { .port = isrc->xi_port };
1027 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
1028 }
1029 }
1030
1031 evtchn_port_t
xen_intr_port(xen_intr_handle_t handle)1032 xen_intr_port(xen_intr_handle_t handle)
1033 {
1034 struct xenisrc *isrc;
1035
1036 isrc = xen_intr_isrc_from_handle(handle);
1037 if (isrc == NULL)
1038 return (0);
1039
1040 return (isrc->xi_port);
1041 }
1042
1043 int
xen_intr_add_handler(const char * name,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t handle)1044 xen_intr_add_handler(const char *name, driver_filter_t filter,
1045 driver_intr_t handler, void *arg, enum intr_type flags,
1046 xen_intr_handle_t handle)
1047 {
1048 struct xenisrc *isrc;
1049 int error;
1050
1051 isrc = xen_intr_isrc_from_handle(handle);
1052 if (isrc == NULL || isrc->xi_cookie != NULL)
1053 return (EINVAL);
1054
1055 error = xen_arch_intr_add_handler(name, filter, handler, arg,
1056 flags | INTR_EXCL, isrc, &isrc->xi_cookie);
1057 if (error != 0)
1058 printf("%s: %s: add handler failed: %d\n", name, __func__,
1059 error);
1060
1061 return (error);
1062 }
1063
1064 int
xen_intr_get_evtchn_from_port(evtchn_port_t port,xen_intr_handle_t * handlep)1065 xen_intr_get_evtchn_from_port(evtchn_port_t port, xen_intr_handle_t *handlep)
1066 {
1067
1068 if (!is_valid_evtchn(port))
1069 return (EINVAL);
1070
1071 if (handlep == NULL) {
1072 return (EINVAL);
1073 }
1074
1075 mtx_lock(&xen_intr_isrc_lock);
1076 if (xen_intr_port_to_isrc[port] == NULL) {
1077 mtx_unlock(&xen_intr_isrc_lock);
1078 return (EINVAL);
1079 }
1080 refcount_acquire(&xen_intr_port_to_isrc[port]->xi_refcount);
1081 mtx_unlock(&xen_intr_isrc_lock);
1082
1083 /* Assign the opaque handler */
1084 *handlep = xen_intr_handle_from_isrc(xen_intr_port_to_isrc[port]);
1085
1086 return (0);
1087 }
1088
1089 #ifdef DDB
1090 static const char *
xen_intr_print_type(enum evtchn_type type)1091 xen_intr_print_type(enum evtchn_type type)
1092 {
1093 static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = {
1094 [EVTCHN_TYPE_UNBOUND] = "UNBOUND",
1095 [EVTCHN_TYPE_VIRQ] = "VIRQ",
1096 [EVTCHN_TYPE_IPI] = "IPI",
1097 [EVTCHN_TYPE_PORT] = "PORT",
1098 };
1099
1100 if (type >= EVTCHN_TYPE_COUNT)
1101 return ("UNKNOWN");
1102
1103 return (evtchn_type_to_string[type]);
1104 }
1105
1106 static void
xen_intr_dump_port(struct xenisrc * isrc)1107 xen_intr_dump_port(struct xenisrc *isrc)
1108 {
1109 struct xen_intr_pcpu_data *pcpu;
1110 shared_info_t *s = HYPERVISOR_shared_info;
1111 u_int i;
1112
1113 db_printf("Port %d Type: %s\n",
1114 isrc->xi_port, xen_intr_print_type(isrc->xi_type));
1115 if (isrc->xi_type == EVTCHN_TYPE_VIRQ)
1116 db_printf("\tVirq: %u\n", isrc->xi_virq);
1117
1118 db_printf("\tMasked: %d Pending: %d\n",
1119 !!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]),
1120 !!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0]));
1121
1122 db_printf("\tPer-CPU Masks: ");
1123 CPU_FOREACH(i) {
1124 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
1125 db_printf("cpu#%u: %d ", i,
1126 !!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled));
1127 }
1128 db_printf("\n");
1129 }
1130
DB_SHOW_COMMAND(xen_evtchn,db_show_xen_evtchn)1131 DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn)
1132 {
1133 u_int i;
1134
1135 if (!xen_domain()) {
1136 db_printf("Only available on Xen guests\n");
1137 return;
1138 }
1139
1140 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
1141 struct xenisrc *isrc;
1142
1143 isrc = xen_intr_port_to_isrc[i];
1144 if (isrc == NULL)
1145 continue;
1146
1147 xen_intr_dump_port(isrc);
1148 }
1149 }
1150 #endif /* DDB */
1151