1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include "opt_bhyve_snapshot.h"
32
33 #include <sys/param.h>
34 #include <sys/queue.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40
41 #include <x86/apicreg.h>
42 #include <machine/vmm.h>
43 #include <machine/vmm_snapshot.h>
44
45 #include "vmm_ktr.h"
46 #include "vmm_lapic.h"
47 #include "vlapic.h"
48 #include "vioapic.h"
49
50 #define IOREGSEL 0x00
51 #define IOWIN 0x10
52
53 #define REDIR_ENTRIES 32
54 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
55
56 struct vioapic {
57 struct vm *vm;
58 struct mtx mtx;
59 uint32_t id;
60 uint32_t ioregsel;
61 struct {
62 uint64_t reg;
63 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */
64 } rtbl[REDIR_ENTRIES];
65 };
66
67 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx))
68 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx))
69 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx))
70
71 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
72
73 #define VIOAPIC_CTR1(vioapic, fmt, a1) \
74 VM_CTR1((vioapic)->vm, fmt, a1)
75
76 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \
77 VM_CTR2((vioapic)->vm, fmt, a1, a2)
78
79 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \
80 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
81
82 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \
83 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
84
85 #ifdef KTR
86 static const char *
pinstate_str(bool asserted)87 pinstate_str(bool asserted)
88 {
89
90 if (asserted)
91 return ("asserted");
92 else
93 return ("deasserted");
94 }
95 #endif
96
97 static void
vioapic_send_intr(struct vioapic * vioapic,int pin)98 vioapic_send_intr(struct vioapic *vioapic, int pin)
99 {
100 int vector, delmode;
101 uint32_t low, high, dest;
102 bool level, phys;
103
104 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
105 ("vioapic_set_pinstate: invalid pin number %d", pin));
106
107 KASSERT(VIOAPIC_LOCKED(vioapic),
108 ("vioapic_set_pinstate: vioapic is not locked"));
109
110 low = vioapic->rtbl[pin].reg;
111 high = vioapic->rtbl[pin].reg >> 32;
112
113 if ((low & IOART_INTMASK) == IOART_INTMSET) {
114 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
115 return;
116 }
117
118 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
119 delmode = low & IOART_DELMOD;
120 level = low & IOART_TRGRLVL ? true : false;
121 if (level) {
122 if ((low & IOART_REM_IRR) != 0) {
123 VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending",
124 pin);
125 return;
126 }
127 vioapic->rtbl[pin].reg |= IOART_REM_IRR;
128 }
129
130 vector = low & IOART_INTVEC;
131 dest = high >> APIC_ID_SHIFT;
132 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
133 }
134
135 static void
vioapic_set_pinstate(struct vioapic * vioapic,int pin,bool newstate)136 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
137 {
138 int oldcnt, newcnt;
139 bool needintr;
140
141 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
142 ("vioapic_set_pinstate: invalid pin number %d", pin));
143
144 KASSERT(VIOAPIC_LOCKED(vioapic),
145 ("vioapic_set_pinstate: vioapic is not locked"));
146
147 oldcnt = vioapic->rtbl[pin].acnt;
148 if (newstate)
149 vioapic->rtbl[pin].acnt++;
150 else
151 vioapic->rtbl[pin].acnt--;
152 newcnt = vioapic->rtbl[pin].acnt;
153
154 if (newcnt < 0) {
155 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
156 pin, newcnt);
157 }
158
159 needintr = false;
160 if (oldcnt == 0 && newcnt == 1) {
161 needintr = true;
162 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
163 } else if (oldcnt == 1 && newcnt == 0) {
164 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
165 } else {
166 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
167 pin, pinstate_str(newstate), newcnt);
168 }
169
170 if (needintr)
171 vioapic_send_intr(vioapic, pin);
172 }
173
174 enum irqstate {
175 IRQSTATE_ASSERT,
176 IRQSTATE_DEASSERT,
177 IRQSTATE_PULSE
178 };
179
180 static int
vioapic_set_irqstate(struct vm * vm,int irq,enum irqstate irqstate)181 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
182 {
183 struct vioapic *vioapic;
184
185 if (irq < 0 || irq >= REDIR_ENTRIES)
186 return (EINVAL);
187
188 vioapic = vm_ioapic(vm);
189
190 VIOAPIC_LOCK(vioapic);
191 switch (irqstate) {
192 case IRQSTATE_ASSERT:
193 vioapic_set_pinstate(vioapic, irq, true);
194 break;
195 case IRQSTATE_DEASSERT:
196 vioapic_set_pinstate(vioapic, irq, false);
197 break;
198 case IRQSTATE_PULSE:
199 vioapic_set_pinstate(vioapic, irq, true);
200 vioapic_set_pinstate(vioapic, irq, false);
201 break;
202 default:
203 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
204 }
205 VIOAPIC_UNLOCK(vioapic);
206
207 return (0);
208 }
209
210 int
vioapic_assert_irq(struct vm * vm,int irq)211 vioapic_assert_irq(struct vm *vm, int irq)
212 {
213
214 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
215 }
216
217 int
vioapic_deassert_irq(struct vm * vm,int irq)218 vioapic_deassert_irq(struct vm *vm, int irq)
219 {
220
221 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
222 }
223
224 int
vioapic_pulse_irq(struct vm * vm,int irq)225 vioapic_pulse_irq(struct vm *vm, int irq)
226 {
227
228 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
229 }
230
231 /*
232 * Reset the vlapic's trigger-mode register to reflect the ioapic pin
233 * configuration.
234 */
235 static void
vioapic_update_tmr(struct vcpu * vcpu,void * arg)236 vioapic_update_tmr(struct vcpu *vcpu, void *arg)
237 {
238 struct vioapic *vioapic;
239 struct vlapic *vlapic;
240 uint32_t low, high, dest;
241 int delmode, pin, vector;
242 bool level, phys;
243
244 vlapic = vm_lapic(vcpu);
245 vioapic = vm_ioapic(vcpu_vm(vcpu));
246
247 VIOAPIC_LOCK(vioapic);
248 /*
249 * Reset all vectors to be edge-triggered.
250 */
251 vlapic_reset_tmr(vlapic);
252 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
253 low = vioapic->rtbl[pin].reg;
254 high = vioapic->rtbl[pin].reg >> 32;
255
256 level = low & IOART_TRGRLVL ? true : false;
257 if (!level)
258 continue;
259
260 /*
261 * For a level-triggered 'pin' let the vlapic figure out if
262 * an assertion on this 'pin' would result in an interrupt
263 * being delivered to it. If yes, then it will modify the
264 * TMR bit associated with this vector to level-triggered.
265 */
266 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
267 delmode = low & IOART_DELMOD;
268 vector = low & IOART_INTVEC;
269 dest = high >> APIC_ID_SHIFT;
270 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
271 }
272 VIOAPIC_UNLOCK(vioapic);
273 }
274
275 static uint32_t
vioapic_read(struct vioapic * vioapic,struct vcpu * vcpu,uint32_t addr)276 vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr)
277 {
278 int regnum, pin, rshift;
279
280 regnum = addr & 0xff;
281 switch (regnum) {
282 case IOAPIC_ID:
283 return (vioapic->id);
284 break;
285 case IOAPIC_VER:
286 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
287 break;
288 case IOAPIC_ARB:
289 return (vioapic->id);
290 break;
291 default:
292 break;
293 }
294
295 /* redirection table entries */
296 if (regnum >= IOAPIC_REDTBL &&
297 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
298 pin = (regnum - IOAPIC_REDTBL) / 2;
299 if ((regnum - IOAPIC_REDTBL) % 2)
300 rshift = 32;
301 else
302 rshift = 0;
303
304 return (vioapic->rtbl[pin].reg >> rshift);
305 }
306
307 return (0);
308 }
309
310 static void
vioapic_write(struct vioapic * vioapic,struct vcpu * vcpu,uint32_t addr,uint32_t data)311 vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr,
312 uint32_t data)
313 {
314 uint64_t data64, mask64;
315 uint64_t last, changed;
316 int regnum, pin, lshift;
317 cpuset_t allvcpus;
318
319 regnum = addr & 0xff;
320 switch (regnum) {
321 case IOAPIC_ID:
322 vioapic->id = data & APIC_ID_MASK;
323 break;
324 case IOAPIC_VER:
325 case IOAPIC_ARB:
326 /* readonly */
327 break;
328 default:
329 break;
330 }
331
332 /* redirection table entries */
333 if (regnum >= IOAPIC_REDTBL &&
334 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
335 pin = (regnum - IOAPIC_REDTBL) / 2;
336 if ((regnum - IOAPIC_REDTBL) % 2)
337 lshift = 32;
338 else
339 lshift = 0;
340
341 last = vioapic->rtbl[pin].reg;
342
343 data64 = (uint64_t)data << lshift;
344 mask64 = (uint64_t)0xffffffff << lshift;
345 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
346 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
347
348 /*
349 * Switching from level to edge triggering will clear the IRR
350 * bit. This is what FreeBSD will do in order to EOI an
351 * interrupt when the IO-APIC doesn't support targeted EOI (see
352 * _ioapic_eoi_source).
353 */
354 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
355 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
356 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
357
358 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
359 pin, vioapic->rtbl[pin].reg);
360
361 /*
362 * If any fields in the redirection table entry (except mask
363 * or polarity) have changed then rendezvous all the vcpus
364 * to update their vlapic trigger-mode registers.
365 */
366 changed = last ^ vioapic->rtbl[pin].reg;
367 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
368 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
369 "vlapic trigger-mode register", pin);
370 VIOAPIC_UNLOCK(vioapic);
371 allvcpus = vm_active_cpus(vioapic->vm);
372 (void)vm_smp_rendezvous(vcpu, allvcpus,
373 vioapic_update_tmr, NULL);
374 VIOAPIC_LOCK(vioapic);
375 }
376
377 /*
378 * Generate an interrupt if the following conditions are met:
379 * - pin trigger mode is level
380 * - pin level is asserted
381 */
382 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
383 (vioapic->rtbl[pin].acnt > 0)) {
384 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
385 "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
386 vioapic_send_intr(vioapic, pin);
387 }
388 }
389 }
390
391 static int
vioapic_mmio_rw(struct vioapic * vioapic,struct vcpu * vcpu,uint64_t gpa,uint64_t * data,int size,bool doread)392 vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa,
393 uint64_t *data, int size, bool doread)
394 {
395 uint64_t offset;
396
397 offset = gpa - VIOAPIC_BASE;
398
399 /*
400 * The IOAPIC specification allows 32-bit wide accesses to the
401 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
402 */
403 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
404 if (doread)
405 *data = 0;
406 return (0);
407 }
408
409 VIOAPIC_LOCK(vioapic);
410 if (offset == IOREGSEL) {
411 if (doread)
412 *data = vioapic->ioregsel;
413 else
414 vioapic->ioregsel = *data;
415 } else {
416 if (doread) {
417 *data = vioapic_read(vioapic, vcpu,
418 vioapic->ioregsel);
419 } else {
420 vioapic_write(vioapic, vcpu, vioapic->ioregsel,
421 *data);
422 }
423 }
424 VIOAPIC_UNLOCK(vioapic);
425
426 return (0);
427 }
428
429 int
vioapic_mmio_read(struct vcpu * vcpu,uint64_t gpa,uint64_t * rval,int size,void * arg)430 vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval,
431 int size, void *arg)
432 {
433 int error;
434 struct vioapic *vioapic;
435
436 vioapic = vm_ioapic(vcpu_vm(vcpu));
437 error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true);
438 return (error);
439 }
440
441 int
vioapic_mmio_write(struct vcpu * vcpu,uint64_t gpa,uint64_t wval,int size,void * arg)442 vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval,
443 int size, void *arg)
444 {
445 int error;
446 struct vioapic *vioapic;
447
448 vioapic = vm_ioapic(vcpu_vm(vcpu));
449 error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false);
450 return (error);
451 }
452
453 void
vioapic_process_eoi(struct vm * vm,int vector)454 vioapic_process_eoi(struct vm *vm, int vector)
455 {
456 struct vioapic *vioapic;
457 int pin;
458
459 KASSERT(vector >= 0 && vector < 256,
460 ("vioapic_process_eoi: invalid vector %d", vector));
461
462 vioapic = vm_ioapic(vm);
463 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
464
465 /*
466 * XXX keep track of the pins associated with this vector instead
467 * of iterating on every single pin each time.
468 */
469 VIOAPIC_LOCK(vioapic);
470 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
471 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
472 continue;
473 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
474 continue;
475 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
476 if (vioapic->rtbl[pin].acnt > 0) {
477 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
478 "acnt %d", pin, vioapic->rtbl[pin].acnt);
479 vioapic_send_intr(vioapic, pin);
480 }
481 }
482 VIOAPIC_UNLOCK(vioapic);
483 }
484
485 struct vioapic *
vioapic_init(struct vm * vm)486 vioapic_init(struct vm *vm)
487 {
488 int i;
489 struct vioapic *vioapic;
490
491 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
492
493 vioapic->vm = vm;
494 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
495
496 /* Initialize all redirection entries to mask all interrupts */
497 for (i = 0; i < REDIR_ENTRIES; i++)
498 vioapic->rtbl[i].reg = 0x0001000000010000UL;
499
500 return (vioapic);
501 }
502
503 void
vioapic_cleanup(struct vioapic * vioapic)504 vioapic_cleanup(struct vioapic *vioapic)
505 {
506
507 mtx_destroy(&vioapic->mtx);
508 free(vioapic, M_VIOAPIC);
509 }
510
511 int
vioapic_pincount(struct vm * vm)512 vioapic_pincount(struct vm *vm)
513 {
514
515 return (REDIR_ENTRIES);
516 }
517
518 #ifdef BHYVE_SNAPSHOT
519 int
vioapic_snapshot(struct vioapic * vioapic,struct vm_snapshot_meta * meta)520 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta)
521 {
522 int ret;
523 int i;
524
525 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done);
526
527 for (i = 0; i < nitems(vioapic->rtbl); i++) {
528 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done);
529 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done);
530 }
531
532 done:
533 return (ret);
534 }
535 #endif
536