1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_bhyve_snapshot.h" 36 37 #include <sys/param.h> 38 #include <sys/queue.h> 39 #include <sys/lock.h> 40 #include <sys/mutex.h> 41 #include <sys/systm.h> 42 #include <sys/kernel.h> 43 #include <sys/malloc.h> 44 45 #include <x86/apicreg.h> 46 #include <machine/vmm.h> 47 #include <machine/vmm_snapshot.h> 48 49 #include "vmm_ktr.h" 50 #include "vmm_lapic.h" 51 #include "vlapic.h" 52 #include "vioapic.h" 53 54 #define IOREGSEL 0x00 55 #define IOWIN 0x10 56 57 #define REDIR_ENTRIES 32 58 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 59 60 struct vioapic { 61 struct vm *vm; 62 struct mtx mtx; 63 uint32_t id; 64 uint32_t ioregsel; 65 struct { 66 uint64_t reg; 67 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ 68 } rtbl[REDIR_ENTRIES]; 69 }; 70 71 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) 72 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) 73 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) 74 75 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); 76 77 #define VIOAPIC_CTR1(vioapic, fmt, a1) \ 78 VM_CTR1((vioapic)->vm, fmt, a1) 79 80 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ 81 VM_CTR2((vioapic)->vm, fmt, a1, a2) 82 83 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \ 84 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3) 85 86 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \ 87 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4) 88 89 #ifdef KTR 90 static const char * 91 pinstate_str(bool asserted) 92 { 93 94 if (asserted) 95 return ("asserted"); 96 else 97 return ("deasserted"); 98 } 99 #endif 100 101 static void 102 vioapic_send_intr(struct vioapic *vioapic, int pin) 103 { 104 int vector, delmode; 105 uint32_t low, high, dest; 106 bool level, phys; 107 108 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 109 ("vioapic_set_pinstate: invalid pin number %d", pin)); 110 111 KASSERT(VIOAPIC_LOCKED(vioapic), 112 ("vioapic_set_pinstate: vioapic is not locked")); 113 114 low = vioapic->rtbl[pin].reg; 115 high = vioapic->rtbl[pin].reg >> 32; 116 117 if ((low & IOART_INTMASK) == IOART_INTMSET) { 118 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin); 119 return; 120 } 121 122 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 123 delmode = low & IOART_DELMOD; 124 level = low & IOART_TRGRLVL ? true : false; 125 if (level) 126 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 127 128 vector = low & IOART_INTVEC; 129 dest = high >> APIC_ID_SHIFT; 130 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 131 } 132 133 static void 134 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 135 { 136 int oldcnt, newcnt; 137 bool needintr; 138 139 KASSERT(pin >= 0 && pin < REDIR_ENTRIES, 140 ("vioapic_set_pinstate: invalid pin number %d", pin)); 141 142 KASSERT(VIOAPIC_LOCKED(vioapic), 143 ("vioapic_set_pinstate: vioapic is not locked")); 144 145 oldcnt = vioapic->rtbl[pin].acnt; 146 if (newstate) 147 vioapic->rtbl[pin].acnt++; 148 else 149 vioapic->rtbl[pin].acnt--; 150 newcnt = vioapic->rtbl[pin].acnt; 151 152 if (newcnt < 0) { 153 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d", 154 pin, newcnt); 155 } 156 157 needintr = false; 158 if (oldcnt == 0 && newcnt == 1) { 159 needintr = true; 160 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin); 161 } else if (oldcnt == 1 && newcnt == 0) { 162 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin); 163 } else { 164 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d", 165 pin, pinstate_str(newstate), newcnt); 166 } 167 168 if (needintr) 169 vioapic_send_intr(vioapic, pin); 170 } 171 172 enum irqstate { 173 IRQSTATE_ASSERT, 174 IRQSTATE_DEASSERT, 175 IRQSTATE_PULSE 176 }; 177 178 static int 179 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 180 { 181 struct vioapic *vioapic; 182 183 if (irq < 0 || irq >= REDIR_ENTRIES) 184 return (EINVAL); 185 186 vioapic = vm_ioapic(vm); 187 188 VIOAPIC_LOCK(vioapic); 189 switch (irqstate) { 190 case IRQSTATE_ASSERT: 191 vioapic_set_pinstate(vioapic, irq, true); 192 break; 193 case IRQSTATE_DEASSERT: 194 vioapic_set_pinstate(vioapic, irq, false); 195 break; 196 case IRQSTATE_PULSE: 197 vioapic_set_pinstate(vioapic, irq, true); 198 vioapic_set_pinstate(vioapic, irq, false); 199 break; 200 default: 201 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 202 } 203 VIOAPIC_UNLOCK(vioapic); 204 205 return (0); 206 } 207 208 int 209 vioapic_assert_irq(struct vm *vm, int irq) 210 { 211 212 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 213 } 214 215 int 216 vioapic_deassert_irq(struct vm *vm, int irq) 217 { 218 219 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 220 } 221 222 int 223 vioapic_pulse_irq(struct vm *vm, int irq) 224 { 225 226 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 227 } 228 229 /* 230 * Reset the vlapic's trigger-mode register to reflect the ioapic pin 231 * configuration. 232 */ 233 static void 234 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg) 235 { 236 struct vioapic *vioapic; 237 struct vlapic *vlapic; 238 uint32_t low, high, dest; 239 int delmode, pin, vector; 240 bool level, phys; 241 242 vlapic = vm_lapic(vm, vcpuid); 243 vioapic = vm_ioapic(vm); 244 245 VIOAPIC_LOCK(vioapic); 246 /* 247 * Reset all vectors to be edge-triggered. 248 */ 249 vlapic_reset_tmr(vlapic); 250 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 251 low = vioapic->rtbl[pin].reg; 252 high = vioapic->rtbl[pin].reg >> 32; 253 254 level = low & IOART_TRGRLVL ? true : false; 255 if (!level) 256 continue; 257 258 /* 259 * For a level-triggered 'pin' let the vlapic figure out if 260 * an assertion on this 'pin' would result in an interrupt 261 * being delivered to it. If yes, then it will modify the 262 * TMR bit associated with this vector to level-triggered. 263 */ 264 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 265 delmode = low & IOART_DELMOD; 266 vector = low & IOART_INTVEC; 267 dest = high >> APIC_ID_SHIFT; 268 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); 269 } 270 VIOAPIC_UNLOCK(vioapic); 271 } 272 273 static uint32_t 274 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 275 { 276 int regnum, pin, rshift; 277 278 regnum = addr & 0xff; 279 switch (regnum) { 280 case IOAPIC_ID: 281 return (vioapic->id); 282 break; 283 case IOAPIC_VER: 284 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 285 break; 286 case IOAPIC_ARB: 287 return (vioapic->id); 288 break; 289 default: 290 break; 291 } 292 293 /* redirection table entries */ 294 if (regnum >= IOAPIC_REDTBL && 295 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 296 pin = (regnum - IOAPIC_REDTBL) / 2; 297 if ((regnum - IOAPIC_REDTBL) % 2) 298 rshift = 32; 299 else 300 rshift = 0; 301 302 return (vioapic->rtbl[pin].reg >> rshift); 303 } 304 305 return (0); 306 } 307 308 static void 309 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 310 { 311 uint64_t data64, mask64; 312 uint64_t last, changed; 313 int regnum, pin, lshift; 314 cpuset_t allvcpus; 315 316 regnum = addr & 0xff; 317 switch (regnum) { 318 case IOAPIC_ID: 319 vioapic->id = data & APIC_ID_MASK; 320 break; 321 case IOAPIC_VER: 322 case IOAPIC_ARB: 323 /* readonly */ 324 break; 325 default: 326 break; 327 } 328 329 /* redirection table entries */ 330 if (regnum >= IOAPIC_REDTBL && 331 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 332 pin = (regnum - IOAPIC_REDTBL) / 2; 333 if ((regnum - IOAPIC_REDTBL) % 2) 334 lshift = 32; 335 else 336 lshift = 0; 337 338 last = vioapic->rtbl[pin].reg; 339 340 data64 = (uint64_t)data << lshift; 341 mask64 = (uint64_t)0xffffffff << lshift; 342 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 343 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 344 345 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", 346 pin, vioapic->rtbl[pin].reg); 347 348 /* 349 * If any fields in the redirection table entry (except mask 350 * or polarity) have changed then rendezvous all the vcpus 351 * to update their vlapic trigger-mode registers. 352 */ 353 changed = last ^ vioapic->rtbl[pin].reg; 354 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { 355 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " 356 "vlapic trigger-mode register", pin); 357 VIOAPIC_UNLOCK(vioapic); 358 allvcpus = vm_active_cpus(vioapic->vm); 359 (void)vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus, 360 vioapic_update_tmr, NULL); 361 VIOAPIC_LOCK(vioapic); 362 } 363 364 /* 365 * Generate an interrupt if the following conditions are met: 366 * - pin is not masked 367 * - previous interrupt has been EOIed 368 * - pin level is asserted 369 */ 370 if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR && 371 (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 && 372 (vioapic->rtbl[pin].acnt > 0)) { 373 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl " 374 "write, acnt %d", pin, vioapic->rtbl[pin].acnt); 375 vioapic_send_intr(vioapic, pin); 376 } 377 } 378 } 379 380 static int 381 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 382 uint64_t *data, int size, bool doread) 383 { 384 uint64_t offset; 385 386 offset = gpa - VIOAPIC_BASE; 387 388 /* 389 * The IOAPIC specification allows 32-bit wide accesses to the 390 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 391 */ 392 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 393 if (doread) 394 *data = 0; 395 return (0); 396 } 397 398 VIOAPIC_LOCK(vioapic); 399 if (offset == IOREGSEL) { 400 if (doread) 401 *data = vioapic->ioregsel; 402 else 403 vioapic->ioregsel = *data; 404 } else { 405 if (doread) { 406 *data = vioapic_read(vioapic, vcpuid, 407 vioapic->ioregsel); 408 } else { 409 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 410 *data); 411 } 412 } 413 VIOAPIC_UNLOCK(vioapic); 414 415 return (0); 416 } 417 418 int 419 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 420 int size, void *arg) 421 { 422 int error; 423 struct vioapic *vioapic; 424 425 vioapic = vm_ioapic(vm); 426 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 427 return (error); 428 } 429 430 int 431 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval, 432 int size, void *arg) 433 { 434 int error; 435 struct vioapic *vioapic; 436 437 vioapic = vm_ioapic(vm); 438 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 439 return (error); 440 } 441 442 void 443 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 444 { 445 struct vioapic *vioapic; 446 int pin; 447 448 KASSERT(vector >= 0 && vector < 256, 449 ("vioapic_process_eoi: invalid vector %d", vector)); 450 451 vioapic = vm_ioapic(vm); 452 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector); 453 454 /* 455 * XXX keep track of the pins associated with this vector instead 456 * of iterating on every single pin each time. 457 */ 458 VIOAPIC_LOCK(vioapic); 459 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 460 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 461 continue; 462 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 463 continue; 464 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 465 if (vioapic->rtbl[pin].acnt > 0) { 466 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, " 467 "acnt %d", pin, vioapic->rtbl[pin].acnt); 468 vioapic_send_intr(vioapic, pin); 469 } 470 } 471 VIOAPIC_UNLOCK(vioapic); 472 } 473 474 struct vioapic * 475 vioapic_init(struct vm *vm) 476 { 477 int i; 478 struct vioapic *vioapic; 479 480 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); 481 482 vioapic->vm = vm; 483 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); 484 485 /* Initialize all redirection entries to mask all interrupts */ 486 for (i = 0; i < REDIR_ENTRIES; i++) 487 vioapic->rtbl[i].reg = 0x0001000000010000UL; 488 489 return (vioapic); 490 } 491 492 void 493 vioapic_cleanup(struct vioapic *vioapic) 494 { 495 496 free(vioapic, M_VIOAPIC); 497 } 498 499 int 500 vioapic_pincount(struct vm *vm) 501 { 502 503 return (REDIR_ENTRIES); 504 } 505 506 #ifdef BHYVE_SNAPSHOT 507 int 508 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta) 509 { 510 int ret; 511 int i; 512 513 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done); 514 515 for (i = 0; i < nitems(vioapic->rtbl); i++) { 516 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done); 517 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done); 518 } 519 520 done: 521 return (ret); 522 } 523 #endif 524