1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 /* 32 * This file and its contents are supplied under the terms of the 33 * Common Development and Distribution License ("CDDL"), version 1.0. 34 * You may only use this file in accordance with the terms of version 35 * 1.0 of the CDDL. 36 * 37 * A full copy of the text of the CDDL should have accompanied this 38 * source. A copy of the CDDL is also available via the Internet at 39 * http://www.illumos.org/license/CDDL. 40 * 41 * Copyright 2014 Pluribus Networks Inc. 42 * Copyright 2017 Joyent, Inc. 43 * Copyright 2021 Oxide Computer Company 44 */ 45 46 #include <sys/cdefs.h> 47 __FBSDID("$FreeBSD$"); 48 49 #include <sys/param.h> 50 #include <sys/queue.h> 51 #include <sys/mutex.h> 52 #include <sys/systm.h> 53 #include <sys/kernel.h> 54 #include <sys/kmem.h> 55 #include <sys/cpuset.h> 56 57 #include <x86/apicreg.h> 58 #include <machine/vmm.h> 59 #include <sys/vmm_data.h> 60 61 #include "vmm_lapic.h" 62 #include "vlapic.h" 63 #include "vioapic.h" 64 65 #define IOREGSEL 0x00 66 #define IOWIN 0x10 67 68 #define REDIR_ENTRIES 32 69 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 70 71 struct ioapic_stats { 72 uint64_t is_interrupts; 73 uint64_t is_saturate_low; 74 uint64_t is_saturate_high; 75 }; 76 77 struct vioapic { 78 struct vm *vm; 79 kmutex_t lock; 80 uint32_t id; 81 uint32_t ioregsel; 82 struct { 83 uint64_t reg; 84 /* 85 * The sum of pin asserts (+1) and deasserts (-1) are tracked in 86 * 'acnt'. It is clamped to prevent overflow or underflow 87 * should emulation consumers feed it an invalid set of 88 * transitions. 89 */ 90 uint_t acnt; 91 } rtbl[REDIR_ENTRIES]; 92 struct ioapic_stats stats; 93 }; 94 95 #define VIOAPIC_LOCK(vioapic) mutex_enter(&((vioapic)->lock)) 96 #define VIOAPIC_UNLOCK(vioapic) mutex_exit(&((vioapic)->lock)) 97 #define VIOAPIC_LOCKED(vioapic) MUTEX_HELD(&((vioapic)->lock)) 98 99 100 static void 101 vioapic_send_intr(struct vioapic *vioapic, int pin) 102 { 103 int vector, delmode; 104 uint32_t low, high, dest; 105 bool level, phys; 106 107 VERIFY(pin >= 0 && pin < REDIR_ENTRIES); 108 ASSERT(VIOAPIC_LOCKED(vioapic)); 109 110 low = vioapic->rtbl[pin].reg; 111 high = vioapic->rtbl[pin].reg >> 32; 112 113 if ((low & IOART_INTMASK) == IOART_INTMSET) { 114 /* Pin is masked */ 115 return; 116 } 117 118 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 119 delmode = low & IOART_DELMOD; 120 level = low & IOART_TRGRLVL ? true : false; 121 if (level) { 122 if ((low & IOART_REM_IRR) != 0) { 123 /* IRR already pending */ 124 return; 125 } 126 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 127 } 128 129 vector = low & IOART_INTVEC; 130 dest = high >> APIC_ID_SHIFT; 131 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 132 vioapic->stats.is_interrupts++; 133 } 134 135 static int 136 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 137 { 138 uint_t oldcnt, newcnt; 139 bool needintr = false; 140 int err = 0; 141 142 VERIFY(pin >= 0 && pin < REDIR_ENTRIES); 143 ASSERT(VIOAPIC_LOCKED(vioapic)); 144 145 oldcnt = newcnt = vioapic->rtbl[pin].acnt; 146 if (newstate) { 147 if (newcnt != UINT_MAX) { 148 newcnt++; 149 } else { 150 err = E2BIG; 151 DTRACE_PROBE2(vioapic__sat_high, 152 struct vioapic *, vioapic, int, pin); 153 vioapic->stats.is_saturate_high++; 154 } 155 } else { 156 if (newcnt != 0) { 157 newcnt--; 158 } else { 159 err = ERANGE; 160 DTRACE_PROBE2(vioapic__sat_low, 161 struct vioapic *, vioapic, int, pin); 162 vioapic->stats.is_saturate_low++; 163 } 164 } 165 vioapic->rtbl[pin].acnt = newcnt; 166 167 if (oldcnt == 0 && newcnt == 1) { 168 needintr = true; 169 DTRACE_PROBE2(vioapic__assert, struct vioapic *, vioapic, 170 int, pin); 171 } else if (oldcnt == 1 && newcnt == 0) { 172 DTRACE_PROBE2(vioapic__deassert, struct vioapic *, vioapic, 173 int, pin); 174 } 175 176 if (needintr) { 177 vioapic_send_intr(vioapic, pin); 178 } 179 return (err); 180 } 181 182 enum irqstate { 183 IRQSTATE_ASSERT, 184 IRQSTATE_DEASSERT, 185 IRQSTATE_PULSE 186 }; 187 188 static int 189 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 190 { 191 struct vioapic *vioapic; 192 int err = 0; 193 194 if (irq < 0 || irq >= REDIR_ENTRIES) 195 return (EINVAL); 196 197 vioapic = vm_ioapic(vm); 198 199 VIOAPIC_LOCK(vioapic); 200 switch (irqstate) { 201 case IRQSTATE_ASSERT: 202 err = vioapic_set_pinstate(vioapic, irq, true); 203 break; 204 case IRQSTATE_DEASSERT: 205 err = vioapic_set_pinstate(vioapic, irq, false); 206 break; 207 case IRQSTATE_PULSE: 208 err = vioapic_set_pinstate(vioapic, irq, true); 209 if (err == 0) { 210 err = vioapic_set_pinstate(vioapic, irq, false); 211 } 212 break; 213 default: 214 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 215 } 216 VIOAPIC_UNLOCK(vioapic); 217 218 return (err); 219 } 220 221 int 222 vioapic_assert_irq(struct vm *vm, int irq) 223 { 224 225 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 226 } 227 228 int 229 vioapic_deassert_irq(struct vm *vm, int irq) 230 { 231 232 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 233 } 234 235 int 236 vioapic_pulse_irq(struct vm *vm, int irq) 237 { 238 239 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 240 } 241 242 static uint32_t 243 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 244 { 245 int regnum, pin, rshift; 246 247 regnum = addr & 0xff; 248 switch (regnum) { 249 case IOAPIC_ID: 250 return (vioapic->id); 251 break; 252 case IOAPIC_VER: 253 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 254 break; 255 case IOAPIC_ARB: 256 return (vioapic->id); 257 break; 258 default: 259 break; 260 } 261 262 /* redirection table entries */ 263 if (regnum >= IOAPIC_REDTBL && 264 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 265 pin = (regnum - IOAPIC_REDTBL) / 2; 266 if ((regnum - IOAPIC_REDTBL) % 2) 267 rshift = 32; 268 else 269 rshift = 0; 270 271 return (vioapic->rtbl[pin].reg >> rshift); 272 } 273 274 return (0); 275 } 276 277 static void 278 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 279 { 280 uint64_t data64, mask64; 281 int regnum, pin, lshift; 282 283 regnum = addr & 0xff; 284 switch (regnum) { 285 case IOAPIC_ID: 286 vioapic->id = data & APIC_ID_MASK; 287 break; 288 case IOAPIC_VER: 289 case IOAPIC_ARB: 290 /* readonly */ 291 break; 292 default: 293 break; 294 } 295 296 /* redirection table entries */ 297 if (regnum >= IOAPIC_REDTBL && 298 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 299 pin = (regnum - IOAPIC_REDTBL) / 2; 300 if ((regnum - IOAPIC_REDTBL) % 2) 301 lshift = 32; 302 else 303 lshift = 0; 304 305 data64 = (uint64_t)data << lshift; 306 mask64 = (uint64_t)0xffffffff << lshift; 307 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 308 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 309 310 /* 311 * Switching from level to edge triggering will clear the IRR 312 * bit. This is what FreeBSD will do in order to EOI an 313 * interrupt when the IO-APIC doesn't support targeted EOI (see 314 * _ioapic_eoi_source). 315 */ 316 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 317 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 318 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 319 320 /* 321 * Generate an interrupt if the following conditions are met: 322 * - pin trigger mode is level 323 * - pin level is asserted 324 */ 325 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 326 (vioapic->rtbl[pin].acnt > 0)) { 327 vioapic_send_intr(vioapic, pin); 328 } 329 } 330 } 331 332 static int 333 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 334 uint64_t *data, int size, bool doread) 335 { 336 uint64_t offset; 337 338 offset = gpa - VIOAPIC_BASE; 339 340 /* 341 * The IOAPIC specification allows 32-bit wide accesses to the 342 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 343 */ 344 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 345 if (doread) 346 *data = 0; 347 return (0); 348 } 349 350 VIOAPIC_LOCK(vioapic); 351 if (offset == IOREGSEL) { 352 if (doread) 353 *data = vioapic->ioregsel; 354 else 355 vioapic->ioregsel = *data; 356 } else { 357 if (doread) { 358 *data = vioapic_read(vioapic, vcpuid, 359 vioapic->ioregsel); 360 } else { 361 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 362 *data); 363 } 364 } 365 VIOAPIC_UNLOCK(vioapic); 366 367 return (0); 368 } 369 370 int 371 vioapic_mmio_read(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 372 int size) 373 { 374 int error; 375 struct vioapic *vioapic; 376 377 vioapic = vm_ioapic(vm); 378 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 379 return (error); 380 } 381 382 int 383 vioapic_mmio_write(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t wval, 384 int size) 385 { 386 int error; 387 struct vioapic *vioapic; 388 389 vioapic = vm_ioapic(vm); 390 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 391 return (error); 392 } 393 394 void 395 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 396 { 397 struct vioapic *vioapic; 398 int pin; 399 400 KASSERT(vector >= 0 && vector < 256, 401 ("vioapic_process_eoi: invalid vector %d", vector)); 402 403 vioapic = vm_ioapic(vm); 404 405 /* 406 * XXX keep track of the pins associated with this vector instead 407 * of iterating on every single pin each time. 408 */ 409 VIOAPIC_LOCK(vioapic); 410 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 411 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 412 continue; 413 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 414 continue; 415 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 416 if (vioapic->rtbl[pin].acnt > 0) { 417 /* Pin asserted at EOI */ 418 vioapic_send_intr(vioapic, pin); 419 } 420 } 421 VIOAPIC_UNLOCK(vioapic); 422 } 423 424 struct vioapic * 425 vioapic_init(struct vm *vm) 426 { 427 int i; 428 struct vioapic *vioapic; 429 430 vioapic = kmem_zalloc(sizeof (struct vioapic), KM_SLEEP); 431 432 vioapic->vm = vm; 433 mutex_init(&vioapic->lock, NULL, MUTEX_ADAPTIVE, NULL); 434 435 /* Initialize all redirection entries to mask all interrupts */ 436 for (i = 0; i < REDIR_ENTRIES; i++) 437 vioapic->rtbl[i].reg = 0x0001000000010000UL; 438 439 return (vioapic); 440 } 441 442 void 443 vioapic_cleanup(struct vioapic *vioapic) 444 { 445 mutex_destroy(&vioapic->lock); 446 kmem_free(vioapic, sizeof (*vioapic)); 447 } 448 449 int 450 vioapic_pincount(struct vm *vm) 451 { 452 453 return (REDIR_ENTRIES); 454 } 455 456 static int 457 vioapic_data_read(void *datap, const vmm_data_req_t *req) 458 { 459 VERIFY3U(req->vdr_class, ==, VDC_IOAPIC); 460 VERIFY3U(req->vdr_version, ==, 1); 461 VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_ioapic_v1)); 462 463 struct vioapic *vioapic = datap; 464 struct vdi_ioapic_v1 *out = req->vdr_data; 465 466 VIOAPIC_LOCK(vioapic); 467 out->vi_id = vioapic->id; 468 out->vi_reg_sel = vioapic->ioregsel; 469 for (uint_t i = 0; i < REDIR_ENTRIES; i++) { 470 out->vi_pin_reg[i] = vioapic->rtbl[i].reg; 471 out->vi_pin_level[i] = vioapic->rtbl[i].acnt; 472 } 473 VIOAPIC_UNLOCK(vioapic); 474 475 return (0); 476 } 477 478 static int 479 vioapic_data_write(void *datap, const vmm_data_req_t *req) 480 { 481 VERIFY3U(req->vdr_class, ==, VDC_IOAPIC); 482 VERIFY3U(req->vdr_version, ==, 1); 483 VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_ioapic_v1)); 484 485 struct vioapic *vioapic = datap; 486 const struct vdi_ioapic_v1 *src = req->vdr_data; 487 488 VIOAPIC_LOCK(vioapic); 489 vioapic->id = src->vi_id; 490 vioapic->ioregsel = src->vi_reg_sel; 491 for (uint_t i = 0; i < REDIR_ENTRIES; i++) { 492 vioapic->rtbl[i].reg = src->vi_pin_reg[i] & ~RTBL_RO_BITS; 493 vioapic->rtbl[i].acnt = src->vi_pin_level[i]; 494 } 495 VIOAPIC_UNLOCK(vioapic); 496 497 return (0); 498 } 499 500 static const vmm_data_version_entry_t ioapic_v1 = { 501 .vdve_class = VDC_IOAPIC, 502 .vdve_version = 1, 503 .vdve_len_expect = sizeof (struct vdi_ioapic_v1), 504 .vdve_readf = vioapic_data_read, 505 .vdve_writef = vioapic_data_write, 506 }; 507 VMM_DATA_VERSION(ioapic_v1); 508