1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> 5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 /* 30 * This file and its contents are supplied under the terms of the 31 * Common Development and Distribution License ("CDDL"), version 1.0. 32 * You may only use this file in accordance with the terms of version 33 * 1.0 of the CDDL. 34 * 35 * A full copy of the text of the CDDL should have accompanied this 36 * source. A copy of the CDDL is also available via the Internet at 37 * http://www.illumos.org/license/CDDL. 38 * 39 * Copyright 2014 Pluribus Networks Inc. 40 * Copyright 2017 Joyent, Inc. 41 * Copyright 2021 Oxide Computer Company 42 */ 43 44 #include <sys/cdefs.h> 45 46 #include <sys/param.h> 47 #include <sys/queue.h> 48 #include <sys/mutex.h> 49 #include <sys/systm.h> 50 #include <sys/kernel.h> 51 #include <sys/kmem.h> 52 #include <sys/cpuset.h> 53 54 #include <x86/apicreg.h> 55 #include <machine/vmm.h> 56 #include <sys/vmm_data.h> 57 58 #include "vmm_lapic.h" 59 #include "vlapic.h" 60 #include "vioapic.h" 61 62 #define IOREGSEL 0x00 63 #define IOWIN 0x10 64 65 #define REDIR_ENTRIES 32 66 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) 67 68 struct ioapic_stats { 69 uint64_t is_interrupts; 70 uint64_t is_saturate_low; 71 uint64_t is_saturate_high; 72 }; 73 74 struct vioapic { 75 struct vm *vm; 76 kmutex_t lock; 77 uint32_t id; 78 uint32_t ioregsel; 79 struct { 80 uint64_t reg; 81 /* 82 * The sum of pin asserts (+1) and deasserts (-1) are tracked in 83 * 'acnt'. It is clamped to prevent overflow or underflow 84 * should emulation consumers feed it an invalid set of 85 * transitions. 86 */ 87 uint_t acnt; 88 } rtbl[REDIR_ENTRIES]; 89 struct ioapic_stats stats; 90 }; 91 92 #define VIOAPIC_LOCK(vioapic) mutex_enter(&((vioapic)->lock)) 93 #define VIOAPIC_UNLOCK(vioapic) mutex_exit(&((vioapic)->lock)) 94 #define VIOAPIC_LOCKED(vioapic) MUTEX_HELD(&((vioapic)->lock)) 95 96 97 static void 98 vioapic_send_intr(struct vioapic *vioapic, int pin) 99 { 100 int vector, delmode; 101 uint32_t low, high, dest; 102 bool level, phys; 103 104 VERIFY(pin >= 0 && pin < REDIR_ENTRIES); 105 ASSERT(VIOAPIC_LOCKED(vioapic)); 106 107 low = vioapic->rtbl[pin].reg; 108 high = vioapic->rtbl[pin].reg >> 32; 109 110 if ((low & IOART_INTMASK) == IOART_INTMSET) { 111 /* Pin is masked */ 112 return; 113 } 114 115 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); 116 delmode = low & IOART_DELMOD; 117 level = low & IOART_TRGRLVL ? true : false; 118 if (level) { 119 if ((low & IOART_REM_IRR) != 0) { 120 /* IRR already pending */ 121 return; 122 } 123 vioapic->rtbl[pin].reg |= IOART_REM_IRR; 124 } 125 126 vector = low & IOART_INTVEC; 127 dest = high >> APIC_ID_SHIFT; 128 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); 129 vioapic->stats.is_interrupts++; 130 } 131 132 static int 133 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate) 134 { 135 uint_t oldcnt, newcnt; 136 bool needintr = false; 137 int err = 0; 138 139 VERIFY(pin >= 0 && pin < REDIR_ENTRIES); 140 ASSERT(VIOAPIC_LOCKED(vioapic)); 141 142 oldcnt = newcnt = vioapic->rtbl[pin].acnt; 143 if (newstate) { 144 if (newcnt != UINT_MAX) { 145 newcnt++; 146 } else { 147 err = E2BIG; 148 DTRACE_PROBE2(vioapic__sat_high, 149 struct vioapic *, vioapic, int, pin); 150 vioapic->stats.is_saturate_high++; 151 } 152 } else { 153 if (newcnt != 0) { 154 newcnt--; 155 } else { 156 err = ERANGE; 157 DTRACE_PROBE2(vioapic__sat_low, 158 struct vioapic *, vioapic, int, pin); 159 vioapic->stats.is_saturate_low++; 160 } 161 } 162 vioapic->rtbl[pin].acnt = newcnt; 163 164 if (oldcnt == 0 && newcnt == 1) { 165 needintr = true; 166 DTRACE_PROBE2(vioapic__assert, struct vioapic *, vioapic, 167 int, pin); 168 } else if (oldcnt == 1 && newcnt == 0) { 169 DTRACE_PROBE2(vioapic__deassert, struct vioapic *, vioapic, 170 int, pin); 171 } 172 173 if (needintr) { 174 vioapic_send_intr(vioapic, pin); 175 } 176 return (err); 177 } 178 179 enum irqstate { 180 IRQSTATE_ASSERT, 181 IRQSTATE_DEASSERT, 182 IRQSTATE_PULSE 183 }; 184 185 static int 186 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate) 187 { 188 struct vioapic *vioapic; 189 int err = 0; 190 191 if (irq < 0 || irq >= REDIR_ENTRIES) 192 return (EINVAL); 193 194 vioapic = vm_ioapic(vm); 195 196 VIOAPIC_LOCK(vioapic); 197 switch (irqstate) { 198 case IRQSTATE_ASSERT: 199 err = vioapic_set_pinstate(vioapic, irq, true); 200 break; 201 case IRQSTATE_DEASSERT: 202 err = vioapic_set_pinstate(vioapic, irq, false); 203 break; 204 case IRQSTATE_PULSE: 205 err = vioapic_set_pinstate(vioapic, irq, true); 206 if (err == 0) { 207 err = vioapic_set_pinstate(vioapic, irq, false); 208 } 209 break; 210 default: 211 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate); 212 } 213 VIOAPIC_UNLOCK(vioapic); 214 215 return (err); 216 } 217 218 int 219 vioapic_assert_irq(struct vm *vm, int irq) 220 { 221 222 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT)); 223 } 224 225 int 226 vioapic_deassert_irq(struct vm *vm, int irq) 227 { 228 229 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT)); 230 } 231 232 int 233 vioapic_pulse_irq(struct vm *vm, int irq) 234 { 235 236 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); 237 } 238 239 static uint32_t 240 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) 241 { 242 int regnum, pin, rshift; 243 244 regnum = addr & 0xff; 245 switch (regnum) { 246 case IOAPIC_ID: 247 return (vioapic->id); 248 break; 249 case IOAPIC_VER: 250 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11); 251 break; 252 case IOAPIC_ARB: 253 return (vioapic->id); 254 break; 255 default: 256 break; 257 } 258 259 /* redirection table entries */ 260 if (regnum >= IOAPIC_REDTBL && 261 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 262 pin = (regnum - IOAPIC_REDTBL) / 2; 263 if ((regnum - IOAPIC_REDTBL) % 2) 264 rshift = 32; 265 else 266 rshift = 0; 267 268 return (vioapic->rtbl[pin].reg >> rshift); 269 } 270 271 return (0); 272 } 273 274 static void 275 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) 276 { 277 uint64_t data64, mask64; 278 int regnum, pin, lshift; 279 280 regnum = addr & 0xff; 281 switch (regnum) { 282 case IOAPIC_ID: 283 vioapic->id = data & APIC_ID_MASK; 284 break; 285 case IOAPIC_VER: 286 case IOAPIC_ARB: 287 /* readonly */ 288 break; 289 default: 290 break; 291 } 292 293 /* redirection table entries */ 294 if (regnum >= IOAPIC_REDTBL && 295 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) { 296 pin = (regnum - IOAPIC_REDTBL) / 2; 297 if ((regnum - IOAPIC_REDTBL) % 2) 298 lshift = 32; 299 else 300 lshift = 0; 301 302 data64 = (uint64_t)data << lshift; 303 mask64 = (uint64_t)0xffffffff << lshift; 304 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; 305 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; 306 307 /* 308 * Switching from level to edge triggering will clear the IRR 309 * bit. This is what FreeBSD will do in order to EOI an 310 * interrupt when the IO-APIC doesn't support targeted EOI (see 311 * _ioapic_eoi_source). 312 */ 313 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG && 314 (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0) 315 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 316 317 /* 318 * Generate an interrupt if the following conditions are met: 319 * - pin trigger mode is level 320 * - pin level is asserted 321 */ 322 if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL && 323 (vioapic->rtbl[pin].acnt > 0)) { 324 vioapic_send_intr(vioapic, pin); 325 } 326 } 327 } 328 329 static int 330 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa, 331 uint64_t *data, int size, bool doread) 332 { 333 uint64_t offset; 334 335 offset = gpa - VIOAPIC_BASE; 336 337 /* 338 * The IOAPIC specification allows 32-bit wide accesses to the 339 * IOREGSEL (offset 0) and IOWIN (offset 16) registers. 340 */ 341 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { 342 if (doread) 343 *data = 0; 344 return (0); 345 } 346 347 VIOAPIC_LOCK(vioapic); 348 if (offset == IOREGSEL) { 349 if (doread) 350 *data = vioapic->ioregsel; 351 else 352 vioapic->ioregsel = *data; 353 } else { 354 if (doread) { 355 *data = vioapic_read(vioapic, vcpuid, 356 vioapic->ioregsel); 357 } else { 358 vioapic_write(vioapic, vcpuid, vioapic->ioregsel, 359 *data); 360 } 361 } 362 VIOAPIC_UNLOCK(vioapic); 363 364 return (0); 365 } 366 367 int 368 vioapic_mmio_read(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t *rval, 369 int size) 370 { 371 int error; 372 struct vioapic *vioapic; 373 374 vioapic = vm_ioapic(vm); 375 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true); 376 return (error); 377 } 378 379 int 380 vioapic_mmio_write(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t wval, 381 int size) 382 { 383 int error; 384 struct vioapic *vioapic; 385 386 vioapic = vm_ioapic(vm); 387 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false); 388 return (error); 389 } 390 391 void 392 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector) 393 { 394 struct vioapic *vioapic; 395 int pin; 396 397 KASSERT(vector >= 0 && vector < 256, 398 ("vioapic_process_eoi: invalid vector %d", vector)); 399 400 vioapic = vm_ioapic(vm); 401 402 /* 403 * XXX keep track of the pins associated with this vector instead 404 * of iterating on every single pin each time. 405 */ 406 VIOAPIC_LOCK(vioapic); 407 for (pin = 0; pin < REDIR_ENTRIES; pin++) { 408 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0) 409 continue; 410 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector) 411 continue; 412 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR; 413 if (vioapic->rtbl[pin].acnt > 0) { 414 /* Pin asserted at EOI */ 415 vioapic_send_intr(vioapic, pin); 416 } 417 } 418 VIOAPIC_UNLOCK(vioapic); 419 } 420 421 struct vioapic * 422 vioapic_init(struct vm *vm) 423 { 424 int i; 425 struct vioapic *vioapic; 426 427 vioapic = kmem_zalloc(sizeof (struct vioapic), KM_SLEEP); 428 429 vioapic->vm = vm; 430 mutex_init(&vioapic->lock, NULL, MUTEX_ADAPTIVE, NULL); 431 432 /* Initialize all redirection entries to mask all interrupts */ 433 for (i = 0; i < REDIR_ENTRIES; i++) 434 vioapic->rtbl[i].reg = 0x0001000000010000UL; 435 436 return (vioapic); 437 } 438 439 void 440 vioapic_cleanup(struct vioapic *vioapic) 441 { 442 mutex_destroy(&vioapic->lock); 443 kmem_free(vioapic, sizeof (*vioapic)); 444 } 445 446 int 447 vioapic_pincount(struct vm *vm) 448 { 449 450 return (REDIR_ENTRIES); 451 } 452 453 static int 454 vioapic_data_read(void *datap, const vmm_data_req_t *req) 455 { 456 VERIFY3U(req->vdr_class, ==, VDC_IOAPIC); 457 VERIFY3U(req->vdr_version, ==, 1); 458 VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_ioapic_v1)); 459 460 struct vioapic *vioapic = datap; 461 struct vdi_ioapic_v1 *out = req->vdr_data; 462 463 VIOAPIC_LOCK(vioapic); 464 out->vi_id = vioapic->id; 465 out->vi_reg_sel = vioapic->ioregsel; 466 for (uint_t i = 0; i < REDIR_ENTRIES; i++) { 467 out->vi_pin_reg[i] = vioapic->rtbl[i].reg; 468 out->vi_pin_level[i] = vioapic->rtbl[i].acnt; 469 } 470 VIOAPIC_UNLOCK(vioapic); 471 472 return (0); 473 } 474 475 static int 476 vioapic_data_write(void *datap, const vmm_data_req_t *req) 477 { 478 VERIFY3U(req->vdr_class, ==, VDC_IOAPIC); 479 VERIFY3U(req->vdr_version, ==, 1); 480 VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_ioapic_v1)); 481 482 struct vioapic *vioapic = datap; 483 const struct vdi_ioapic_v1 *src = req->vdr_data; 484 485 VIOAPIC_LOCK(vioapic); 486 vioapic->id = src->vi_id; 487 vioapic->ioregsel = src->vi_reg_sel; 488 for (uint_t i = 0; i < REDIR_ENTRIES; i++) { 489 vioapic->rtbl[i].reg = src->vi_pin_reg[i] & ~RTBL_RO_BITS; 490 vioapic->rtbl[i].acnt = src->vi_pin_level[i]; 491 } 492 VIOAPIC_UNLOCK(vioapic); 493 494 return (0); 495 } 496 497 static const vmm_data_version_entry_t ioapic_v1 = { 498 .vdve_class = VDC_IOAPIC, 499 .vdve_version = 1, 500 .vdve_len_expect = sizeof (struct vdi_ioapic_v1), 501 .vdve_readf = vioapic_data_read, 502 .vdve_writef = vioapic_data_write, 503 }; 504 VMM_DATA_VERSION(ioapic_v1); 505