1 /* 2 * Copyright (c) 1991 The Regents of the University of California. 3 * Copyright (c) 1996, by Steve Passe. All rights reserved. 4 * Copyright (c) 2005,2008 The DragonFly Project. All rights reserved. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The DragonFly Project 8 * by Matthew Dillon <dillon@backplane.com> 9 * 10 * This code is derived from software contributed to Berkeley by 11 * William Jolitz. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 3. Neither the name of The DragonFly Project nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific, prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 30 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 31 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 34 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 35 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 36 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 37 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/kernel.h> 44 #include <sys/machintr.h> 45 #include <sys/interrupt.h> 46 #include <sys/bus.h> 47 #include <sys/rman.h> 48 #include <sys/thread2.h> 49 50 #include <machine/smp.h> 51 #include <machine/segments.h> 52 #include <machine/md_var.h> 53 #include <machine/intr_machdep.h> 54 #include <machine/globaldata.h> 55 #include <machine/msi_var.h> 56 57 #include <machine_base/isa/isa_intr.h> 58 #include <machine_base/icu/icu.h> 59 #include <machine_base/icu/icu_var.h> 60 #include <machine_base/apic/ioapic.h> 61 #include <machine_base/apic/ioapic_abi.h> 62 #include <machine_base/apic/ioapic_ipl.h> 63 #include <machine_base/apic/apicreg.h> 64 65 #include <dev/acpica5/acpi_sci_var.h> 66 67 #define IOAPIC_HWI_VECTORS IDT_HWI_VECTORS 68 69 extern inthand_t 70 IDTVEC(ioapic_intr0), 71 IDTVEC(ioapic_intr1), 72 IDTVEC(ioapic_intr2), 73 IDTVEC(ioapic_intr3), 74 IDTVEC(ioapic_intr4), 75 IDTVEC(ioapic_intr5), 76 IDTVEC(ioapic_intr6), 77 IDTVEC(ioapic_intr7), 78 IDTVEC(ioapic_intr8), 79 IDTVEC(ioapic_intr9), 80 IDTVEC(ioapic_intr10), 81 IDTVEC(ioapic_intr11), 82 IDTVEC(ioapic_intr12), 83 IDTVEC(ioapic_intr13), 84 IDTVEC(ioapic_intr14), 85 IDTVEC(ioapic_intr15), 86 IDTVEC(ioapic_intr16), 87 IDTVEC(ioapic_intr17), 88 IDTVEC(ioapic_intr18), 89 IDTVEC(ioapic_intr19), 90 IDTVEC(ioapic_intr20), 91 IDTVEC(ioapic_intr21), 92 IDTVEC(ioapic_intr22), 93 IDTVEC(ioapic_intr23), 94 IDTVEC(ioapic_intr24), 95 IDTVEC(ioapic_intr25), 96 IDTVEC(ioapic_intr26), 97 IDTVEC(ioapic_intr27), 98 IDTVEC(ioapic_intr28), 99 IDTVEC(ioapic_intr29), 100 IDTVEC(ioapic_intr30), 101 IDTVEC(ioapic_intr31), 102 IDTVEC(ioapic_intr32), 103 IDTVEC(ioapic_intr33), 104 IDTVEC(ioapic_intr34), 105 IDTVEC(ioapic_intr35), 106 IDTVEC(ioapic_intr36), 107 IDTVEC(ioapic_intr37), 108 IDTVEC(ioapic_intr38), 109 IDTVEC(ioapic_intr39), 110 IDTVEC(ioapic_intr40), 111 IDTVEC(ioapic_intr41), 112 IDTVEC(ioapic_intr42), 113 IDTVEC(ioapic_intr43), 114 IDTVEC(ioapic_intr44), 115 IDTVEC(ioapic_intr45), 116 IDTVEC(ioapic_intr46), 117 IDTVEC(ioapic_intr47), 118 IDTVEC(ioapic_intr48), 119 IDTVEC(ioapic_intr49), 120 IDTVEC(ioapic_intr50), 121 IDTVEC(ioapic_intr51), 122 IDTVEC(ioapic_intr52), 123 IDTVEC(ioapic_intr53), 124 IDTVEC(ioapic_intr54), 125 IDTVEC(ioapic_intr55), 126 IDTVEC(ioapic_intr56), 127 IDTVEC(ioapic_intr57), 128 IDTVEC(ioapic_intr58), 129 IDTVEC(ioapic_intr59), 130 IDTVEC(ioapic_intr60), 131 IDTVEC(ioapic_intr61), 132 IDTVEC(ioapic_intr62), 133 IDTVEC(ioapic_intr63), 134 IDTVEC(ioapic_intr64), 135 IDTVEC(ioapic_intr65), 136 IDTVEC(ioapic_intr66), 137 IDTVEC(ioapic_intr67), 138 IDTVEC(ioapic_intr68), 139 IDTVEC(ioapic_intr69), 140 IDTVEC(ioapic_intr70), 141 IDTVEC(ioapic_intr71), 142 IDTVEC(ioapic_intr72), 143 IDTVEC(ioapic_intr73), 144 IDTVEC(ioapic_intr74), 145 IDTVEC(ioapic_intr75), 146 IDTVEC(ioapic_intr76), 147 IDTVEC(ioapic_intr77), 148 IDTVEC(ioapic_intr78), 149 IDTVEC(ioapic_intr79), 150 IDTVEC(ioapic_intr80), 151 IDTVEC(ioapic_intr81), 152 IDTVEC(ioapic_intr82), 153 IDTVEC(ioapic_intr83), 154 IDTVEC(ioapic_intr84), 155 IDTVEC(ioapic_intr85), 156 IDTVEC(ioapic_intr86), 157 IDTVEC(ioapic_intr87), 158 IDTVEC(ioapic_intr88), 159 IDTVEC(ioapic_intr89), 160 IDTVEC(ioapic_intr90), 161 IDTVEC(ioapic_intr91), 162 IDTVEC(ioapic_intr92), 163 IDTVEC(ioapic_intr93), 164 IDTVEC(ioapic_intr94), 165 IDTVEC(ioapic_intr95), 166 IDTVEC(ioapic_intr96), 167 IDTVEC(ioapic_intr97), 168 IDTVEC(ioapic_intr98), 169 IDTVEC(ioapic_intr99), 170 IDTVEC(ioapic_intr100), 171 IDTVEC(ioapic_intr101), 172 IDTVEC(ioapic_intr102), 173 IDTVEC(ioapic_intr103), 174 IDTVEC(ioapic_intr104), 175 IDTVEC(ioapic_intr105), 176 IDTVEC(ioapic_intr106), 177 IDTVEC(ioapic_intr107), 178 IDTVEC(ioapic_intr108), 179 IDTVEC(ioapic_intr109), 180 IDTVEC(ioapic_intr110), 181 IDTVEC(ioapic_intr111), 182 IDTVEC(ioapic_intr112), 183 IDTVEC(ioapic_intr113), 184 IDTVEC(ioapic_intr114), 185 IDTVEC(ioapic_intr115), 186 IDTVEC(ioapic_intr116), 187 IDTVEC(ioapic_intr117), 188 IDTVEC(ioapic_intr118), 189 IDTVEC(ioapic_intr119), 190 IDTVEC(ioapic_intr120), 191 IDTVEC(ioapic_intr121), 192 IDTVEC(ioapic_intr122), 193 IDTVEC(ioapic_intr123), 194 IDTVEC(ioapic_intr124), 195 IDTVEC(ioapic_intr125), 196 IDTVEC(ioapic_intr126), 197 IDTVEC(ioapic_intr127), 198 IDTVEC(ioapic_intr128), 199 IDTVEC(ioapic_intr129), 200 IDTVEC(ioapic_intr130), 201 IDTVEC(ioapic_intr131), 202 IDTVEC(ioapic_intr132), 203 IDTVEC(ioapic_intr133), 204 IDTVEC(ioapic_intr134), 205 IDTVEC(ioapic_intr135), 206 IDTVEC(ioapic_intr136), 207 IDTVEC(ioapic_intr137), 208 IDTVEC(ioapic_intr138), 209 IDTVEC(ioapic_intr139), 210 IDTVEC(ioapic_intr140), 211 IDTVEC(ioapic_intr141), 212 IDTVEC(ioapic_intr142), 213 IDTVEC(ioapic_intr143), 214 IDTVEC(ioapic_intr144), 215 IDTVEC(ioapic_intr145), 216 IDTVEC(ioapic_intr146), 217 IDTVEC(ioapic_intr147), 218 IDTVEC(ioapic_intr148), 219 IDTVEC(ioapic_intr149), 220 IDTVEC(ioapic_intr150), 221 IDTVEC(ioapic_intr151), 222 IDTVEC(ioapic_intr152), 223 IDTVEC(ioapic_intr153), 224 IDTVEC(ioapic_intr154), 225 IDTVEC(ioapic_intr155), 226 IDTVEC(ioapic_intr156), 227 IDTVEC(ioapic_intr157), 228 IDTVEC(ioapic_intr158), 229 IDTVEC(ioapic_intr159), 230 IDTVEC(ioapic_intr160), 231 IDTVEC(ioapic_intr161), 232 IDTVEC(ioapic_intr162), 233 IDTVEC(ioapic_intr163), 234 IDTVEC(ioapic_intr164), 235 IDTVEC(ioapic_intr165), 236 IDTVEC(ioapic_intr166), 237 IDTVEC(ioapic_intr167), 238 IDTVEC(ioapic_intr168), 239 IDTVEC(ioapic_intr169), 240 IDTVEC(ioapic_intr170), 241 IDTVEC(ioapic_intr171), 242 IDTVEC(ioapic_intr172), 243 IDTVEC(ioapic_intr173), 244 IDTVEC(ioapic_intr174), 245 IDTVEC(ioapic_intr175), 246 IDTVEC(ioapic_intr176), 247 IDTVEC(ioapic_intr177), 248 IDTVEC(ioapic_intr178), 249 IDTVEC(ioapic_intr179), 250 IDTVEC(ioapic_intr180), 251 IDTVEC(ioapic_intr181), 252 IDTVEC(ioapic_intr182), 253 IDTVEC(ioapic_intr183), 254 IDTVEC(ioapic_intr184), 255 IDTVEC(ioapic_intr185), 256 IDTVEC(ioapic_intr186), 257 IDTVEC(ioapic_intr187), 258 IDTVEC(ioapic_intr188), 259 IDTVEC(ioapic_intr189), 260 IDTVEC(ioapic_intr190), 261 IDTVEC(ioapic_intr191); 262 263 static inthand_t *ioapic_intr[IOAPIC_HWI_VECTORS] = { 264 &IDTVEC(ioapic_intr0), 265 &IDTVEC(ioapic_intr1), 266 &IDTVEC(ioapic_intr2), 267 &IDTVEC(ioapic_intr3), 268 &IDTVEC(ioapic_intr4), 269 &IDTVEC(ioapic_intr5), 270 &IDTVEC(ioapic_intr6), 271 &IDTVEC(ioapic_intr7), 272 &IDTVEC(ioapic_intr8), 273 &IDTVEC(ioapic_intr9), 274 &IDTVEC(ioapic_intr10), 275 &IDTVEC(ioapic_intr11), 276 &IDTVEC(ioapic_intr12), 277 &IDTVEC(ioapic_intr13), 278 &IDTVEC(ioapic_intr14), 279 &IDTVEC(ioapic_intr15), 280 &IDTVEC(ioapic_intr16), 281 &IDTVEC(ioapic_intr17), 282 &IDTVEC(ioapic_intr18), 283 &IDTVEC(ioapic_intr19), 284 &IDTVEC(ioapic_intr20), 285 &IDTVEC(ioapic_intr21), 286 &IDTVEC(ioapic_intr22), 287 &IDTVEC(ioapic_intr23), 288 &IDTVEC(ioapic_intr24), 289 &IDTVEC(ioapic_intr25), 290 &IDTVEC(ioapic_intr26), 291 &IDTVEC(ioapic_intr27), 292 &IDTVEC(ioapic_intr28), 293 &IDTVEC(ioapic_intr29), 294 &IDTVEC(ioapic_intr30), 295 &IDTVEC(ioapic_intr31), 296 &IDTVEC(ioapic_intr32), 297 &IDTVEC(ioapic_intr33), 298 &IDTVEC(ioapic_intr34), 299 &IDTVEC(ioapic_intr35), 300 &IDTVEC(ioapic_intr36), 301 &IDTVEC(ioapic_intr37), 302 &IDTVEC(ioapic_intr38), 303 &IDTVEC(ioapic_intr39), 304 &IDTVEC(ioapic_intr40), 305 &IDTVEC(ioapic_intr41), 306 &IDTVEC(ioapic_intr42), 307 &IDTVEC(ioapic_intr43), 308 &IDTVEC(ioapic_intr44), 309 &IDTVEC(ioapic_intr45), 310 &IDTVEC(ioapic_intr46), 311 &IDTVEC(ioapic_intr47), 312 &IDTVEC(ioapic_intr48), 313 &IDTVEC(ioapic_intr49), 314 &IDTVEC(ioapic_intr50), 315 &IDTVEC(ioapic_intr51), 316 &IDTVEC(ioapic_intr52), 317 &IDTVEC(ioapic_intr53), 318 &IDTVEC(ioapic_intr54), 319 &IDTVEC(ioapic_intr55), 320 &IDTVEC(ioapic_intr56), 321 &IDTVEC(ioapic_intr57), 322 &IDTVEC(ioapic_intr58), 323 &IDTVEC(ioapic_intr59), 324 &IDTVEC(ioapic_intr60), 325 &IDTVEC(ioapic_intr61), 326 &IDTVEC(ioapic_intr62), 327 &IDTVEC(ioapic_intr63), 328 &IDTVEC(ioapic_intr64), 329 &IDTVEC(ioapic_intr65), 330 &IDTVEC(ioapic_intr66), 331 &IDTVEC(ioapic_intr67), 332 &IDTVEC(ioapic_intr68), 333 &IDTVEC(ioapic_intr69), 334 &IDTVEC(ioapic_intr70), 335 &IDTVEC(ioapic_intr71), 336 &IDTVEC(ioapic_intr72), 337 &IDTVEC(ioapic_intr73), 338 &IDTVEC(ioapic_intr74), 339 &IDTVEC(ioapic_intr75), 340 &IDTVEC(ioapic_intr76), 341 &IDTVEC(ioapic_intr77), 342 &IDTVEC(ioapic_intr78), 343 &IDTVEC(ioapic_intr79), 344 &IDTVEC(ioapic_intr80), 345 &IDTVEC(ioapic_intr81), 346 &IDTVEC(ioapic_intr82), 347 &IDTVEC(ioapic_intr83), 348 &IDTVEC(ioapic_intr84), 349 &IDTVEC(ioapic_intr85), 350 &IDTVEC(ioapic_intr86), 351 &IDTVEC(ioapic_intr87), 352 &IDTVEC(ioapic_intr88), 353 &IDTVEC(ioapic_intr89), 354 &IDTVEC(ioapic_intr90), 355 &IDTVEC(ioapic_intr91), 356 &IDTVEC(ioapic_intr92), 357 &IDTVEC(ioapic_intr93), 358 &IDTVEC(ioapic_intr94), 359 &IDTVEC(ioapic_intr95), 360 &IDTVEC(ioapic_intr96), 361 &IDTVEC(ioapic_intr97), 362 &IDTVEC(ioapic_intr98), 363 &IDTVEC(ioapic_intr99), 364 &IDTVEC(ioapic_intr100), 365 &IDTVEC(ioapic_intr101), 366 &IDTVEC(ioapic_intr102), 367 &IDTVEC(ioapic_intr103), 368 &IDTVEC(ioapic_intr104), 369 &IDTVEC(ioapic_intr105), 370 &IDTVEC(ioapic_intr106), 371 &IDTVEC(ioapic_intr107), 372 &IDTVEC(ioapic_intr108), 373 &IDTVEC(ioapic_intr109), 374 &IDTVEC(ioapic_intr110), 375 &IDTVEC(ioapic_intr111), 376 &IDTVEC(ioapic_intr112), 377 &IDTVEC(ioapic_intr113), 378 &IDTVEC(ioapic_intr114), 379 &IDTVEC(ioapic_intr115), 380 &IDTVEC(ioapic_intr116), 381 &IDTVEC(ioapic_intr117), 382 &IDTVEC(ioapic_intr118), 383 &IDTVEC(ioapic_intr119), 384 &IDTVEC(ioapic_intr120), 385 &IDTVEC(ioapic_intr121), 386 &IDTVEC(ioapic_intr122), 387 &IDTVEC(ioapic_intr123), 388 &IDTVEC(ioapic_intr124), 389 &IDTVEC(ioapic_intr125), 390 &IDTVEC(ioapic_intr126), 391 &IDTVEC(ioapic_intr127), 392 &IDTVEC(ioapic_intr128), 393 &IDTVEC(ioapic_intr129), 394 &IDTVEC(ioapic_intr130), 395 &IDTVEC(ioapic_intr131), 396 &IDTVEC(ioapic_intr132), 397 &IDTVEC(ioapic_intr133), 398 &IDTVEC(ioapic_intr134), 399 &IDTVEC(ioapic_intr135), 400 &IDTVEC(ioapic_intr136), 401 &IDTVEC(ioapic_intr137), 402 &IDTVEC(ioapic_intr138), 403 &IDTVEC(ioapic_intr139), 404 &IDTVEC(ioapic_intr140), 405 &IDTVEC(ioapic_intr141), 406 &IDTVEC(ioapic_intr142), 407 &IDTVEC(ioapic_intr143), 408 &IDTVEC(ioapic_intr144), 409 &IDTVEC(ioapic_intr145), 410 &IDTVEC(ioapic_intr146), 411 &IDTVEC(ioapic_intr147), 412 &IDTVEC(ioapic_intr148), 413 &IDTVEC(ioapic_intr149), 414 &IDTVEC(ioapic_intr150), 415 &IDTVEC(ioapic_intr151), 416 &IDTVEC(ioapic_intr152), 417 &IDTVEC(ioapic_intr153), 418 &IDTVEC(ioapic_intr154), 419 &IDTVEC(ioapic_intr155), 420 &IDTVEC(ioapic_intr156), 421 &IDTVEC(ioapic_intr157), 422 &IDTVEC(ioapic_intr158), 423 &IDTVEC(ioapic_intr159), 424 &IDTVEC(ioapic_intr160), 425 &IDTVEC(ioapic_intr161), 426 &IDTVEC(ioapic_intr162), 427 &IDTVEC(ioapic_intr163), 428 &IDTVEC(ioapic_intr164), 429 &IDTVEC(ioapic_intr165), 430 &IDTVEC(ioapic_intr166), 431 &IDTVEC(ioapic_intr167), 432 &IDTVEC(ioapic_intr168), 433 &IDTVEC(ioapic_intr169), 434 &IDTVEC(ioapic_intr170), 435 &IDTVEC(ioapic_intr171), 436 &IDTVEC(ioapic_intr172), 437 &IDTVEC(ioapic_intr173), 438 &IDTVEC(ioapic_intr174), 439 &IDTVEC(ioapic_intr175), 440 &IDTVEC(ioapic_intr176), 441 &IDTVEC(ioapic_intr177), 442 &IDTVEC(ioapic_intr178), 443 &IDTVEC(ioapic_intr179), 444 &IDTVEC(ioapic_intr180), 445 &IDTVEC(ioapic_intr181), 446 &IDTVEC(ioapic_intr182), 447 &IDTVEC(ioapic_intr183), 448 &IDTVEC(ioapic_intr184), 449 &IDTVEC(ioapic_intr185), 450 &IDTVEC(ioapic_intr186), 451 &IDTVEC(ioapic_intr187), 452 &IDTVEC(ioapic_intr188), 453 &IDTVEC(ioapic_intr189), 454 &IDTVEC(ioapic_intr190), 455 &IDTVEC(ioapic_intr191) 456 }; 457 458 #define IOAPIC_HWI_SYSCALL (IDT_OFFSET_SYSCALL - IDT_OFFSET) 459 460 static struct ioapic_irqmap { 461 int im_type; /* IOAPIC_IMT_ */ 462 enum intr_trigger im_trig; 463 enum intr_polarity im_pola; 464 int im_gsi; 465 int im_msi_base; 466 uint32_t im_flags; /* IOAPIC_IMF_ */ 467 } ioapic_irqmaps[MAXCPU][IOAPIC_HWI_VECTORS]; 468 469 static struct lwkt_token ioapic_irqmap_tok = 470 LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token); 471 472 #define IOAPIC_IMT_UNUSED 0 473 #define IOAPIC_IMT_RESERVED 1 474 #define IOAPIC_IMT_LEGACY 2 475 #define IOAPIC_IMT_SYSCALL 3 476 #define IOAPIC_IMT_MSI 4 477 #define IOAPIC_IMT_MSIX 5 478 479 #define IOAPIC_IMT_ISHWI(map) ((map)->im_type != IOAPIC_IMT_RESERVED && \ 480 (map)->im_type != IOAPIC_IMT_SYSCALL) 481 482 #define IOAPIC_IMF_CONF 0x1 483 484 extern void IOAPIC_INTREN(int); 485 extern void IOAPIC_INTRDIS(int); 486 487 extern int imcr_present; 488 489 static void ioapic_abi_intr_enable(int); 490 static void ioapic_abi_intr_disable(int); 491 static void ioapic_abi_intr_setup(int, int); 492 static void ioapic_abi_intr_teardown(int); 493 494 static void ioapic_abi_legacy_intr_config(int, 495 enum intr_trigger, enum intr_polarity); 496 static int ioapic_abi_legacy_intr_cpuid(int); 497 498 static int ioapic_abi_msi_alloc(int [], int, int); 499 static void ioapic_abi_msi_release(const int [], int, int); 500 static void ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int); 501 static int ioapic_abi_msix_alloc(int *, int); 502 static void ioapic_abi_msix_release(int, int); 503 504 static int ioapic_abi_msi_alloc_intern(int, const char *, 505 int [], int, int); 506 static void ioapic_abi_msi_release_intern(int, const char *, 507 const int [], int, int); 508 509 static void ioapic_abi_finalize(void); 510 static void ioapic_abi_cleanup(void); 511 static void ioapic_abi_setdefault(void); 512 static void ioapic_abi_stabilize(void); 513 static void ioapic_abi_initmap(void); 514 static void ioapic_abi_rman_setup(struct rman *); 515 516 static int ioapic_abi_gsi_cpuid(int, int); 517 518 struct machintr_abi MachIntrABI_IOAPIC = { 519 MACHINTR_IOAPIC, 520 .intr_disable = ioapic_abi_intr_disable, 521 .intr_enable = ioapic_abi_intr_enable, 522 .intr_setup = ioapic_abi_intr_setup, 523 .intr_teardown = ioapic_abi_intr_teardown, 524 525 .legacy_intr_config = ioapic_abi_legacy_intr_config, 526 .legacy_intr_cpuid = ioapic_abi_legacy_intr_cpuid, 527 528 .msi_alloc = ioapic_abi_msi_alloc, 529 .msi_release = ioapic_abi_msi_release, 530 .msi_map = ioapic_abi_msi_map, 531 .msix_alloc = ioapic_abi_msix_alloc, 532 .msix_release = ioapic_abi_msix_release, 533 534 .finalize = ioapic_abi_finalize, 535 .cleanup = ioapic_abi_cleanup, 536 .setdefault = ioapic_abi_setdefault, 537 .stabilize = ioapic_abi_stabilize, 538 .initmap = ioapic_abi_initmap, 539 .rman_setup = ioapic_abi_rman_setup 540 }; 541 542 static int ioapic_abi_extint_irq = -1; 543 static int ioapic_abi_legacy_irq_max; 544 static int ioapic_abi_gsi_balance; 545 static int ioapic_abi_msi_start; /* NOTE: for testing only */ 546 547 struct ioapic_irqinfo ioapic_irqs[IOAPIC_HWI_VECTORS]; 548 549 static void 550 ioapic_abi_intr_enable(int irq) 551 { 552 const struct ioapic_irqmap *map; 553 554 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 555 ("ioapic enable, invalid irq %d", irq)); 556 557 map = &ioapic_irqmaps[mycpuid][irq]; 558 KASSERT(IOAPIC_IMT_ISHWI(map), 559 ("ioapic enable, not hwi irq %d, type %d, cpu%d", 560 irq, map->im_type, mycpuid)); 561 if (map->im_type != IOAPIC_IMT_LEGACY) 562 return; 563 564 IOAPIC_INTREN(irq); 565 } 566 567 static void 568 ioapic_abi_intr_disable(int irq) 569 { 570 const struct ioapic_irqmap *map; 571 572 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 573 ("ioapic disable, invalid irq %d", irq)); 574 575 map = &ioapic_irqmaps[mycpuid][irq]; 576 KASSERT(IOAPIC_IMT_ISHWI(map), 577 ("ioapic disable, not hwi irq %d, type %d, cpu%d", 578 irq, map->im_type, mycpuid)); 579 if (map->im_type != IOAPIC_IMT_LEGACY) 580 return; 581 582 IOAPIC_INTRDIS(irq); 583 } 584 585 static void 586 ioapic_abi_finalize(void) 587 { 588 KKASSERT(MachIntrABI.type == MACHINTR_IOAPIC); 589 KKASSERT(ioapic_enable); 590 591 /* 592 * If an IMCR is present, program bit 0 to disconnect the 8259 593 * from the BSP. 594 */ 595 if (imcr_present) { 596 outb(0x22, 0x70); /* select IMCR */ 597 outb(0x23, 0x01); /* disconnect 8259 */ 598 } 599 } 600 601 /* 602 * This routine is called after physical interrupts are enabled but before 603 * the critical section is released. We need to clean out any interrupts 604 * that had already been posted to the cpu. 605 */ 606 static void 607 ioapic_abi_cleanup(void) 608 { 609 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending)); 610 } 611 612 /* Must never be called */ 613 static void 614 ioapic_abi_stabilize(void) 615 { 616 panic("ioapic_stabilize is called"); 617 } 618 619 static void 620 ioapic_abi_intr_setup(int intr, int flags) 621 { 622 const struct ioapic_irqmap *map; 623 int vector, select; 624 uint32_t value; 625 register_t ef; 626 627 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 628 ("ioapic setup, invalid irq %d", intr)); 629 630 map = &ioapic_irqmaps[mycpuid][intr]; 631 KASSERT(IOAPIC_IMT_ISHWI(map), 632 ("ioapic setup, not hwi irq %d, type %d, cpu%d", 633 intr, map->im_type, mycpuid)); 634 if (map->im_type != IOAPIC_IMT_LEGACY) 635 return; 636 637 KASSERT(ioapic_irqs[intr].io_addr != NULL, 638 ("ioapic setup, no GSI information, irq %d", intr)); 639 640 ef = read_rflags(); 641 cpu_disable_intr(); 642 643 vector = IDT_OFFSET + intr; 644 645 /* 646 * Now reprogram the vector in the IO APIC. In order to avoid 647 * losing an EOI for a level interrupt, which is vector based, 648 * make sure that the IO APIC is programmed for edge-triggering 649 * first, then reprogrammed with the new vector. This should 650 * clear the IRR bit. 651 */ 652 imen_lock(); 653 654 select = ioapic_irqs[intr].io_idx; 655 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 656 value |= IOART_INTMSET; 657 658 ioapic_write(ioapic_irqs[intr].io_addr, select, 659 (value & ~APIC_TRIGMOD_MASK)); 660 ioapic_write(ioapic_irqs[intr].io_addr, select, 661 (value & ~IOART_INTVEC) | vector); 662 663 imen_unlock(); 664 665 IOAPIC_INTREN(intr); 666 667 write_rflags(ef); 668 } 669 670 static void 671 ioapic_abi_intr_teardown(int intr) 672 { 673 const struct ioapic_irqmap *map; 674 int vector, select; 675 uint32_t value; 676 register_t ef; 677 678 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 679 ("ioapic teardown, invalid irq %d", intr)); 680 681 map = &ioapic_irqmaps[mycpuid][intr]; 682 KASSERT(IOAPIC_IMT_ISHWI(map), 683 ("ioapic teardown, not hwi irq %d, type %d, cpu%d", 684 intr, map->im_type, mycpuid)); 685 if (map->im_type != IOAPIC_IMT_LEGACY) 686 return; 687 688 KASSERT(ioapic_irqs[intr].io_addr != NULL, 689 ("ioapic teardown, no GSI information, irq %d", intr)); 690 691 ef = read_rflags(); 692 cpu_disable_intr(); 693 694 /* 695 * Teardown an interrupt vector. The vector should already be 696 * installed in the cpu's IDT, but make sure. 697 */ 698 IOAPIC_INTRDIS(intr); 699 700 vector = IDT_OFFSET + intr; 701 702 /* 703 * In order to avoid losing an EOI for a level interrupt, which 704 * is vector based, make sure that the IO APIC is programmed for 705 * edge-triggering first, then reprogrammed with the new vector. 706 * This should clear the IRR bit. 707 */ 708 imen_lock(); 709 710 select = ioapic_irqs[intr].io_idx; 711 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 712 713 ioapic_write(ioapic_irqs[intr].io_addr, select, 714 (value & ~APIC_TRIGMOD_MASK)); 715 ioapic_write(ioapic_irqs[intr].io_addr, select, 716 (value & ~IOART_INTVEC) | vector); 717 718 imen_unlock(); 719 720 write_rflags(ef); 721 } 722 723 static void 724 ioapic_abi_setdefault(void) 725 { 726 int intr; 727 728 for (intr = 0; intr < IOAPIC_HWI_VECTORS; ++intr) { 729 if (intr == IOAPIC_HWI_SYSCALL) 730 continue; 731 setidt_global(IDT_OFFSET + intr, ioapic_intr[intr], 732 SDT_SYSIGT, SEL_KPL, 0); 733 } 734 } 735 736 static void 737 ioapic_abi_initmap(void) 738 { 739 int cpu; 740 741 kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance); 742 743 kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start); 744 ioapic_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */ 745 746 /* 747 * NOTE: ncpus is not ready yet 748 */ 749 for (cpu = 0; cpu < MAXCPU; ++cpu) { 750 int i; 751 752 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 753 ioapic_irqmaps[cpu][i].im_gsi = -1; 754 ioapic_irqmaps[cpu][i].im_msi_base = -1; 755 } 756 ioapic_irqmaps[cpu][IOAPIC_HWI_SYSCALL].im_type = 757 IOAPIC_IMT_SYSCALL; 758 } 759 } 760 761 void 762 ioapic_set_legacy_irqmap(int irq, int gsi, enum intr_trigger trig, 763 enum intr_polarity pola) 764 { 765 struct ioapic_irqinfo *info; 766 struct ioapic_irqmap *map; 767 void *ioaddr; 768 int pin, cpuid; 769 770 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 771 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 772 773 KKASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS); 774 if (irq > ioapic_abi_legacy_irq_max) 775 ioapic_abi_legacy_irq_max = irq; 776 777 cpuid = ioapic_abi_gsi_cpuid(irq, gsi); 778 779 map = &ioapic_irqmaps[cpuid][irq]; 780 781 KKASSERT(map->im_type == IOAPIC_IMT_UNUSED); 782 map->im_type = IOAPIC_IMT_LEGACY; 783 784 map->im_gsi = gsi; 785 map->im_trig = trig; 786 map->im_pola = pola; 787 788 if (bootverbose) { 789 kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n", 790 irq, map->im_gsi, 791 intr_str_trigger(map->im_trig), 792 intr_str_polarity(map->im_pola)); 793 } 794 795 pin = ioapic_gsi_pin(map->im_gsi); 796 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 797 798 info = &ioapic_irqs[irq]; 799 800 imen_lock(); 801 802 info->io_addr = ioaddr; 803 info->io_idx = IOAPIC_REDTBL + (2 * pin); 804 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 805 if (map->im_trig == INTR_TRIGGER_LEVEL) 806 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 807 808 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 809 map->im_trig, map->im_pola, cpuid); 810 811 imen_unlock(); 812 } 813 814 void 815 ioapic_fixup_legacy_irqmaps(void) 816 { 817 int cpu; 818 819 for (cpu = 0; cpu < ncpus; ++cpu) { 820 int i; 821 822 for (i = 0; i < ISA_IRQ_CNT; ++i) { 823 struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][i]; 824 825 if (map->im_type == IOAPIC_IMT_UNUSED) { 826 map->im_type = IOAPIC_IMT_RESERVED; 827 if (bootverbose) { 828 kprintf("IOAPIC: " 829 "cpu%d irq %d reserved\n", cpu, i); 830 } 831 } 832 } 833 } 834 835 ioapic_abi_legacy_irq_max += 1; 836 if (bootverbose) { 837 kprintf("IOAPIC: legacy irq max %d\n", 838 ioapic_abi_legacy_irq_max); 839 } 840 } 841 842 int 843 ioapic_find_legacy_by_gsi(int gsi, enum intr_trigger trig, 844 enum intr_polarity pola) 845 { 846 int cpu; 847 848 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 849 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 850 851 for (cpu = 0; cpu < ncpus; ++cpu) { 852 int irq; 853 854 for (irq = 0; irq < ioapic_abi_legacy_irq_max; ++irq) { 855 const struct ioapic_irqmap *map = 856 &ioapic_irqmaps[cpu][irq]; 857 858 if (map->im_gsi == gsi) { 859 KKASSERT(map->im_type == IOAPIC_IMT_LEGACY); 860 861 if (map->im_flags & IOAPIC_IMF_CONF) { 862 if (map->im_trig != trig || 863 map->im_pola != pola) 864 return -1; 865 } 866 return irq; 867 } 868 } 869 } 870 return -1; 871 } 872 873 int 874 ioapic_find_legacy_by_irq(int irq, enum intr_trigger trig, 875 enum intr_polarity pola) 876 { 877 int cpu; 878 879 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 880 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 881 882 if (irq < 0 || irq >= ioapic_abi_legacy_irq_max) 883 return -1; 884 885 for (cpu = 0; cpu < ncpus; ++cpu) { 886 const struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][irq]; 887 888 if (map->im_type == IOAPIC_IMT_LEGACY) { 889 if (map->im_flags & IOAPIC_IMF_CONF) { 890 if (map->im_trig != trig || 891 map->im_pola != pola) 892 return -1; 893 } 894 return irq; 895 } 896 } 897 return -1; 898 } 899 900 static void 901 ioapic_abi_legacy_intr_config(int irq, enum intr_trigger trig, 902 enum intr_polarity pola) 903 { 904 struct ioapic_irqinfo *info; 905 struct ioapic_irqmap *map = NULL; 906 void *ioaddr; 907 int pin, cpuid; 908 909 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 910 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 911 912 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 913 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 914 map = &ioapic_irqmaps[cpuid][irq]; 915 if (map->im_type == IOAPIC_IMT_LEGACY) 916 break; 917 } 918 KKASSERT(cpuid < ncpus); 919 920 #ifdef notyet 921 if (map->im_flags & IOAPIC_IMF_CONF) { 922 if (trig != map->im_trig) { 923 panic("ioapic_intr_config: trig %s -> %s", 924 intr_str_trigger(map->im_trig), 925 intr_str_trigger(trig)); 926 } 927 if (pola != map->im_pola) { 928 panic("ioapic_intr_config: pola %s -> %s", 929 intr_str_polarity(map->im_pola), 930 intr_str_polarity(pola)); 931 } 932 return; 933 } 934 #endif 935 map->im_flags |= IOAPIC_IMF_CONF; 936 937 if (trig == map->im_trig && pola == map->im_pola) 938 return; 939 940 if (bootverbose) { 941 kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n", 942 irq, map->im_gsi, 943 intr_str_trigger(map->im_trig), 944 intr_str_polarity(map->im_pola), 945 intr_str_trigger(trig), 946 intr_str_polarity(pola)); 947 } 948 map->im_trig = trig; 949 map->im_pola = pola; 950 951 pin = ioapic_gsi_pin(map->im_gsi); 952 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 953 954 info = &ioapic_irqs[irq]; 955 956 imen_lock(); 957 958 info->io_flags &= ~IOAPIC_IRQI_FLAG_LEVEL; 959 if (map->im_trig == INTR_TRIGGER_LEVEL) 960 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 961 962 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 963 map->im_trig, map->im_pola, cpuid); 964 965 imen_unlock(); 966 } 967 968 int 969 ioapic_conf_legacy_extint(int irq) 970 { 971 struct ioapic_irqinfo *info; 972 struct ioapic_irqmap *map; 973 void *ioaddr; 974 int pin, error, vec; 975 976 /* XXX only irq0 is allowed */ 977 KKASSERT(irq == 0); 978 979 vec = IDT_OFFSET + irq; 980 981 if (ioapic_abi_extint_irq == irq) 982 return 0; 983 else if (ioapic_abi_extint_irq >= 0) 984 return EEXIST; 985 986 error = icu_ioapic_extint(irq, vec); 987 if (error) 988 return error; 989 990 /* ExtINT is always targeted to cpu0 */ 991 map = &ioapic_irqmaps[0][irq]; 992 993 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED || 994 map->im_type == IOAPIC_IMT_LEGACY); 995 if (map->im_type == IOAPIC_IMT_LEGACY) { 996 if (map->im_flags & IOAPIC_IMF_CONF) 997 return EEXIST; 998 } 999 ioapic_abi_extint_irq = irq; 1000 1001 map->im_type = IOAPIC_IMT_LEGACY; 1002 map->im_trig = INTR_TRIGGER_EDGE; 1003 map->im_pola = INTR_POLARITY_HIGH; 1004 map->im_flags = IOAPIC_IMF_CONF; 1005 1006 map->im_gsi = ioapic_extpin_gsi(); 1007 KKASSERT(map->im_gsi >= 0); 1008 1009 if (bootverbose) { 1010 kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n", 1011 irq, map->im_gsi, 1012 intr_str_trigger(map->im_trig), 1013 intr_str_polarity(map->im_pola)); 1014 } 1015 1016 pin = ioapic_gsi_pin(map->im_gsi); 1017 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1018 1019 info = &ioapic_irqs[irq]; 1020 1021 imen_lock(); 1022 1023 info->io_addr = ioaddr; 1024 info->io_idx = IOAPIC_REDTBL + (2 * pin); 1025 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 1026 1027 ioapic_extpin_setup(ioaddr, pin, vec); 1028 1029 imen_unlock(); 1030 1031 return 0; 1032 } 1033 1034 static int 1035 ioapic_abi_legacy_intr_cpuid(int irq) 1036 { 1037 const struct ioapic_irqmap *map = NULL; 1038 int cpuid; 1039 1040 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 1041 1042 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1043 map = &ioapic_irqmaps[cpuid][irq]; 1044 if (map->im_type == IOAPIC_IMT_LEGACY) 1045 return cpuid; 1046 } 1047 1048 /* XXX some drivers tries to peek at reserved IRQs */ 1049 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1050 map = &ioapic_irqmaps[cpuid][irq]; 1051 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED); 1052 } 1053 return 0; 1054 } 1055 1056 static int 1057 ioapic_abi_gsi_cpuid(int irq, int gsi) 1058 { 1059 char envpath[32]; 1060 int cpuid = -1; 1061 1062 KKASSERT(gsi >= 0); 1063 1064 if (irq == 0 || gsi == 0) { 1065 if (bootverbose) { 1066 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n", 1067 irq, gsi); 1068 } 1069 return 0; 1070 } 1071 1072 if (irq == acpi_sci_irqno()) { 1073 if (bootverbose) { 1074 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n", 1075 irq, gsi); 1076 } 1077 return 0; 1078 } 1079 1080 ksnprintf(envpath, sizeof(envpath), "hw.ioapic.gsi.%d.cpu", gsi); 1081 kgetenv_int(envpath, &cpuid); 1082 1083 if (cpuid < 0) { 1084 if (!ioapic_abi_gsi_balance) { 1085 if (bootverbose) { 1086 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 " 1087 "(fixed)\n", irq, gsi); 1088 } 1089 return 0; 1090 } 1091 1092 cpuid = gsi % ncpus; 1093 if (bootverbose) { 1094 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n", 1095 irq, gsi, cpuid); 1096 } 1097 } else if (cpuid >= ncpus) { 1098 cpuid = ncpus - 1; 1099 if (bootverbose) { 1100 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n", 1101 irq, gsi, cpuid); 1102 } 1103 } else { 1104 if (bootverbose) { 1105 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n", 1106 irq, gsi, cpuid); 1107 } 1108 } 1109 return cpuid; 1110 } 1111 1112 static void 1113 ioapic_abi_rman_setup(struct rman *rm) 1114 { 1115 int start, end, i; 1116 1117 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU, 1118 ("invalid rman cpuid %d", rm->rm_cpuid)); 1119 1120 start = end = -1; 1121 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 1122 const struct ioapic_irqmap *map = 1123 &ioapic_irqmaps[rm->rm_cpuid][i]; 1124 1125 if (start < 0) { 1126 if (IOAPIC_IMT_ISHWI(map)) 1127 start = end = i; 1128 } else { 1129 if (IOAPIC_IMT_ISHWI(map)) { 1130 end = i; 1131 } else { 1132 KKASSERT(end >= 0); 1133 if (bootverbose) { 1134 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1135 rm->rm_cpuid, start, end); 1136 } 1137 if (rman_manage_region(rm, start, end)) { 1138 panic("rman_manage_region" 1139 "(cpu%d %d - %d)", rm->rm_cpuid, 1140 start, end); 1141 } 1142 start = end = -1; 1143 } 1144 } 1145 } 1146 if (start >= 0) { 1147 KKASSERT(end >= 0); 1148 if (bootverbose) { 1149 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1150 rm->rm_cpuid, start, end); 1151 } 1152 if (rman_manage_region(rm, start, end)) { 1153 panic("rman_manage_region(cpu%d %d - %d)", 1154 rm->rm_cpuid, start, end); 1155 } 1156 } 1157 } 1158 1159 static int 1160 ioapic_abi_msi_alloc_intern(int type, const char *desc, 1161 int intrs[], int count, int cpuid) 1162 { 1163 int i, error; 1164 1165 KASSERT(cpuid >= 0 && cpuid < ncpus, 1166 ("invalid cpuid %d", cpuid)); 1167 1168 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 1169 KASSERT((count & (count - 1)) == 0, 1170 ("count %d is not power of 2", count)); 1171 1172 lwkt_gettoken(&ioapic_irqmap_tok); 1173 1174 /* 1175 * NOTE: 1176 * Since IDT_OFFSET is 32, which is the maximum valid 'count', 1177 * we do not need to find out the first properly aligned 1178 * interrupt vector. 1179 */ 1180 1181 error = EMSGSIZE; 1182 for (i = ioapic_abi_msi_start; i < IOAPIC_HWI_VECTORS; i += count) { 1183 int j; 1184 1185 if (ioapic_irqmaps[cpuid][i].im_type != IOAPIC_IMT_UNUSED) 1186 continue; 1187 1188 for (j = 1; j < count; ++j) { 1189 if (ioapic_irqmaps[cpuid][i + j].im_type != 1190 IOAPIC_IMT_UNUSED) 1191 break; 1192 } 1193 if (j != count) 1194 continue; 1195 1196 for (j = 0; j < count; ++j) { 1197 struct ioapic_irqmap *map; 1198 int intr = i + j; 1199 1200 map = &ioapic_irqmaps[cpuid][intr]; 1201 KASSERT(map->im_msi_base < 0, 1202 ("intr %d, stale %s-base %d", 1203 intr, desc, map->im_msi_base)); 1204 1205 map->im_type = type; 1206 map->im_msi_base = i; 1207 1208 intrs[j] = intr; 1209 msi_setup(intr, cpuid); 1210 1211 if (bootverbose) { 1212 kprintf("alloc %s intr %d on cpu%d\n", 1213 desc, intr, cpuid); 1214 } 1215 } 1216 error = 0; 1217 break; 1218 } 1219 1220 lwkt_reltoken(&ioapic_irqmap_tok); 1221 1222 return error; 1223 } 1224 1225 static void 1226 ioapic_abi_msi_release_intern(int type, const char *desc, 1227 const int intrs[], int count, int cpuid) 1228 { 1229 int i, msi_base = -1, intr_next = -1, mask; 1230 1231 KASSERT(cpuid >= 0 && cpuid < ncpus, 1232 ("invalid cpuid %d", cpuid)); 1233 1234 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 1235 1236 mask = count - 1; 1237 KASSERT((count & mask) == 0, ("count %d is not power of 2", count)); 1238 1239 lwkt_gettoken(&ioapic_irqmap_tok); 1240 1241 for (i = 0; i < count; ++i) { 1242 struct ioapic_irqmap *map; 1243 int intr = intrs[i]; 1244 1245 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1246 ("invalid intr %d", intr)); 1247 1248 map = &ioapic_irqmaps[cpuid][intr]; 1249 KASSERT(map->im_type == type, 1250 ("trying to release non-%s intr %d, type %d", desc, 1251 intr, map->im_type)); 1252 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1253 ("intr %d, invalid %s-base %d", intr, desc, 1254 map->im_msi_base)); 1255 KASSERT((map->im_msi_base & mask) == 0, 1256 ("intr %d, %s-base %d is not properly aligned %d", 1257 intr, desc, map->im_msi_base, count)); 1258 1259 if (msi_base < 0) { 1260 msi_base = map->im_msi_base; 1261 } else { 1262 KASSERT(map->im_msi_base == msi_base, 1263 ("intr %d, inconsistent %s-base, " 1264 "was %d, now %d", 1265 intr, desc, msi_base, map->im_msi_base)); 1266 } 1267 1268 if (intr_next < intr) 1269 intr_next = intr; 1270 1271 map->im_type = IOAPIC_IMT_UNUSED; 1272 map->im_msi_base = -1; 1273 1274 if (bootverbose) { 1275 kprintf("release %s intr %d on cpu%d\n", 1276 desc, intr, cpuid); 1277 } 1278 } 1279 1280 KKASSERT(intr_next > 0); 1281 KKASSERT(msi_base >= 0); 1282 1283 ++intr_next; 1284 if (intr_next < IOAPIC_HWI_VECTORS) { 1285 const struct ioapic_irqmap *map = 1286 &ioapic_irqmaps[cpuid][intr_next]; 1287 1288 if (map->im_type == type) { 1289 KASSERT(map->im_msi_base != msi_base, 1290 ("more than %d %s was allocated", count, desc)); 1291 } 1292 } 1293 1294 lwkt_reltoken(&ioapic_irqmap_tok); 1295 } 1296 1297 static int 1298 ioapic_abi_msi_alloc(int intrs[], int count, int cpuid) 1299 { 1300 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSI, "MSI", 1301 intrs, count, cpuid); 1302 } 1303 1304 static void 1305 ioapic_abi_msi_release(const int intrs[], int count, int cpuid) 1306 { 1307 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSI, "MSI", 1308 intrs, count, cpuid); 1309 } 1310 1311 static int 1312 ioapic_abi_msix_alloc(int *intr, int cpuid) 1313 { 1314 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSIX, "MSI-X", 1315 intr, 1, cpuid); 1316 } 1317 1318 static void 1319 ioapic_abi_msix_release(int intr, int cpuid) 1320 { 1321 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSIX, "MSI-X", 1322 &intr, 1, cpuid); 1323 } 1324 1325 static void 1326 ioapic_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid) 1327 { 1328 const struct ioapic_irqmap *map; 1329 1330 KASSERT(cpuid >= 0 && cpuid < ncpus, 1331 ("invalid cpuid %d", cpuid)); 1332 1333 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1334 ("invalid intr %d", intr)); 1335 1336 lwkt_gettoken(&ioapic_irqmap_tok); 1337 1338 map = &ioapic_irqmaps[cpuid][intr]; 1339 KASSERT(map->im_type == IOAPIC_IMT_MSI || 1340 map->im_type == IOAPIC_IMT_MSIX, 1341 ("trying to map non-MSI/MSI-X intr %d, type %d", intr, map->im_type)); 1342 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1343 ("intr %d, invalid %s-base %d", intr, 1344 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1345 map->im_msi_base)); 1346 1347 msi_map(map->im_msi_base, addr, data, cpuid); 1348 1349 if (bootverbose) { 1350 kprintf("map %s intr %d on cpu%d\n", 1351 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1352 intr, cpuid); 1353 } 1354 1355 lwkt_reltoken(&ioapic_irqmap_tok); 1356 } 1357