1 /* 2 * Copyright (c) 1991 The Regents of the University of California. 3 * Copyright (c) 1996, by Steve Passe. All rights reserved. 4 * Copyright (c) 2005,2008 The DragonFly Project. All rights reserved. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The DragonFly Project 8 * by Matthew Dillon <dillon@backplane.com> 9 * 10 * This code is derived from software contributed to Berkeley by 11 * William Jolitz. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 3. Neither the name of The DragonFly Project nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific, prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 30 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 31 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 34 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 35 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 36 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 37 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/kernel.h> 44 #include <sys/machintr.h> 45 #include <sys/interrupt.h> 46 #include <sys/bus.h> 47 #include <sys/rman.h> 48 49 #include <machine/smp.h> 50 #include <machine/segments.h> 51 #include <machine/md_var.h> 52 #include <machine/intr_machdep.h> 53 #include <machine/globaldata.h> 54 #include <machine/msi_var.h> 55 56 #include <machine_base/isa/isa_intr.h> 57 #include <machine_base/icu/icu.h> 58 #include <machine_base/icu/icu_var.h> 59 #include <machine_base/apic/ioapic.h> 60 #include <machine_base/apic/ioapic_abi.h> 61 #include <machine_base/apic/ioapic_ipl.h> 62 #include <machine_base/apic/apicreg.h> 63 64 #include <dev/acpica/acpi_sci_var.h> 65 66 #define IOAPIC_HWI_VECTORS IDT_HWI_VECTORS 67 68 extern inthand_t 69 IDTVEC(ioapic_intr0), 70 IDTVEC(ioapic_intr1), 71 IDTVEC(ioapic_intr2), 72 IDTVEC(ioapic_intr3), 73 IDTVEC(ioapic_intr4), 74 IDTVEC(ioapic_intr5), 75 IDTVEC(ioapic_intr6), 76 IDTVEC(ioapic_intr7), 77 IDTVEC(ioapic_intr8), 78 IDTVEC(ioapic_intr9), 79 IDTVEC(ioapic_intr10), 80 IDTVEC(ioapic_intr11), 81 IDTVEC(ioapic_intr12), 82 IDTVEC(ioapic_intr13), 83 IDTVEC(ioapic_intr14), 84 IDTVEC(ioapic_intr15), 85 IDTVEC(ioapic_intr16), 86 IDTVEC(ioapic_intr17), 87 IDTVEC(ioapic_intr18), 88 IDTVEC(ioapic_intr19), 89 IDTVEC(ioapic_intr20), 90 IDTVEC(ioapic_intr21), 91 IDTVEC(ioapic_intr22), 92 IDTVEC(ioapic_intr23), 93 IDTVEC(ioapic_intr24), 94 IDTVEC(ioapic_intr25), 95 IDTVEC(ioapic_intr26), 96 IDTVEC(ioapic_intr27), 97 IDTVEC(ioapic_intr28), 98 IDTVEC(ioapic_intr29), 99 IDTVEC(ioapic_intr30), 100 IDTVEC(ioapic_intr31), 101 IDTVEC(ioapic_intr32), 102 IDTVEC(ioapic_intr33), 103 IDTVEC(ioapic_intr34), 104 IDTVEC(ioapic_intr35), 105 IDTVEC(ioapic_intr36), 106 IDTVEC(ioapic_intr37), 107 IDTVEC(ioapic_intr38), 108 IDTVEC(ioapic_intr39), 109 IDTVEC(ioapic_intr40), 110 IDTVEC(ioapic_intr41), 111 IDTVEC(ioapic_intr42), 112 IDTVEC(ioapic_intr43), 113 IDTVEC(ioapic_intr44), 114 IDTVEC(ioapic_intr45), 115 IDTVEC(ioapic_intr46), 116 IDTVEC(ioapic_intr47), 117 IDTVEC(ioapic_intr48), 118 IDTVEC(ioapic_intr49), 119 IDTVEC(ioapic_intr50), 120 IDTVEC(ioapic_intr51), 121 IDTVEC(ioapic_intr52), 122 IDTVEC(ioapic_intr53), 123 IDTVEC(ioapic_intr54), 124 IDTVEC(ioapic_intr55), 125 IDTVEC(ioapic_intr56), 126 IDTVEC(ioapic_intr57), 127 IDTVEC(ioapic_intr58), 128 IDTVEC(ioapic_intr59), 129 IDTVEC(ioapic_intr60), 130 IDTVEC(ioapic_intr61), 131 IDTVEC(ioapic_intr62), 132 IDTVEC(ioapic_intr63), 133 IDTVEC(ioapic_intr64), 134 IDTVEC(ioapic_intr65), 135 IDTVEC(ioapic_intr66), 136 IDTVEC(ioapic_intr67), 137 IDTVEC(ioapic_intr68), 138 IDTVEC(ioapic_intr69), 139 IDTVEC(ioapic_intr70), 140 IDTVEC(ioapic_intr71), 141 IDTVEC(ioapic_intr72), 142 IDTVEC(ioapic_intr73), 143 IDTVEC(ioapic_intr74), 144 IDTVEC(ioapic_intr75), 145 IDTVEC(ioapic_intr76), 146 IDTVEC(ioapic_intr77), 147 IDTVEC(ioapic_intr78), 148 IDTVEC(ioapic_intr79), 149 IDTVEC(ioapic_intr80), 150 IDTVEC(ioapic_intr81), 151 IDTVEC(ioapic_intr82), 152 IDTVEC(ioapic_intr83), 153 IDTVEC(ioapic_intr84), 154 IDTVEC(ioapic_intr85), 155 IDTVEC(ioapic_intr86), 156 IDTVEC(ioapic_intr87), 157 IDTVEC(ioapic_intr88), 158 IDTVEC(ioapic_intr89), 159 IDTVEC(ioapic_intr90), 160 IDTVEC(ioapic_intr91), 161 IDTVEC(ioapic_intr92), 162 IDTVEC(ioapic_intr93), 163 IDTVEC(ioapic_intr94), 164 IDTVEC(ioapic_intr95), 165 IDTVEC(ioapic_intr96), 166 IDTVEC(ioapic_intr97), 167 IDTVEC(ioapic_intr98), 168 IDTVEC(ioapic_intr99), 169 IDTVEC(ioapic_intr100), 170 IDTVEC(ioapic_intr101), 171 IDTVEC(ioapic_intr102), 172 IDTVEC(ioapic_intr103), 173 IDTVEC(ioapic_intr104), 174 IDTVEC(ioapic_intr105), 175 IDTVEC(ioapic_intr106), 176 IDTVEC(ioapic_intr107), 177 IDTVEC(ioapic_intr108), 178 IDTVEC(ioapic_intr109), 179 IDTVEC(ioapic_intr110), 180 IDTVEC(ioapic_intr111), 181 IDTVEC(ioapic_intr112), 182 IDTVEC(ioapic_intr113), 183 IDTVEC(ioapic_intr114), 184 IDTVEC(ioapic_intr115), 185 IDTVEC(ioapic_intr116), 186 IDTVEC(ioapic_intr117), 187 IDTVEC(ioapic_intr118), 188 IDTVEC(ioapic_intr119), 189 IDTVEC(ioapic_intr120), 190 IDTVEC(ioapic_intr121), 191 IDTVEC(ioapic_intr122), 192 IDTVEC(ioapic_intr123), 193 IDTVEC(ioapic_intr124), 194 IDTVEC(ioapic_intr125), 195 IDTVEC(ioapic_intr126), 196 IDTVEC(ioapic_intr127), 197 IDTVEC(ioapic_intr128), 198 IDTVEC(ioapic_intr129), 199 IDTVEC(ioapic_intr130), 200 IDTVEC(ioapic_intr131), 201 IDTVEC(ioapic_intr132), 202 IDTVEC(ioapic_intr133), 203 IDTVEC(ioapic_intr134), 204 IDTVEC(ioapic_intr135), 205 IDTVEC(ioapic_intr136), 206 IDTVEC(ioapic_intr137), 207 IDTVEC(ioapic_intr138), 208 IDTVEC(ioapic_intr139), 209 IDTVEC(ioapic_intr140), 210 IDTVEC(ioapic_intr141), 211 IDTVEC(ioapic_intr142), 212 IDTVEC(ioapic_intr143), 213 IDTVEC(ioapic_intr144), 214 IDTVEC(ioapic_intr145), 215 IDTVEC(ioapic_intr146), 216 IDTVEC(ioapic_intr147), 217 IDTVEC(ioapic_intr148), 218 IDTVEC(ioapic_intr149), 219 IDTVEC(ioapic_intr150), 220 IDTVEC(ioapic_intr151), 221 IDTVEC(ioapic_intr152), 222 IDTVEC(ioapic_intr153), 223 IDTVEC(ioapic_intr154), 224 IDTVEC(ioapic_intr155), 225 IDTVEC(ioapic_intr156), 226 IDTVEC(ioapic_intr157), 227 IDTVEC(ioapic_intr158), 228 IDTVEC(ioapic_intr159), 229 IDTVEC(ioapic_intr160), 230 IDTVEC(ioapic_intr161), 231 IDTVEC(ioapic_intr162), 232 IDTVEC(ioapic_intr163), 233 IDTVEC(ioapic_intr164), 234 IDTVEC(ioapic_intr165), 235 IDTVEC(ioapic_intr166), 236 IDTVEC(ioapic_intr167), 237 IDTVEC(ioapic_intr168), 238 IDTVEC(ioapic_intr169), 239 IDTVEC(ioapic_intr170), 240 IDTVEC(ioapic_intr171), 241 IDTVEC(ioapic_intr172), 242 IDTVEC(ioapic_intr173), 243 IDTVEC(ioapic_intr174), 244 IDTVEC(ioapic_intr175), 245 IDTVEC(ioapic_intr176), 246 IDTVEC(ioapic_intr177), 247 IDTVEC(ioapic_intr178), 248 IDTVEC(ioapic_intr179), 249 IDTVEC(ioapic_intr180), 250 IDTVEC(ioapic_intr181), 251 IDTVEC(ioapic_intr182), 252 IDTVEC(ioapic_intr183), 253 IDTVEC(ioapic_intr184), 254 IDTVEC(ioapic_intr185), 255 IDTVEC(ioapic_intr186), 256 IDTVEC(ioapic_intr187), 257 IDTVEC(ioapic_intr188), 258 IDTVEC(ioapic_intr189), 259 IDTVEC(ioapic_intr190), 260 IDTVEC(ioapic_intr191); 261 262 static inthand_t *ioapic_intr[IOAPIC_HWI_VECTORS] = { 263 &IDTVEC(ioapic_intr0), 264 &IDTVEC(ioapic_intr1), 265 &IDTVEC(ioapic_intr2), 266 &IDTVEC(ioapic_intr3), 267 &IDTVEC(ioapic_intr4), 268 &IDTVEC(ioapic_intr5), 269 &IDTVEC(ioapic_intr6), 270 &IDTVEC(ioapic_intr7), 271 &IDTVEC(ioapic_intr8), 272 &IDTVEC(ioapic_intr9), 273 &IDTVEC(ioapic_intr10), 274 &IDTVEC(ioapic_intr11), 275 &IDTVEC(ioapic_intr12), 276 &IDTVEC(ioapic_intr13), 277 &IDTVEC(ioapic_intr14), 278 &IDTVEC(ioapic_intr15), 279 &IDTVEC(ioapic_intr16), 280 &IDTVEC(ioapic_intr17), 281 &IDTVEC(ioapic_intr18), 282 &IDTVEC(ioapic_intr19), 283 &IDTVEC(ioapic_intr20), 284 &IDTVEC(ioapic_intr21), 285 &IDTVEC(ioapic_intr22), 286 &IDTVEC(ioapic_intr23), 287 &IDTVEC(ioapic_intr24), 288 &IDTVEC(ioapic_intr25), 289 &IDTVEC(ioapic_intr26), 290 &IDTVEC(ioapic_intr27), 291 &IDTVEC(ioapic_intr28), 292 &IDTVEC(ioapic_intr29), 293 &IDTVEC(ioapic_intr30), 294 &IDTVEC(ioapic_intr31), 295 &IDTVEC(ioapic_intr32), 296 &IDTVEC(ioapic_intr33), 297 &IDTVEC(ioapic_intr34), 298 &IDTVEC(ioapic_intr35), 299 &IDTVEC(ioapic_intr36), 300 &IDTVEC(ioapic_intr37), 301 &IDTVEC(ioapic_intr38), 302 &IDTVEC(ioapic_intr39), 303 &IDTVEC(ioapic_intr40), 304 &IDTVEC(ioapic_intr41), 305 &IDTVEC(ioapic_intr42), 306 &IDTVEC(ioapic_intr43), 307 &IDTVEC(ioapic_intr44), 308 &IDTVEC(ioapic_intr45), 309 &IDTVEC(ioapic_intr46), 310 &IDTVEC(ioapic_intr47), 311 &IDTVEC(ioapic_intr48), 312 &IDTVEC(ioapic_intr49), 313 &IDTVEC(ioapic_intr50), 314 &IDTVEC(ioapic_intr51), 315 &IDTVEC(ioapic_intr52), 316 &IDTVEC(ioapic_intr53), 317 &IDTVEC(ioapic_intr54), 318 &IDTVEC(ioapic_intr55), 319 &IDTVEC(ioapic_intr56), 320 &IDTVEC(ioapic_intr57), 321 &IDTVEC(ioapic_intr58), 322 &IDTVEC(ioapic_intr59), 323 &IDTVEC(ioapic_intr60), 324 &IDTVEC(ioapic_intr61), 325 &IDTVEC(ioapic_intr62), 326 &IDTVEC(ioapic_intr63), 327 &IDTVEC(ioapic_intr64), 328 &IDTVEC(ioapic_intr65), 329 &IDTVEC(ioapic_intr66), 330 &IDTVEC(ioapic_intr67), 331 &IDTVEC(ioapic_intr68), 332 &IDTVEC(ioapic_intr69), 333 &IDTVEC(ioapic_intr70), 334 &IDTVEC(ioapic_intr71), 335 &IDTVEC(ioapic_intr72), 336 &IDTVEC(ioapic_intr73), 337 &IDTVEC(ioapic_intr74), 338 &IDTVEC(ioapic_intr75), 339 &IDTVEC(ioapic_intr76), 340 &IDTVEC(ioapic_intr77), 341 &IDTVEC(ioapic_intr78), 342 &IDTVEC(ioapic_intr79), 343 &IDTVEC(ioapic_intr80), 344 &IDTVEC(ioapic_intr81), 345 &IDTVEC(ioapic_intr82), 346 &IDTVEC(ioapic_intr83), 347 &IDTVEC(ioapic_intr84), 348 &IDTVEC(ioapic_intr85), 349 &IDTVEC(ioapic_intr86), 350 &IDTVEC(ioapic_intr87), 351 &IDTVEC(ioapic_intr88), 352 &IDTVEC(ioapic_intr89), 353 &IDTVEC(ioapic_intr90), 354 &IDTVEC(ioapic_intr91), 355 &IDTVEC(ioapic_intr92), 356 &IDTVEC(ioapic_intr93), 357 &IDTVEC(ioapic_intr94), 358 &IDTVEC(ioapic_intr95), 359 &IDTVEC(ioapic_intr96), 360 &IDTVEC(ioapic_intr97), 361 &IDTVEC(ioapic_intr98), 362 &IDTVEC(ioapic_intr99), 363 &IDTVEC(ioapic_intr100), 364 &IDTVEC(ioapic_intr101), 365 &IDTVEC(ioapic_intr102), 366 &IDTVEC(ioapic_intr103), 367 &IDTVEC(ioapic_intr104), 368 &IDTVEC(ioapic_intr105), 369 &IDTVEC(ioapic_intr106), 370 &IDTVEC(ioapic_intr107), 371 &IDTVEC(ioapic_intr108), 372 &IDTVEC(ioapic_intr109), 373 &IDTVEC(ioapic_intr110), 374 &IDTVEC(ioapic_intr111), 375 &IDTVEC(ioapic_intr112), 376 &IDTVEC(ioapic_intr113), 377 &IDTVEC(ioapic_intr114), 378 &IDTVEC(ioapic_intr115), 379 &IDTVEC(ioapic_intr116), 380 &IDTVEC(ioapic_intr117), 381 &IDTVEC(ioapic_intr118), 382 &IDTVEC(ioapic_intr119), 383 &IDTVEC(ioapic_intr120), 384 &IDTVEC(ioapic_intr121), 385 &IDTVEC(ioapic_intr122), 386 &IDTVEC(ioapic_intr123), 387 &IDTVEC(ioapic_intr124), 388 &IDTVEC(ioapic_intr125), 389 &IDTVEC(ioapic_intr126), 390 &IDTVEC(ioapic_intr127), 391 &IDTVEC(ioapic_intr128), 392 &IDTVEC(ioapic_intr129), 393 &IDTVEC(ioapic_intr130), 394 &IDTVEC(ioapic_intr131), 395 &IDTVEC(ioapic_intr132), 396 &IDTVEC(ioapic_intr133), 397 &IDTVEC(ioapic_intr134), 398 &IDTVEC(ioapic_intr135), 399 &IDTVEC(ioapic_intr136), 400 &IDTVEC(ioapic_intr137), 401 &IDTVEC(ioapic_intr138), 402 &IDTVEC(ioapic_intr139), 403 &IDTVEC(ioapic_intr140), 404 &IDTVEC(ioapic_intr141), 405 &IDTVEC(ioapic_intr142), 406 &IDTVEC(ioapic_intr143), 407 &IDTVEC(ioapic_intr144), 408 &IDTVEC(ioapic_intr145), 409 &IDTVEC(ioapic_intr146), 410 &IDTVEC(ioapic_intr147), 411 &IDTVEC(ioapic_intr148), 412 &IDTVEC(ioapic_intr149), 413 &IDTVEC(ioapic_intr150), 414 &IDTVEC(ioapic_intr151), 415 &IDTVEC(ioapic_intr152), 416 &IDTVEC(ioapic_intr153), 417 &IDTVEC(ioapic_intr154), 418 &IDTVEC(ioapic_intr155), 419 &IDTVEC(ioapic_intr156), 420 &IDTVEC(ioapic_intr157), 421 &IDTVEC(ioapic_intr158), 422 &IDTVEC(ioapic_intr159), 423 &IDTVEC(ioapic_intr160), 424 &IDTVEC(ioapic_intr161), 425 &IDTVEC(ioapic_intr162), 426 &IDTVEC(ioapic_intr163), 427 &IDTVEC(ioapic_intr164), 428 &IDTVEC(ioapic_intr165), 429 &IDTVEC(ioapic_intr166), 430 &IDTVEC(ioapic_intr167), 431 &IDTVEC(ioapic_intr168), 432 &IDTVEC(ioapic_intr169), 433 &IDTVEC(ioapic_intr170), 434 &IDTVEC(ioapic_intr171), 435 &IDTVEC(ioapic_intr172), 436 &IDTVEC(ioapic_intr173), 437 &IDTVEC(ioapic_intr174), 438 &IDTVEC(ioapic_intr175), 439 &IDTVEC(ioapic_intr176), 440 &IDTVEC(ioapic_intr177), 441 &IDTVEC(ioapic_intr178), 442 &IDTVEC(ioapic_intr179), 443 &IDTVEC(ioapic_intr180), 444 &IDTVEC(ioapic_intr181), 445 &IDTVEC(ioapic_intr182), 446 &IDTVEC(ioapic_intr183), 447 &IDTVEC(ioapic_intr184), 448 &IDTVEC(ioapic_intr185), 449 &IDTVEC(ioapic_intr186), 450 &IDTVEC(ioapic_intr187), 451 &IDTVEC(ioapic_intr188), 452 &IDTVEC(ioapic_intr189), 453 &IDTVEC(ioapic_intr190), 454 &IDTVEC(ioapic_intr191) 455 }; 456 457 #define IOAPIC_HWI_SYSCALL (IDT_OFFSET_SYSCALL - IDT_OFFSET) 458 459 /* 460 * NOTE: Initialized before VM so cannot use kmalloc() for this array. 461 */ 462 static struct ioapic_irqmap { 463 int im_type; /* IOAPIC_IMT_ */ 464 enum intr_trigger im_trig; 465 enum intr_polarity im_pola; 466 int im_gsi; 467 int im_msi_base; 468 uint32_t im_flags; /* IOAPIC_IMF_ */ 469 } ioapic_irqmaps[MAXCPU][IOAPIC_HWI_VECTORS]; 470 471 static struct lwkt_token ioapic_irqmap_tok = 472 LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token); 473 474 #define IOAPIC_IMT_UNUSED 0 475 #define IOAPIC_IMT_RESERVED 1 476 #define IOAPIC_IMT_LEGACY 2 477 #define IOAPIC_IMT_SYSCALL 3 478 #define IOAPIC_IMT_MSI 4 479 #define IOAPIC_IMT_MSIX 5 480 481 #define IOAPIC_IMT_ISHWI(map) ((map)->im_type != IOAPIC_IMT_RESERVED && \ 482 (map)->im_type != IOAPIC_IMT_SYSCALL) 483 484 #define IOAPIC_IMF_CONF 0x1 485 486 extern void IOAPIC_INTREN(int); 487 extern void IOAPIC_INTRDIS(int); 488 489 extern int imcr_present; 490 491 static void ioapic_abi_intr_enable(int); 492 static void ioapic_abi_intr_disable(int); 493 static void ioapic_abi_intr_setup(int, int); 494 static void ioapic_abi_intr_teardown(int); 495 496 static void ioapic_abi_legacy_intr_config(int, 497 enum intr_trigger, enum intr_polarity); 498 static int ioapic_abi_legacy_intr_cpuid(int); 499 static int ioapic_abi_legacy_intr_find(int, 500 enum intr_trigger, enum intr_polarity); 501 static int ioapic_abi_legacy_intr_find_bygsi(int, 502 enum intr_trigger, enum intr_polarity); 503 504 static int ioapic_abi_msi_alloc(int [], int, int); 505 static void ioapic_abi_msi_release(const int [], int, int); 506 static void ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int); 507 static int ioapic_abi_msix_alloc(int *, int); 508 static void ioapic_abi_msix_release(int, int); 509 510 static int ioapic_abi_msi_alloc_intern(int, const char *, 511 int [], int, int); 512 static void ioapic_abi_msi_release_intern(int, const char *, 513 const int [], int, int); 514 515 static void ioapic_abi_finalize(void); 516 static void ioapic_abi_cleanup(void); 517 static void ioapic_abi_setdefault(void); 518 static void ioapic_abi_stabilize(void); 519 static void ioapic_abi_initmap(void); 520 static void ioapic_abi_rman_setup(struct rman *); 521 522 static int ioapic_abi_gsi_cpuid(int, int); 523 static int ioapic_unused_legacy_irqmap(void); 524 static int ioapic_is_legacy_irqmap_used(int); 525 526 struct machintr_abi MachIntrABI_IOAPIC = { 527 MACHINTR_IOAPIC, 528 .intr_disable = ioapic_abi_intr_disable, 529 .intr_enable = ioapic_abi_intr_enable, 530 .intr_setup = ioapic_abi_intr_setup, 531 .intr_teardown = ioapic_abi_intr_teardown, 532 533 .legacy_intr_config = ioapic_abi_legacy_intr_config, 534 .legacy_intr_cpuid = ioapic_abi_legacy_intr_cpuid, 535 .legacy_intr_find = ioapic_abi_legacy_intr_find, 536 .legacy_intr_find_bygsi = ioapic_abi_legacy_intr_find_bygsi, 537 538 .msi_alloc = ioapic_abi_msi_alloc, 539 .msi_release = ioapic_abi_msi_release, 540 .msi_map = ioapic_abi_msi_map, 541 .msix_alloc = ioapic_abi_msix_alloc, 542 .msix_release = ioapic_abi_msix_release, 543 544 .finalize = ioapic_abi_finalize, 545 .cleanup = ioapic_abi_cleanup, 546 .setdefault = ioapic_abi_setdefault, 547 .stabilize = ioapic_abi_stabilize, 548 .initmap = ioapic_abi_initmap, 549 .rman_setup = ioapic_abi_rman_setup 550 }; 551 552 static int ioapic_abi_extint_irq = -1; 553 static int ioapic_abi_legacy_irq_max; 554 static int ioapic_abi_gsi_balance = 1; 555 static int ioapic_abi_msi_start; /* NOTE: for testing only */ 556 557 struct ioapic_irqinfo ioapic_irqs[IOAPIC_HWI_VECTORS]; 558 559 static void 560 ioapic_abi_intr_enable(int irq) 561 { 562 const struct ioapic_irqmap *map; 563 564 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 565 ("ioapic enable, invalid irq %d", irq)); 566 567 map = &ioapic_irqmaps[mycpuid][irq]; 568 KASSERT(IOAPIC_IMT_ISHWI(map), 569 ("ioapic enable, not hwi irq %d, type %d, cpu%d", 570 irq, map->im_type, mycpuid)); 571 if (map->im_type != IOAPIC_IMT_LEGACY) 572 return; 573 574 IOAPIC_INTREN(irq); 575 } 576 577 static void 578 ioapic_abi_intr_disable(int irq) 579 { 580 const struct ioapic_irqmap *map; 581 582 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 583 ("ioapic disable, invalid irq %d", irq)); 584 585 map = &ioapic_irqmaps[mycpuid][irq]; 586 KASSERT(IOAPIC_IMT_ISHWI(map), 587 ("ioapic disable, not hwi irq %d, type %d, cpu%d", 588 irq, map->im_type, mycpuid)); 589 if (map->im_type != IOAPIC_IMT_LEGACY) 590 return; 591 592 IOAPIC_INTRDIS(irq); 593 } 594 595 static void 596 ioapic_abi_finalize(void) 597 { 598 KKASSERT(MachIntrABI.type == MACHINTR_IOAPIC); 599 KKASSERT(ioapic_enable); 600 601 /* 602 * If an IMCR is present, program bit 0 to disconnect the 8259 603 * from the BSP. 604 */ 605 if (imcr_present) { 606 outb(0x22, 0x70); /* select IMCR */ 607 outb(0x23, 0x01); /* disconnect 8259 */ 608 } 609 } 610 611 /* 612 * This routine is called after physical interrupts are enabled but before 613 * the critical section is released. We need to clean out any interrupts 614 * that had already been posted to the cpu. 615 */ 616 static void 617 ioapic_abi_cleanup(void) 618 { 619 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending)); 620 } 621 622 /* Must never be called */ 623 static void 624 ioapic_abi_stabilize(void) 625 { 626 panic("ioapic_stabilize is called"); 627 } 628 629 static void 630 ioapic_abi_intr_setup(int intr, int flags) 631 { 632 const struct ioapic_irqmap *map; 633 int vector, select; 634 uint32_t value; 635 register_t ef; 636 637 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 638 ("ioapic setup, invalid irq %d", intr)); 639 640 map = &ioapic_irqmaps[mycpuid][intr]; 641 KASSERT(IOAPIC_IMT_ISHWI(map), 642 ("ioapic setup, not hwi irq %d, type %d, cpu%d", 643 intr, map->im_type, mycpuid)); 644 if (map->im_type != IOAPIC_IMT_LEGACY) 645 return; 646 647 KASSERT(ioapic_irqs[intr].io_addr != NULL, 648 ("ioapic setup, no GSI information, irq %d", intr)); 649 650 ef = read_rflags(); 651 cpu_disable_intr(); 652 653 vector = IDT_OFFSET + intr; 654 655 /* 656 * Now reprogram the vector in the IO APIC. In order to avoid 657 * losing an EOI for a level interrupt, which is vector based, 658 * make sure that the IO APIC is programmed for edge-triggering 659 * first, then reprogrammed with the new vector. This should 660 * clear the IRR bit. 661 */ 662 imen_lock(); 663 664 select = ioapic_irqs[intr].io_idx; 665 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 666 value |= IOART_INTMSET; 667 668 ioapic_write(ioapic_irqs[intr].io_addr, select, 669 (value & ~APIC_TRIGMOD_MASK)); 670 ioapic_write(ioapic_irqs[intr].io_addr, select, 671 (value & ~IOART_INTVEC) | vector); 672 673 imen_unlock(); 674 675 IOAPIC_INTREN(intr); 676 677 write_rflags(ef); 678 } 679 680 static void 681 ioapic_abi_intr_teardown(int intr) 682 { 683 const struct ioapic_irqmap *map; 684 int vector, select; 685 uint32_t value; 686 register_t ef; 687 688 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 689 ("ioapic teardown, invalid irq %d", intr)); 690 691 map = &ioapic_irqmaps[mycpuid][intr]; 692 KASSERT(IOAPIC_IMT_ISHWI(map), 693 ("ioapic teardown, not hwi irq %d, type %d, cpu%d", 694 intr, map->im_type, mycpuid)); 695 if (map->im_type != IOAPIC_IMT_LEGACY) 696 return; 697 698 KASSERT(ioapic_irqs[intr].io_addr != NULL, 699 ("ioapic teardown, no GSI information, irq %d", intr)); 700 701 ef = read_rflags(); 702 cpu_disable_intr(); 703 704 /* 705 * Teardown an interrupt vector. The vector should already be 706 * installed in the cpu's IDT, but make sure. 707 */ 708 IOAPIC_INTRDIS(intr); 709 710 vector = IDT_OFFSET + intr; 711 712 /* 713 * In order to avoid losing an EOI for a level interrupt, which 714 * is vector based, make sure that the IO APIC is programmed for 715 * edge-triggering first, then reprogrammed with the new vector. 716 * This should clear the IRR bit. 717 */ 718 imen_lock(); 719 720 select = ioapic_irqs[intr].io_idx; 721 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 722 723 ioapic_write(ioapic_irqs[intr].io_addr, select, 724 (value & ~APIC_TRIGMOD_MASK)); 725 ioapic_write(ioapic_irqs[intr].io_addr, select, 726 (value & ~IOART_INTVEC) | vector); 727 728 imen_unlock(); 729 730 write_rflags(ef); 731 } 732 733 static void 734 ioapic_abi_setdefault(void) 735 { 736 int intr; 737 738 for (intr = 0; intr < IOAPIC_HWI_VECTORS; ++intr) { 739 if (intr == IOAPIC_HWI_SYSCALL) 740 continue; 741 setidt_global(IDT_OFFSET + intr, ioapic_intr[intr], 742 SDT_SYSIGT, SEL_KPL, 0); 743 } 744 } 745 746 static void 747 ioapic_abi_initmap(void) 748 { 749 int cpu; 750 751 kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance); 752 753 kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start); 754 ioapic_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */ 755 756 /* 757 * NOTE: ncpus is not ready yet 758 */ 759 for (cpu = 0; cpu < MAXCPU; ++cpu) { 760 int i; 761 762 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 763 ioapic_irqmaps[cpu][i].im_gsi = -1; 764 ioapic_irqmaps[cpu][i].im_msi_base = -1; 765 } 766 ioapic_irqmaps[cpu][IOAPIC_HWI_SYSCALL].im_type = 767 IOAPIC_IMT_SYSCALL; 768 } 769 } 770 771 /* 772 * Only one CPU can have the legacy IRQ map. 773 */ 774 static int 775 ioapic_is_legacy_irqmap_used(int irq) 776 { 777 int cpu; 778 779 for (cpu = 0; cpu < ncpus; ++cpu) { 780 if (ioapic_irqmaps[cpu][irq].im_type != IOAPIC_IMT_UNUSED) 781 return 1; 782 } 783 return 0; 784 } 785 786 static int 787 ioapic_unused_legacy_irqmap(void) 788 { 789 int i; 790 791 for (i = ISA_IRQ_CNT; i < IOAPIC_HWI_VECTORS; ++i) { 792 if (i == acpi_sci_irqno()) 793 continue; 794 if (!ioapic_is_legacy_irqmap_used(i)) 795 return i; 796 } 797 return -1; 798 } 799 800 void 801 ioapic_set_legacy_irqmap(int irq, int gsi, enum intr_trigger trig, 802 enum intr_polarity pola) 803 { 804 struct ioapic_irqinfo *info; 805 struct ioapic_irqmap *map; 806 void *ioaddr; 807 int pin, cpuid; 808 809 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 810 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 811 812 KKASSERT(irq >= 0); 813 if (irq >= IOAPIC_HWI_VECTORS) { 814 /* 815 * Some BIOSes seem to assume that all 256 IDT vectors 816 * could be used, while we limit the available IDT 817 * vectors to 192; find an unused IRQ for this GSI. 818 */ 819 irq = ioapic_unused_legacy_irqmap(); 820 if (irq < 0) { 821 kprintf("failed to find unused irq for gsi %d, " 822 "overflow\n", gsi); 823 return; 824 } 825 } 826 KKASSERT(irq < IOAPIC_HWI_VECTORS); 827 828 if (ioapic_is_legacy_irqmap_used(irq)) { 829 /* 830 * There are so many IOAPICs, that 1:1 mapping 831 * of GSI and IRQ hits SYSCALL entry. 832 */ 833 irq = ioapic_unused_legacy_irqmap(); 834 if (irq < 0) { 835 kprintf("failed to find unused irq for gsi %d, " 836 "conflict\n", gsi); 837 return; 838 } 839 KKASSERT(irq < IOAPIC_HWI_VECTORS); 840 841 } 842 cpuid = ioapic_abi_gsi_cpuid(irq, gsi); 843 map = &ioapic_irqmaps[cpuid][irq]; 844 845 if (irq > ioapic_abi_legacy_irq_max) 846 ioapic_abi_legacy_irq_max = irq; 847 848 KKASSERT(map->im_type == IOAPIC_IMT_UNUSED); 849 map->im_type = IOAPIC_IMT_LEGACY; 850 851 map->im_gsi = gsi; 852 map->im_trig = trig; 853 map->im_pola = pola; 854 855 if (bootverbose) { 856 kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n", 857 irq, map->im_gsi, 858 intr_str_trigger(map->im_trig), 859 intr_str_polarity(map->im_pola)); 860 } 861 862 pin = ioapic_gsi_pin(map->im_gsi); 863 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 864 865 info = &ioapic_irqs[irq]; 866 867 imen_lock(); 868 869 info->io_addr = ioaddr; 870 info->io_idx = IOAPIC_REDTBL + (2 * pin); 871 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 872 if (map->im_trig == INTR_TRIGGER_LEVEL) 873 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 874 875 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 876 map->im_trig, map->im_pola, cpuid); 877 878 imen_unlock(); 879 } 880 881 void 882 ioapic_fixup_legacy_irqmaps(void) 883 { 884 int cpu; 885 886 for (cpu = 0; cpu < ncpus; ++cpu) { 887 int i; 888 889 for (i = 0; i < ISA_IRQ_CNT; ++i) { 890 struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][i]; 891 892 if (map->im_type == IOAPIC_IMT_UNUSED) { 893 map->im_type = IOAPIC_IMT_RESERVED; 894 if (bootverbose) { 895 kprintf("IOAPIC: " 896 "cpu%d irq %d reserved\n", cpu, i); 897 } 898 } 899 } 900 } 901 902 ioapic_abi_legacy_irq_max += 1; 903 if (bootverbose) { 904 kprintf("IOAPIC: legacy irq max %d\n", 905 ioapic_abi_legacy_irq_max); 906 } 907 } 908 909 static int 910 ioapic_abi_legacy_intr_find_bygsi(int gsi, enum intr_trigger trig, 911 enum intr_polarity pola) 912 { 913 int cpu; 914 915 #ifdef INVARIANTS 916 if (trig == INTR_TRIGGER_CONFORM) { 917 KKASSERT(pola == INTR_POLARITY_CONFORM); 918 } else { 919 KKASSERT(trig == INTR_TRIGGER_EDGE || 920 trig == INTR_TRIGGER_LEVEL); 921 KKASSERT(pola == INTR_POLARITY_HIGH || 922 pola == INTR_POLARITY_LOW); 923 } 924 #endif 925 926 for (cpu = 0; cpu < ncpus; ++cpu) { 927 int irq; 928 929 for (irq = 0; irq < ioapic_abi_legacy_irq_max; ++irq) { 930 const struct ioapic_irqmap *map = 931 &ioapic_irqmaps[cpu][irq]; 932 933 if (map->im_gsi == gsi) { 934 KKASSERT(map->im_type == IOAPIC_IMT_LEGACY); 935 936 if ((map->im_flags & IOAPIC_IMF_CONF) && 937 trig != INTR_TRIGGER_CONFORM && 938 pola != INTR_POLARITY_CONFORM) { 939 if (map->im_trig != trig || 940 map->im_pola != pola) 941 return -1; 942 } 943 return irq; 944 } 945 } 946 } 947 return -1; 948 } 949 950 static int 951 ioapic_abi_legacy_intr_find(int irq, enum intr_trigger trig, 952 enum intr_polarity pola) 953 { 954 int cpu; 955 956 #ifdef INVARIANTS 957 if (trig == INTR_TRIGGER_CONFORM) { 958 KKASSERT(pola == INTR_POLARITY_CONFORM); 959 } else { 960 KKASSERT(trig == INTR_TRIGGER_EDGE || 961 trig == INTR_TRIGGER_LEVEL); 962 KKASSERT(pola == INTR_POLARITY_HIGH || 963 pola == INTR_POLARITY_LOW); 964 } 965 #endif 966 967 if (irq < 0 || irq >= ioapic_abi_legacy_irq_max) 968 return -1; 969 970 for (cpu = 0; cpu < ncpus; ++cpu) { 971 const struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][irq]; 972 973 if (map->im_type == IOAPIC_IMT_LEGACY) { 974 if ((map->im_flags & IOAPIC_IMF_CONF) && 975 trig != INTR_TRIGGER_CONFORM && 976 pola != INTR_POLARITY_CONFORM) { 977 if (map->im_trig != trig || 978 map->im_pola != pola) 979 return -1; 980 } 981 return irq; 982 } 983 } 984 return -1; 985 } 986 987 static void 988 ioapic_abi_legacy_intr_config(int irq, enum intr_trigger trig, 989 enum intr_polarity pola) 990 { 991 struct ioapic_irqinfo *info; 992 struct ioapic_irqmap *map = NULL; 993 void *ioaddr; 994 int pin, cpuid; 995 996 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 997 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 998 999 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 1000 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1001 map = &ioapic_irqmaps[cpuid][irq]; 1002 if (map->im_type == IOAPIC_IMT_LEGACY) 1003 break; 1004 } 1005 KKASSERT(cpuid < ncpus); 1006 1007 #ifdef notyet 1008 if (map->im_flags & IOAPIC_IMF_CONF) { 1009 if (trig != map->im_trig) { 1010 panic("ioapic_intr_config: trig %s -> %s", 1011 intr_str_trigger(map->im_trig), 1012 intr_str_trigger(trig)); 1013 } 1014 if (pola != map->im_pola) { 1015 panic("ioapic_intr_config: pola %s -> %s", 1016 intr_str_polarity(map->im_pola), 1017 intr_str_polarity(pola)); 1018 } 1019 return; 1020 } 1021 #endif 1022 map->im_flags |= IOAPIC_IMF_CONF; 1023 1024 if (trig == map->im_trig && pola == map->im_pola) 1025 return; 1026 1027 if (bootverbose) { 1028 kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n", 1029 irq, map->im_gsi, 1030 intr_str_trigger(map->im_trig), 1031 intr_str_polarity(map->im_pola), 1032 intr_str_trigger(trig), 1033 intr_str_polarity(pola)); 1034 } 1035 map->im_trig = trig; 1036 map->im_pola = pola; 1037 1038 pin = ioapic_gsi_pin(map->im_gsi); 1039 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1040 1041 info = &ioapic_irqs[irq]; 1042 1043 imen_lock(); 1044 1045 info->io_flags &= ~IOAPIC_IRQI_FLAG_LEVEL; 1046 if (map->im_trig == INTR_TRIGGER_LEVEL) 1047 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 1048 1049 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 1050 map->im_trig, map->im_pola, cpuid); 1051 1052 imen_unlock(); 1053 } 1054 1055 int 1056 ioapic_conf_legacy_extint(int irq) 1057 { 1058 struct ioapic_irqinfo *info; 1059 struct ioapic_irqmap *map; 1060 void *ioaddr; 1061 int pin, error, vec; 1062 1063 /* XXX only irq0 is allowed */ 1064 KKASSERT(irq == 0); 1065 1066 vec = IDT_OFFSET + irq; 1067 1068 if (ioapic_abi_extint_irq == irq) 1069 return 0; 1070 else if (ioapic_abi_extint_irq >= 0) 1071 return EEXIST; 1072 1073 error = icu_ioapic_extint(irq, vec); 1074 if (error) 1075 return error; 1076 1077 /* ExtINT is always targeted to cpu0 */ 1078 map = &ioapic_irqmaps[0][irq]; 1079 1080 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED || 1081 map->im_type == IOAPIC_IMT_LEGACY); 1082 if (map->im_type == IOAPIC_IMT_LEGACY) { 1083 if (map->im_flags & IOAPIC_IMF_CONF) 1084 return EEXIST; 1085 } 1086 ioapic_abi_extint_irq = irq; 1087 1088 map->im_type = IOAPIC_IMT_LEGACY; 1089 map->im_trig = INTR_TRIGGER_EDGE; 1090 map->im_pola = INTR_POLARITY_HIGH; 1091 map->im_flags = IOAPIC_IMF_CONF; 1092 1093 map->im_gsi = ioapic_extpin_gsi(); 1094 KKASSERT(map->im_gsi >= 0); 1095 1096 if (bootverbose) { 1097 kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n", 1098 irq, map->im_gsi, 1099 intr_str_trigger(map->im_trig), 1100 intr_str_polarity(map->im_pola)); 1101 } 1102 1103 pin = ioapic_gsi_pin(map->im_gsi); 1104 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1105 1106 info = &ioapic_irqs[irq]; 1107 1108 imen_lock(); 1109 1110 info->io_addr = ioaddr; 1111 info->io_idx = IOAPIC_REDTBL + (2 * pin); 1112 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 1113 1114 ioapic_extpin_setup(ioaddr, pin, vec); 1115 1116 imen_unlock(); 1117 1118 return 0; 1119 } 1120 1121 static int 1122 ioapic_abi_legacy_intr_cpuid(int irq) 1123 { 1124 const struct ioapic_irqmap *map = NULL; 1125 int cpuid; 1126 1127 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 1128 1129 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1130 map = &ioapic_irqmaps[cpuid][irq]; 1131 if (map->im_type == IOAPIC_IMT_LEGACY) 1132 return cpuid; 1133 } 1134 1135 /* XXX some drivers tries to peek at reserved IRQs */ 1136 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1137 map = &ioapic_irqmaps[cpuid][irq]; 1138 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED); 1139 } 1140 return 0; 1141 } 1142 1143 static int 1144 ioapic_abi_gsi_cpuid(int irq, int gsi) 1145 { 1146 char envpath[32]; 1147 int cpuid = -1; 1148 1149 KKASSERT(gsi >= 0); 1150 1151 if (irq == 0 || gsi == 0) { 1152 KKASSERT(irq >= 0); 1153 if (bootverbose) { 1154 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n", 1155 irq, gsi); 1156 } 1157 return 0; 1158 } 1159 1160 if (irq >= 0 && irq == acpi_sci_irqno()) { 1161 if (bootverbose) { 1162 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n", 1163 irq, gsi); 1164 } 1165 return 0; 1166 } 1167 1168 ksnprintf(envpath, sizeof(envpath), "hw.ioapic.gsi.%d.cpu", gsi); 1169 kgetenv_int(envpath, &cpuid); 1170 1171 if (cpuid < 0) { 1172 if (!ioapic_abi_gsi_balance) { 1173 if (irq >= 0 && bootverbose) { 1174 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 " 1175 "(fixed)\n", irq, gsi); 1176 } 1177 return 0; 1178 } 1179 1180 cpuid = gsi % ncpus; 1181 if (irq >= 0 && bootverbose) { 1182 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n", 1183 irq, gsi, cpuid); 1184 } 1185 } else if (cpuid >= ncpus) { 1186 cpuid = ncpus - 1; 1187 if (irq >= 0 && bootverbose) { 1188 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n", 1189 irq, gsi, cpuid); 1190 } 1191 } else { 1192 if (irq >= 0 && bootverbose) { 1193 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n", 1194 irq, gsi, cpuid); 1195 } 1196 } 1197 return cpuid; 1198 } 1199 1200 static void 1201 ioapic_abi_rman_setup(struct rman *rm) 1202 { 1203 int start, end, i; 1204 1205 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU, 1206 ("invalid rman cpuid %d", rm->rm_cpuid)); 1207 1208 start = end = -1; 1209 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 1210 const struct ioapic_irqmap *map = 1211 &ioapic_irqmaps[rm->rm_cpuid][i]; 1212 1213 if (start < 0) { 1214 if (IOAPIC_IMT_ISHWI(map)) 1215 start = end = i; 1216 } else { 1217 if (IOAPIC_IMT_ISHWI(map)) { 1218 end = i; 1219 } else { 1220 KKASSERT(end >= 0); 1221 if (bootverbose) { 1222 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1223 rm->rm_cpuid, start, end); 1224 } 1225 if (rman_manage_region(rm, start, end)) { 1226 panic("rman_manage_region" 1227 "(cpu%d %d - %d)", rm->rm_cpuid, 1228 start, end); 1229 } 1230 start = end = -1; 1231 } 1232 } 1233 } 1234 if (start >= 0) { 1235 KKASSERT(end >= 0); 1236 if (bootverbose) { 1237 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1238 rm->rm_cpuid, start, end); 1239 } 1240 if (rman_manage_region(rm, start, end)) { 1241 panic("rman_manage_region(cpu%d %d - %d)", 1242 rm->rm_cpuid, start, end); 1243 } 1244 } 1245 } 1246 1247 static int 1248 ioapic_abi_msi_alloc_intern(int type, const char *desc, 1249 int intrs[], int count, int cpuid) 1250 { 1251 int i, error; 1252 1253 KASSERT(cpuid >= 0 && cpuid < ncpus, 1254 ("invalid cpuid %d", cpuid)); 1255 1256 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 1257 KASSERT(powerof2(count), ("count %d is not power of 2", count)); 1258 1259 lwkt_gettoken(&ioapic_irqmap_tok); 1260 1261 /* 1262 * NOTE: 1263 * Since IDT_OFFSET is 32, which is the maximum valid 'count', 1264 * we do not need to find out the first properly aligned 1265 * interrupt vector. 1266 */ 1267 1268 error = EMSGSIZE; 1269 for (i = ioapic_abi_msi_start; i < IOAPIC_HWI_VECTORS; i += count) { 1270 int j; 1271 1272 if (ioapic_irqmaps[cpuid][i].im_type != IOAPIC_IMT_UNUSED) 1273 continue; 1274 1275 for (j = 1; j < count; ++j) { 1276 if (ioapic_irqmaps[cpuid][i + j].im_type != 1277 IOAPIC_IMT_UNUSED) 1278 break; 1279 } 1280 if (j != count) 1281 continue; 1282 1283 for (j = 0; j < count; ++j) { 1284 struct ioapic_irqmap *map; 1285 int intr = i + j; 1286 1287 map = &ioapic_irqmaps[cpuid][intr]; 1288 KASSERT(map->im_msi_base < 0, 1289 ("intr %d, stale %s-base %d", 1290 intr, desc, map->im_msi_base)); 1291 1292 map->im_type = type; 1293 map->im_msi_base = i; 1294 1295 intrs[j] = intr; 1296 msi_setup(intr, cpuid); 1297 1298 if (bootverbose) { 1299 kprintf("alloc %s intr %d on cpu%d\n", 1300 desc, intr, cpuid); 1301 } 1302 } 1303 error = 0; 1304 break; 1305 } 1306 1307 lwkt_reltoken(&ioapic_irqmap_tok); 1308 1309 return error; 1310 } 1311 1312 static void 1313 ioapic_abi_msi_release_intern(int type, const char *desc, 1314 const int intrs[], int count, int cpuid) 1315 { 1316 int i, msi_base = -1, intr_next = -1, mask; 1317 1318 KASSERT(cpuid >= 0 && cpuid < ncpus, 1319 ("invalid cpuid %d", cpuid)); 1320 1321 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 1322 1323 mask = count - 1; 1324 KASSERT((count & mask) == 0, ("count %d is not power of 2", count)); 1325 1326 lwkt_gettoken(&ioapic_irqmap_tok); 1327 1328 for (i = 0; i < count; ++i) { 1329 struct ioapic_irqmap *map; 1330 int intr = intrs[i]; 1331 1332 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1333 ("invalid intr %d", intr)); 1334 1335 map = &ioapic_irqmaps[cpuid][intr]; 1336 KASSERT(map->im_type == type, 1337 ("trying to release non-%s intr %d, type %d", desc, 1338 intr, map->im_type)); 1339 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1340 ("intr %d, invalid %s-base %d", intr, desc, 1341 map->im_msi_base)); 1342 KASSERT((map->im_msi_base & mask) == 0, 1343 ("intr %d, %s-base %d is not properly aligned %d", 1344 intr, desc, map->im_msi_base, count)); 1345 1346 if (msi_base < 0) { 1347 msi_base = map->im_msi_base; 1348 } else { 1349 KASSERT(map->im_msi_base == msi_base, 1350 ("intr %d, inconsistent %s-base, " 1351 "was %d, now %d", 1352 intr, desc, msi_base, map->im_msi_base)); 1353 } 1354 1355 if (intr_next < intr) 1356 intr_next = intr; 1357 1358 map->im_type = IOAPIC_IMT_UNUSED; 1359 map->im_msi_base = -1; 1360 1361 if (bootverbose) { 1362 kprintf("release %s intr %d on cpu%d\n", 1363 desc, intr, cpuid); 1364 } 1365 } 1366 1367 KKASSERT(intr_next > 0); 1368 KKASSERT(msi_base >= 0); 1369 1370 ++intr_next; 1371 if (intr_next < IOAPIC_HWI_VECTORS) { 1372 const struct ioapic_irqmap *map = 1373 &ioapic_irqmaps[cpuid][intr_next]; 1374 1375 if (map->im_type == type) { 1376 KASSERT(map->im_msi_base != msi_base, 1377 ("more than %d %s was allocated", count, desc)); 1378 } 1379 } 1380 1381 lwkt_reltoken(&ioapic_irqmap_tok); 1382 } 1383 1384 static int 1385 ioapic_abi_msi_alloc(int intrs[], int count, int cpuid) 1386 { 1387 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSI, "MSI", 1388 intrs, count, cpuid); 1389 } 1390 1391 static void 1392 ioapic_abi_msi_release(const int intrs[], int count, int cpuid) 1393 { 1394 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSI, "MSI", 1395 intrs, count, cpuid); 1396 } 1397 1398 static int 1399 ioapic_abi_msix_alloc(int *intr, int cpuid) 1400 { 1401 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSIX, "MSI-X", 1402 intr, 1, cpuid); 1403 } 1404 1405 static void 1406 ioapic_abi_msix_release(int intr, int cpuid) 1407 { 1408 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSIX, "MSI-X", 1409 &intr, 1, cpuid); 1410 } 1411 1412 static void 1413 ioapic_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid) 1414 { 1415 const struct ioapic_irqmap *map; 1416 1417 KASSERT(cpuid >= 0 && cpuid < ncpus, 1418 ("invalid cpuid %d", cpuid)); 1419 1420 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1421 ("invalid intr %d", intr)); 1422 1423 lwkt_gettoken(&ioapic_irqmap_tok); 1424 1425 map = &ioapic_irqmaps[cpuid][intr]; 1426 KASSERT(map->im_type == IOAPIC_IMT_MSI || 1427 map->im_type == IOAPIC_IMT_MSIX, 1428 ("trying to map non-MSI/MSI-X intr %d, type %d", intr, map->im_type)); 1429 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1430 ("intr %d, invalid %s-base %d", intr, 1431 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1432 map->im_msi_base)); 1433 1434 msi_map(map->im_msi_base, addr, data, cpuid); 1435 1436 if (bootverbose) { 1437 kprintf("map %s intr %d on cpu%d\n", 1438 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1439 intr, cpuid); 1440 } 1441 1442 lwkt_reltoken(&ioapic_irqmap_tok); 1443 } 1444