1 /* 2 * Copyright (c) 1991 The Regents of the University of California. 3 * Copyright (c) 1996, by Steve Passe. All rights reserved. 4 * Copyright (c) 2005,2008 The DragonFly Project. All rights reserved. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The DragonFly Project 8 * by Matthew Dillon <dillon@backplane.com> 9 * 10 * This code is derived from software contributed to Berkeley by 11 * William Jolitz. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 3. Neither the name of The DragonFly Project nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific, prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 30 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 31 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 34 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 35 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 36 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 37 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/kernel.h> 44 #include <sys/machintr.h> 45 #include <sys/interrupt.h> 46 #include <sys/bus.h> 47 #include <sys/rman.h> 48 #include <sys/thread2.h> 49 50 #include <machine/smp.h> 51 #include <machine/segments.h> 52 #include <machine/md_var.h> 53 #include <machine/intr_machdep.h> 54 #include <machine/globaldata.h> 55 #include <machine/msi_var.h> 56 57 #include <machine_base/isa/isa_intr.h> 58 #include <machine_base/icu/icu.h> 59 #include <machine_base/icu/icu_var.h> 60 #include <machine_base/apic/ioapic.h> 61 #include <machine_base/apic/ioapic_abi.h> 62 #include <machine_base/apic/ioapic_ipl.h> 63 #include <machine_base/apic/apicreg.h> 64 65 #include <dev/acpica/acpi_sci_var.h> 66 67 #define IOAPIC_HWI_VECTORS IDT_HWI_VECTORS 68 69 extern inthand_t 70 IDTVEC(ioapic_intr0), 71 IDTVEC(ioapic_intr1), 72 IDTVEC(ioapic_intr2), 73 IDTVEC(ioapic_intr3), 74 IDTVEC(ioapic_intr4), 75 IDTVEC(ioapic_intr5), 76 IDTVEC(ioapic_intr6), 77 IDTVEC(ioapic_intr7), 78 IDTVEC(ioapic_intr8), 79 IDTVEC(ioapic_intr9), 80 IDTVEC(ioapic_intr10), 81 IDTVEC(ioapic_intr11), 82 IDTVEC(ioapic_intr12), 83 IDTVEC(ioapic_intr13), 84 IDTVEC(ioapic_intr14), 85 IDTVEC(ioapic_intr15), 86 IDTVEC(ioapic_intr16), 87 IDTVEC(ioapic_intr17), 88 IDTVEC(ioapic_intr18), 89 IDTVEC(ioapic_intr19), 90 IDTVEC(ioapic_intr20), 91 IDTVEC(ioapic_intr21), 92 IDTVEC(ioapic_intr22), 93 IDTVEC(ioapic_intr23), 94 IDTVEC(ioapic_intr24), 95 IDTVEC(ioapic_intr25), 96 IDTVEC(ioapic_intr26), 97 IDTVEC(ioapic_intr27), 98 IDTVEC(ioapic_intr28), 99 IDTVEC(ioapic_intr29), 100 IDTVEC(ioapic_intr30), 101 IDTVEC(ioapic_intr31), 102 IDTVEC(ioapic_intr32), 103 IDTVEC(ioapic_intr33), 104 IDTVEC(ioapic_intr34), 105 IDTVEC(ioapic_intr35), 106 IDTVEC(ioapic_intr36), 107 IDTVEC(ioapic_intr37), 108 IDTVEC(ioapic_intr38), 109 IDTVEC(ioapic_intr39), 110 IDTVEC(ioapic_intr40), 111 IDTVEC(ioapic_intr41), 112 IDTVEC(ioapic_intr42), 113 IDTVEC(ioapic_intr43), 114 IDTVEC(ioapic_intr44), 115 IDTVEC(ioapic_intr45), 116 IDTVEC(ioapic_intr46), 117 IDTVEC(ioapic_intr47), 118 IDTVEC(ioapic_intr48), 119 IDTVEC(ioapic_intr49), 120 IDTVEC(ioapic_intr50), 121 IDTVEC(ioapic_intr51), 122 IDTVEC(ioapic_intr52), 123 IDTVEC(ioapic_intr53), 124 IDTVEC(ioapic_intr54), 125 IDTVEC(ioapic_intr55), 126 IDTVEC(ioapic_intr56), 127 IDTVEC(ioapic_intr57), 128 IDTVEC(ioapic_intr58), 129 IDTVEC(ioapic_intr59), 130 IDTVEC(ioapic_intr60), 131 IDTVEC(ioapic_intr61), 132 IDTVEC(ioapic_intr62), 133 IDTVEC(ioapic_intr63), 134 IDTVEC(ioapic_intr64), 135 IDTVEC(ioapic_intr65), 136 IDTVEC(ioapic_intr66), 137 IDTVEC(ioapic_intr67), 138 IDTVEC(ioapic_intr68), 139 IDTVEC(ioapic_intr69), 140 IDTVEC(ioapic_intr70), 141 IDTVEC(ioapic_intr71), 142 IDTVEC(ioapic_intr72), 143 IDTVEC(ioapic_intr73), 144 IDTVEC(ioapic_intr74), 145 IDTVEC(ioapic_intr75), 146 IDTVEC(ioapic_intr76), 147 IDTVEC(ioapic_intr77), 148 IDTVEC(ioapic_intr78), 149 IDTVEC(ioapic_intr79), 150 IDTVEC(ioapic_intr80), 151 IDTVEC(ioapic_intr81), 152 IDTVEC(ioapic_intr82), 153 IDTVEC(ioapic_intr83), 154 IDTVEC(ioapic_intr84), 155 IDTVEC(ioapic_intr85), 156 IDTVEC(ioapic_intr86), 157 IDTVEC(ioapic_intr87), 158 IDTVEC(ioapic_intr88), 159 IDTVEC(ioapic_intr89), 160 IDTVEC(ioapic_intr90), 161 IDTVEC(ioapic_intr91), 162 IDTVEC(ioapic_intr92), 163 IDTVEC(ioapic_intr93), 164 IDTVEC(ioapic_intr94), 165 IDTVEC(ioapic_intr95), 166 IDTVEC(ioapic_intr96), 167 IDTVEC(ioapic_intr97), 168 IDTVEC(ioapic_intr98), 169 IDTVEC(ioapic_intr99), 170 IDTVEC(ioapic_intr100), 171 IDTVEC(ioapic_intr101), 172 IDTVEC(ioapic_intr102), 173 IDTVEC(ioapic_intr103), 174 IDTVEC(ioapic_intr104), 175 IDTVEC(ioapic_intr105), 176 IDTVEC(ioapic_intr106), 177 IDTVEC(ioapic_intr107), 178 IDTVEC(ioapic_intr108), 179 IDTVEC(ioapic_intr109), 180 IDTVEC(ioapic_intr110), 181 IDTVEC(ioapic_intr111), 182 IDTVEC(ioapic_intr112), 183 IDTVEC(ioapic_intr113), 184 IDTVEC(ioapic_intr114), 185 IDTVEC(ioapic_intr115), 186 IDTVEC(ioapic_intr116), 187 IDTVEC(ioapic_intr117), 188 IDTVEC(ioapic_intr118), 189 IDTVEC(ioapic_intr119), 190 IDTVEC(ioapic_intr120), 191 IDTVEC(ioapic_intr121), 192 IDTVEC(ioapic_intr122), 193 IDTVEC(ioapic_intr123), 194 IDTVEC(ioapic_intr124), 195 IDTVEC(ioapic_intr125), 196 IDTVEC(ioapic_intr126), 197 IDTVEC(ioapic_intr127), 198 IDTVEC(ioapic_intr128), 199 IDTVEC(ioapic_intr129), 200 IDTVEC(ioapic_intr130), 201 IDTVEC(ioapic_intr131), 202 IDTVEC(ioapic_intr132), 203 IDTVEC(ioapic_intr133), 204 IDTVEC(ioapic_intr134), 205 IDTVEC(ioapic_intr135), 206 IDTVEC(ioapic_intr136), 207 IDTVEC(ioapic_intr137), 208 IDTVEC(ioapic_intr138), 209 IDTVEC(ioapic_intr139), 210 IDTVEC(ioapic_intr140), 211 IDTVEC(ioapic_intr141), 212 IDTVEC(ioapic_intr142), 213 IDTVEC(ioapic_intr143), 214 IDTVEC(ioapic_intr144), 215 IDTVEC(ioapic_intr145), 216 IDTVEC(ioapic_intr146), 217 IDTVEC(ioapic_intr147), 218 IDTVEC(ioapic_intr148), 219 IDTVEC(ioapic_intr149), 220 IDTVEC(ioapic_intr150), 221 IDTVEC(ioapic_intr151), 222 IDTVEC(ioapic_intr152), 223 IDTVEC(ioapic_intr153), 224 IDTVEC(ioapic_intr154), 225 IDTVEC(ioapic_intr155), 226 IDTVEC(ioapic_intr156), 227 IDTVEC(ioapic_intr157), 228 IDTVEC(ioapic_intr158), 229 IDTVEC(ioapic_intr159), 230 IDTVEC(ioapic_intr160), 231 IDTVEC(ioapic_intr161), 232 IDTVEC(ioapic_intr162), 233 IDTVEC(ioapic_intr163), 234 IDTVEC(ioapic_intr164), 235 IDTVEC(ioapic_intr165), 236 IDTVEC(ioapic_intr166), 237 IDTVEC(ioapic_intr167), 238 IDTVEC(ioapic_intr168), 239 IDTVEC(ioapic_intr169), 240 IDTVEC(ioapic_intr170), 241 IDTVEC(ioapic_intr171), 242 IDTVEC(ioapic_intr172), 243 IDTVEC(ioapic_intr173), 244 IDTVEC(ioapic_intr174), 245 IDTVEC(ioapic_intr175), 246 IDTVEC(ioapic_intr176), 247 IDTVEC(ioapic_intr177), 248 IDTVEC(ioapic_intr178), 249 IDTVEC(ioapic_intr179), 250 IDTVEC(ioapic_intr180), 251 IDTVEC(ioapic_intr181), 252 IDTVEC(ioapic_intr182), 253 IDTVEC(ioapic_intr183), 254 IDTVEC(ioapic_intr184), 255 IDTVEC(ioapic_intr185), 256 IDTVEC(ioapic_intr186), 257 IDTVEC(ioapic_intr187), 258 IDTVEC(ioapic_intr188), 259 IDTVEC(ioapic_intr189), 260 IDTVEC(ioapic_intr190), 261 IDTVEC(ioapic_intr191); 262 263 static inthand_t *ioapic_intr[IOAPIC_HWI_VECTORS] = { 264 &IDTVEC(ioapic_intr0), 265 &IDTVEC(ioapic_intr1), 266 &IDTVEC(ioapic_intr2), 267 &IDTVEC(ioapic_intr3), 268 &IDTVEC(ioapic_intr4), 269 &IDTVEC(ioapic_intr5), 270 &IDTVEC(ioapic_intr6), 271 &IDTVEC(ioapic_intr7), 272 &IDTVEC(ioapic_intr8), 273 &IDTVEC(ioapic_intr9), 274 &IDTVEC(ioapic_intr10), 275 &IDTVEC(ioapic_intr11), 276 &IDTVEC(ioapic_intr12), 277 &IDTVEC(ioapic_intr13), 278 &IDTVEC(ioapic_intr14), 279 &IDTVEC(ioapic_intr15), 280 &IDTVEC(ioapic_intr16), 281 &IDTVEC(ioapic_intr17), 282 &IDTVEC(ioapic_intr18), 283 &IDTVEC(ioapic_intr19), 284 &IDTVEC(ioapic_intr20), 285 &IDTVEC(ioapic_intr21), 286 &IDTVEC(ioapic_intr22), 287 &IDTVEC(ioapic_intr23), 288 &IDTVEC(ioapic_intr24), 289 &IDTVEC(ioapic_intr25), 290 &IDTVEC(ioapic_intr26), 291 &IDTVEC(ioapic_intr27), 292 &IDTVEC(ioapic_intr28), 293 &IDTVEC(ioapic_intr29), 294 &IDTVEC(ioapic_intr30), 295 &IDTVEC(ioapic_intr31), 296 &IDTVEC(ioapic_intr32), 297 &IDTVEC(ioapic_intr33), 298 &IDTVEC(ioapic_intr34), 299 &IDTVEC(ioapic_intr35), 300 &IDTVEC(ioapic_intr36), 301 &IDTVEC(ioapic_intr37), 302 &IDTVEC(ioapic_intr38), 303 &IDTVEC(ioapic_intr39), 304 &IDTVEC(ioapic_intr40), 305 &IDTVEC(ioapic_intr41), 306 &IDTVEC(ioapic_intr42), 307 &IDTVEC(ioapic_intr43), 308 &IDTVEC(ioapic_intr44), 309 &IDTVEC(ioapic_intr45), 310 &IDTVEC(ioapic_intr46), 311 &IDTVEC(ioapic_intr47), 312 &IDTVEC(ioapic_intr48), 313 &IDTVEC(ioapic_intr49), 314 &IDTVEC(ioapic_intr50), 315 &IDTVEC(ioapic_intr51), 316 &IDTVEC(ioapic_intr52), 317 &IDTVEC(ioapic_intr53), 318 &IDTVEC(ioapic_intr54), 319 &IDTVEC(ioapic_intr55), 320 &IDTVEC(ioapic_intr56), 321 &IDTVEC(ioapic_intr57), 322 &IDTVEC(ioapic_intr58), 323 &IDTVEC(ioapic_intr59), 324 &IDTVEC(ioapic_intr60), 325 &IDTVEC(ioapic_intr61), 326 &IDTVEC(ioapic_intr62), 327 &IDTVEC(ioapic_intr63), 328 &IDTVEC(ioapic_intr64), 329 &IDTVEC(ioapic_intr65), 330 &IDTVEC(ioapic_intr66), 331 &IDTVEC(ioapic_intr67), 332 &IDTVEC(ioapic_intr68), 333 &IDTVEC(ioapic_intr69), 334 &IDTVEC(ioapic_intr70), 335 &IDTVEC(ioapic_intr71), 336 &IDTVEC(ioapic_intr72), 337 &IDTVEC(ioapic_intr73), 338 &IDTVEC(ioapic_intr74), 339 &IDTVEC(ioapic_intr75), 340 &IDTVEC(ioapic_intr76), 341 &IDTVEC(ioapic_intr77), 342 &IDTVEC(ioapic_intr78), 343 &IDTVEC(ioapic_intr79), 344 &IDTVEC(ioapic_intr80), 345 &IDTVEC(ioapic_intr81), 346 &IDTVEC(ioapic_intr82), 347 &IDTVEC(ioapic_intr83), 348 &IDTVEC(ioapic_intr84), 349 &IDTVEC(ioapic_intr85), 350 &IDTVEC(ioapic_intr86), 351 &IDTVEC(ioapic_intr87), 352 &IDTVEC(ioapic_intr88), 353 &IDTVEC(ioapic_intr89), 354 &IDTVEC(ioapic_intr90), 355 &IDTVEC(ioapic_intr91), 356 &IDTVEC(ioapic_intr92), 357 &IDTVEC(ioapic_intr93), 358 &IDTVEC(ioapic_intr94), 359 &IDTVEC(ioapic_intr95), 360 &IDTVEC(ioapic_intr96), 361 &IDTVEC(ioapic_intr97), 362 &IDTVEC(ioapic_intr98), 363 &IDTVEC(ioapic_intr99), 364 &IDTVEC(ioapic_intr100), 365 &IDTVEC(ioapic_intr101), 366 &IDTVEC(ioapic_intr102), 367 &IDTVEC(ioapic_intr103), 368 &IDTVEC(ioapic_intr104), 369 &IDTVEC(ioapic_intr105), 370 &IDTVEC(ioapic_intr106), 371 &IDTVEC(ioapic_intr107), 372 &IDTVEC(ioapic_intr108), 373 &IDTVEC(ioapic_intr109), 374 &IDTVEC(ioapic_intr110), 375 &IDTVEC(ioapic_intr111), 376 &IDTVEC(ioapic_intr112), 377 &IDTVEC(ioapic_intr113), 378 &IDTVEC(ioapic_intr114), 379 &IDTVEC(ioapic_intr115), 380 &IDTVEC(ioapic_intr116), 381 &IDTVEC(ioapic_intr117), 382 &IDTVEC(ioapic_intr118), 383 &IDTVEC(ioapic_intr119), 384 &IDTVEC(ioapic_intr120), 385 &IDTVEC(ioapic_intr121), 386 &IDTVEC(ioapic_intr122), 387 &IDTVEC(ioapic_intr123), 388 &IDTVEC(ioapic_intr124), 389 &IDTVEC(ioapic_intr125), 390 &IDTVEC(ioapic_intr126), 391 &IDTVEC(ioapic_intr127), 392 &IDTVEC(ioapic_intr128), 393 &IDTVEC(ioapic_intr129), 394 &IDTVEC(ioapic_intr130), 395 &IDTVEC(ioapic_intr131), 396 &IDTVEC(ioapic_intr132), 397 &IDTVEC(ioapic_intr133), 398 &IDTVEC(ioapic_intr134), 399 &IDTVEC(ioapic_intr135), 400 &IDTVEC(ioapic_intr136), 401 &IDTVEC(ioapic_intr137), 402 &IDTVEC(ioapic_intr138), 403 &IDTVEC(ioapic_intr139), 404 &IDTVEC(ioapic_intr140), 405 &IDTVEC(ioapic_intr141), 406 &IDTVEC(ioapic_intr142), 407 &IDTVEC(ioapic_intr143), 408 &IDTVEC(ioapic_intr144), 409 &IDTVEC(ioapic_intr145), 410 &IDTVEC(ioapic_intr146), 411 &IDTVEC(ioapic_intr147), 412 &IDTVEC(ioapic_intr148), 413 &IDTVEC(ioapic_intr149), 414 &IDTVEC(ioapic_intr150), 415 &IDTVEC(ioapic_intr151), 416 &IDTVEC(ioapic_intr152), 417 &IDTVEC(ioapic_intr153), 418 &IDTVEC(ioapic_intr154), 419 &IDTVEC(ioapic_intr155), 420 &IDTVEC(ioapic_intr156), 421 &IDTVEC(ioapic_intr157), 422 &IDTVEC(ioapic_intr158), 423 &IDTVEC(ioapic_intr159), 424 &IDTVEC(ioapic_intr160), 425 &IDTVEC(ioapic_intr161), 426 &IDTVEC(ioapic_intr162), 427 &IDTVEC(ioapic_intr163), 428 &IDTVEC(ioapic_intr164), 429 &IDTVEC(ioapic_intr165), 430 &IDTVEC(ioapic_intr166), 431 &IDTVEC(ioapic_intr167), 432 &IDTVEC(ioapic_intr168), 433 &IDTVEC(ioapic_intr169), 434 &IDTVEC(ioapic_intr170), 435 &IDTVEC(ioapic_intr171), 436 &IDTVEC(ioapic_intr172), 437 &IDTVEC(ioapic_intr173), 438 &IDTVEC(ioapic_intr174), 439 &IDTVEC(ioapic_intr175), 440 &IDTVEC(ioapic_intr176), 441 &IDTVEC(ioapic_intr177), 442 &IDTVEC(ioapic_intr178), 443 &IDTVEC(ioapic_intr179), 444 &IDTVEC(ioapic_intr180), 445 &IDTVEC(ioapic_intr181), 446 &IDTVEC(ioapic_intr182), 447 &IDTVEC(ioapic_intr183), 448 &IDTVEC(ioapic_intr184), 449 &IDTVEC(ioapic_intr185), 450 &IDTVEC(ioapic_intr186), 451 &IDTVEC(ioapic_intr187), 452 &IDTVEC(ioapic_intr188), 453 &IDTVEC(ioapic_intr189), 454 &IDTVEC(ioapic_intr190), 455 &IDTVEC(ioapic_intr191) 456 }; 457 458 #define IOAPIC_HWI_SYSCALL (IDT_OFFSET_SYSCALL - IDT_OFFSET) 459 460 /* 461 * NOTE: Initialized before VM so cannot use kmalloc() for this array. 462 */ 463 static struct ioapic_irqmap { 464 int im_type; /* IOAPIC_IMT_ */ 465 enum intr_trigger im_trig; 466 enum intr_polarity im_pola; 467 int im_gsi; 468 int im_msi_base; 469 uint32_t im_flags; /* IOAPIC_IMF_ */ 470 } ioapic_irqmaps[MAXCPU][IOAPIC_HWI_VECTORS]; 471 472 static struct lwkt_token ioapic_irqmap_tok = 473 LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token); 474 475 #define IOAPIC_IMT_UNUSED 0 476 #define IOAPIC_IMT_RESERVED 1 477 #define IOAPIC_IMT_LEGACY 2 478 #define IOAPIC_IMT_SYSCALL 3 479 #define IOAPIC_IMT_MSI 4 480 #define IOAPIC_IMT_MSIX 5 481 482 #define IOAPIC_IMT_ISHWI(map) ((map)->im_type != IOAPIC_IMT_RESERVED && \ 483 (map)->im_type != IOAPIC_IMT_SYSCALL) 484 485 #define IOAPIC_IMF_CONF 0x1 486 487 extern void IOAPIC_INTREN(int); 488 extern void IOAPIC_INTRDIS(int); 489 490 extern int imcr_present; 491 492 static void ioapic_abi_intr_enable(int); 493 static void ioapic_abi_intr_disable(int); 494 static void ioapic_abi_intr_setup(int, int); 495 static void ioapic_abi_intr_teardown(int); 496 497 static void ioapic_abi_legacy_intr_config(int, 498 enum intr_trigger, enum intr_polarity); 499 static int ioapic_abi_legacy_intr_cpuid(int); 500 static int ioapic_abi_legacy_intr_find(int, 501 enum intr_trigger, enum intr_polarity); 502 static int ioapic_abi_legacy_intr_find_bygsi(int, 503 enum intr_trigger, enum intr_polarity); 504 505 static int ioapic_abi_msi_alloc(int [], int, int); 506 static void ioapic_abi_msi_release(const int [], int, int); 507 static void ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int); 508 static int ioapic_abi_msix_alloc(int *, int); 509 static void ioapic_abi_msix_release(int, int); 510 511 static int ioapic_abi_msi_alloc_intern(int, const char *, 512 int [], int, int); 513 static void ioapic_abi_msi_release_intern(int, const char *, 514 const int [], int, int); 515 516 static void ioapic_abi_finalize(void); 517 static void ioapic_abi_cleanup(void); 518 static void ioapic_abi_setdefault(void); 519 static void ioapic_abi_stabilize(void); 520 static void ioapic_abi_initmap(void); 521 static void ioapic_abi_rman_setup(struct rman *); 522 523 static int ioapic_abi_gsi_cpuid(int, int); 524 static int ioapic_unused_legacy_irqmap(void); 525 static int ioapic_is_legacy_irqmap_used(int); 526 527 struct machintr_abi MachIntrABI_IOAPIC = { 528 MACHINTR_IOAPIC, 529 .intr_disable = ioapic_abi_intr_disable, 530 .intr_enable = ioapic_abi_intr_enable, 531 .intr_setup = ioapic_abi_intr_setup, 532 .intr_teardown = ioapic_abi_intr_teardown, 533 534 .legacy_intr_config = ioapic_abi_legacy_intr_config, 535 .legacy_intr_cpuid = ioapic_abi_legacy_intr_cpuid, 536 .legacy_intr_find = ioapic_abi_legacy_intr_find, 537 .legacy_intr_find_bygsi = ioapic_abi_legacy_intr_find_bygsi, 538 539 .msi_alloc = ioapic_abi_msi_alloc, 540 .msi_release = ioapic_abi_msi_release, 541 .msi_map = ioapic_abi_msi_map, 542 .msix_alloc = ioapic_abi_msix_alloc, 543 .msix_release = ioapic_abi_msix_release, 544 545 .finalize = ioapic_abi_finalize, 546 .cleanup = ioapic_abi_cleanup, 547 .setdefault = ioapic_abi_setdefault, 548 .stabilize = ioapic_abi_stabilize, 549 .initmap = ioapic_abi_initmap, 550 .rman_setup = ioapic_abi_rman_setup 551 }; 552 553 static int ioapic_abi_extint_irq = -1; 554 static int ioapic_abi_legacy_irq_max; 555 static int ioapic_abi_gsi_balance = 1; 556 static int ioapic_abi_msi_start; /* NOTE: for testing only */ 557 558 struct ioapic_irqinfo ioapic_irqs[IOAPIC_HWI_VECTORS]; 559 560 static void 561 ioapic_abi_intr_enable(int irq) 562 { 563 const struct ioapic_irqmap *map; 564 565 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 566 ("ioapic enable, invalid irq %d", irq)); 567 568 map = &ioapic_irqmaps[mycpuid][irq]; 569 KASSERT(IOAPIC_IMT_ISHWI(map), 570 ("ioapic enable, not hwi irq %d, type %d, cpu%d", 571 irq, map->im_type, mycpuid)); 572 if (map->im_type != IOAPIC_IMT_LEGACY) 573 return; 574 575 IOAPIC_INTREN(irq); 576 } 577 578 static void 579 ioapic_abi_intr_disable(int irq) 580 { 581 const struct ioapic_irqmap *map; 582 583 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 584 ("ioapic disable, invalid irq %d", irq)); 585 586 map = &ioapic_irqmaps[mycpuid][irq]; 587 KASSERT(IOAPIC_IMT_ISHWI(map), 588 ("ioapic disable, not hwi irq %d, type %d, cpu%d", 589 irq, map->im_type, mycpuid)); 590 if (map->im_type != IOAPIC_IMT_LEGACY) 591 return; 592 593 IOAPIC_INTRDIS(irq); 594 } 595 596 static void 597 ioapic_abi_finalize(void) 598 { 599 KKASSERT(MachIntrABI.type == MACHINTR_IOAPIC); 600 KKASSERT(ioapic_enable); 601 602 /* 603 * If an IMCR is present, program bit 0 to disconnect the 8259 604 * from the BSP. 605 */ 606 if (imcr_present) { 607 outb(0x22, 0x70); /* select IMCR */ 608 outb(0x23, 0x01); /* disconnect 8259 */ 609 } 610 } 611 612 /* 613 * This routine is called after physical interrupts are enabled but before 614 * the critical section is released. We need to clean out any interrupts 615 * that had already been posted to the cpu. 616 */ 617 static void 618 ioapic_abi_cleanup(void) 619 { 620 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending)); 621 } 622 623 /* Must never be called */ 624 static void 625 ioapic_abi_stabilize(void) 626 { 627 panic("ioapic_stabilize is called"); 628 } 629 630 static void 631 ioapic_abi_intr_setup(int intr, int flags) 632 { 633 const struct ioapic_irqmap *map; 634 int vector, select; 635 uint32_t value; 636 register_t ef; 637 638 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 639 ("ioapic setup, invalid irq %d", intr)); 640 641 map = &ioapic_irqmaps[mycpuid][intr]; 642 KASSERT(IOAPIC_IMT_ISHWI(map), 643 ("ioapic setup, not hwi irq %d, type %d, cpu%d", 644 intr, map->im_type, mycpuid)); 645 if (map->im_type != IOAPIC_IMT_LEGACY) 646 return; 647 648 KASSERT(ioapic_irqs[intr].io_addr != NULL, 649 ("ioapic setup, no GSI information, irq %d", intr)); 650 651 ef = read_rflags(); 652 cpu_disable_intr(); 653 654 vector = IDT_OFFSET + intr; 655 656 /* 657 * Now reprogram the vector in the IO APIC. In order to avoid 658 * losing an EOI for a level interrupt, which is vector based, 659 * make sure that the IO APIC is programmed for edge-triggering 660 * first, then reprogrammed with the new vector. This should 661 * clear the IRR bit. 662 */ 663 imen_lock(); 664 665 select = ioapic_irqs[intr].io_idx; 666 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 667 value |= IOART_INTMSET; 668 669 ioapic_write(ioapic_irqs[intr].io_addr, select, 670 (value & ~APIC_TRIGMOD_MASK)); 671 ioapic_write(ioapic_irqs[intr].io_addr, select, 672 (value & ~IOART_INTVEC) | vector); 673 674 imen_unlock(); 675 676 IOAPIC_INTREN(intr); 677 678 write_rflags(ef); 679 } 680 681 static void 682 ioapic_abi_intr_teardown(int intr) 683 { 684 const struct ioapic_irqmap *map; 685 int vector, select; 686 uint32_t value; 687 register_t ef; 688 689 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 690 ("ioapic teardown, invalid irq %d", intr)); 691 692 map = &ioapic_irqmaps[mycpuid][intr]; 693 KASSERT(IOAPIC_IMT_ISHWI(map), 694 ("ioapic teardown, not hwi irq %d, type %d, cpu%d", 695 intr, map->im_type, mycpuid)); 696 if (map->im_type != IOAPIC_IMT_LEGACY) 697 return; 698 699 KASSERT(ioapic_irqs[intr].io_addr != NULL, 700 ("ioapic teardown, no GSI information, irq %d", intr)); 701 702 ef = read_rflags(); 703 cpu_disable_intr(); 704 705 /* 706 * Teardown an interrupt vector. The vector should already be 707 * installed in the cpu's IDT, but make sure. 708 */ 709 IOAPIC_INTRDIS(intr); 710 711 vector = IDT_OFFSET + intr; 712 713 /* 714 * In order to avoid losing an EOI for a level interrupt, which 715 * is vector based, make sure that the IO APIC is programmed for 716 * edge-triggering first, then reprogrammed with the new vector. 717 * This should clear the IRR bit. 718 */ 719 imen_lock(); 720 721 select = ioapic_irqs[intr].io_idx; 722 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 723 724 ioapic_write(ioapic_irqs[intr].io_addr, select, 725 (value & ~APIC_TRIGMOD_MASK)); 726 ioapic_write(ioapic_irqs[intr].io_addr, select, 727 (value & ~IOART_INTVEC) | vector); 728 729 imen_unlock(); 730 731 write_rflags(ef); 732 } 733 734 static void 735 ioapic_abi_setdefault(void) 736 { 737 int intr; 738 739 for (intr = 0; intr < IOAPIC_HWI_VECTORS; ++intr) { 740 if (intr == IOAPIC_HWI_SYSCALL) 741 continue; 742 setidt_global(IDT_OFFSET + intr, ioapic_intr[intr], 743 SDT_SYSIGT, SEL_KPL, 0); 744 } 745 } 746 747 static void 748 ioapic_abi_initmap(void) 749 { 750 int cpu; 751 752 kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance); 753 754 kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start); 755 ioapic_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */ 756 757 /* 758 * NOTE: ncpus is not ready yet 759 */ 760 for (cpu = 0; cpu < MAXCPU; ++cpu) { 761 int i; 762 763 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 764 ioapic_irqmaps[cpu][i].im_gsi = -1; 765 ioapic_irqmaps[cpu][i].im_msi_base = -1; 766 } 767 ioapic_irqmaps[cpu][IOAPIC_HWI_SYSCALL].im_type = 768 IOAPIC_IMT_SYSCALL; 769 } 770 } 771 772 /* 773 * Only one CPU can have the legacy IRQ map. 774 */ 775 static int 776 ioapic_is_legacy_irqmap_used(int irq) 777 { 778 int cpu; 779 780 for (cpu = 0; cpu < ncpus; ++cpu) { 781 if (ioapic_irqmaps[cpu][irq].im_type != IOAPIC_IMT_UNUSED) 782 return 1; 783 } 784 return 0; 785 } 786 787 static int 788 ioapic_unused_legacy_irqmap(void) 789 { 790 int i; 791 792 for (i = ISA_IRQ_CNT; i < IOAPIC_HWI_VECTORS; ++i) { 793 if (i == acpi_sci_irqno()) 794 continue; 795 if (!ioapic_is_legacy_irqmap_used(i)) 796 return i; 797 } 798 return -1; 799 } 800 801 void 802 ioapic_set_legacy_irqmap(int irq, int gsi, enum intr_trigger trig, 803 enum intr_polarity pola) 804 { 805 struct ioapic_irqinfo *info; 806 struct ioapic_irqmap *map; 807 void *ioaddr; 808 int pin, cpuid; 809 810 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 811 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 812 813 KKASSERT(irq >= 0); 814 if (irq >= IOAPIC_HWI_VECTORS) { 815 /* 816 * Some BIOSes seem to assume that all 256 IDT vectors 817 * could be used, while we limit the available IDT 818 * vectors to 192; find an unused IRQ for this GSI. 819 */ 820 irq = ioapic_unused_legacy_irqmap(); 821 if (irq < 0) { 822 kprintf("failed to find unused irq for gsi %d, " 823 "overflow\n", gsi); 824 return; 825 } 826 } 827 KKASSERT(irq < IOAPIC_HWI_VECTORS); 828 829 if (ioapic_is_legacy_irqmap_used(irq)) { 830 /* 831 * There are so many IOAPICs, that 1:1 mapping 832 * of GSI and IRQ hits SYSCALL entry. 833 */ 834 irq = ioapic_unused_legacy_irqmap(); 835 if (irq < 0) { 836 kprintf("failed to find unused irq for gsi %d, " 837 "conflict\n", gsi); 838 return; 839 } 840 KKASSERT(irq < IOAPIC_HWI_VECTORS); 841 842 } 843 cpuid = ioapic_abi_gsi_cpuid(irq, gsi); 844 map = &ioapic_irqmaps[cpuid][irq]; 845 846 if (irq > ioapic_abi_legacy_irq_max) 847 ioapic_abi_legacy_irq_max = irq; 848 849 KKASSERT(map->im_type == IOAPIC_IMT_UNUSED); 850 map->im_type = IOAPIC_IMT_LEGACY; 851 852 map->im_gsi = gsi; 853 map->im_trig = trig; 854 map->im_pola = pola; 855 856 if (bootverbose) { 857 kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n", 858 irq, map->im_gsi, 859 intr_str_trigger(map->im_trig), 860 intr_str_polarity(map->im_pola)); 861 } 862 863 pin = ioapic_gsi_pin(map->im_gsi); 864 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 865 866 info = &ioapic_irqs[irq]; 867 868 imen_lock(); 869 870 info->io_addr = ioaddr; 871 info->io_idx = IOAPIC_REDTBL + (2 * pin); 872 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 873 if (map->im_trig == INTR_TRIGGER_LEVEL) 874 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 875 876 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 877 map->im_trig, map->im_pola, cpuid); 878 879 imen_unlock(); 880 } 881 882 void 883 ioapic_fixup_legacy_irqmaps(void) 884 { 885 int cpu; 886 887 for (cpu = 0; cpu < ncpus; ++cpu) { 888 int i; 889 890 for (i = 0; i < ISA_IRQ_CNT; ++i) { 891 struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][i]; 892 893 if (map->im_type == IOAPIC_IMT_UNUSED) { 894 map->im_type = IOAPIC_IMT_RESERVED; 895 if (bootverbose) { 896 kprintf("IOAPIC: " 897 "cpu%d irq %d reserved\n", cpu, i); 898 } 899 } 900 } 901 } 902 903 ioapic_abi_legacy_irq_max += 1; 904 if (bootverbose) { 905 kprintf("IOAPIC: legacy irq max %d\n", 906 ioapic_abi_legacy_irq_max); 907 } 908 } 909 910 static int 911 ioapic_abi_legacy_intr_find_bygsi(int gsi, enum intr_trigger trig, 912 enum intr_polarity pola) 913 { 914 int cpu; 915 916 #ifdef INVARIANTS 917 if (trig == INTR_TRIGGER_CONFORM) { 918 KKASSERT(pola == INTR_POLARITY_CONFORM); 919 } else { 920 KKASSERT(trig == INTR_TRIGGER_EDGE || 921 trig == INTR_TRIGGER_LEVEL); 922 KKASSERT(pola == INTR_POLARITY_HIGH || 923 pola == INTR_POLARITY_LOW); 924 } 925 #endif 926 927 for (cpu = 0; cpu < ncpus; ++cpu) { 928 int irq; 929 930 for (irq = 0; irq < ioapic_abi_legacy_irq_max; ++irq) { 931 const struct ioapic_irqmap *map = 932 &ioapic_irqmaps[cpu][irq]; 933 934 if (map->im_gsi == gsi) { 935 KKASSERT(map->im_type == IOAPIC_IMT_LEGACY); 936 937 if ((map->im_flags & IOAPIC_IMF_CONF) && 938 trig != INTR_TRIGGER_CONFORM && 939 pola != INTR_POLARITY_CONFORM) { 940 if (map->im_trig != trig || 941 map->im_pola != pola) 942 return -1; 943 } 944 return irq; 945 } 946 } 947 } 948 return -1; 949 } 950 951 static int 952 ioapic_abi_legacy_intr_find(int irq, enum intr_trigger trig, 953 enum intr_polarity pola) 954 { 955 int cpu; 956 957 #ifdef INVARIANTS 958 if (trig == INTR_TRIGGER_CONFORM) { 959 KKASSERT(pola == INTR_POLARITY_CONFORM); 960 } else { 961 KKASSERT(trig == INTR_TRIGGER_EDGE || 962 trig == INTR_TRIGGER_LEVEL); 963 KKASSERT(pola == INTR_POLARITY_HIGH || 964 pola == INTR_POLARITY_LOW); 965 } 966 #endif 967 968 if (irq < 0 || irq >= ioapic_abi_legacy_irq_max) 969 return -1; 970 971 for (cpu = 0; cpu < ncpus; ++cpu) { 972 const struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][irq]; 973 974 if (map->im_type == IOAPIC_IMT_LEGACY) { 975 if ((map->im_flags & IOAPIC_IMF_CONF) && 976 trig != INTR_TRIGGER_CONFORM && 977 pola != INTR_POLARITY_CONFORM) { 978 if (map->im_trig != trig || 979 map->im_pola != pola) 980 return -1; 981 } 982 return irq; 983 } 984 } 985 return -1; 986 } 987 988 static void 989 ioapic_abi_legacy_intr_config(int irq, enum intr_trigger trig, 990 enum intr_polarity pola) 991 { 992 struct ioapic_irqinfo *info; 993 struct ioapic_irqmap *map = NULL; 994 void *ioaddr; 995 int pin, cpuid; 996 997 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 998 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 999 1000 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 1001 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1002 map = &ioapic_irqmaps[cpuid][irq]; 1003 if (map->im_type == IOAPIC_IMT_LEGACY) 1004 break; 1005 } 1006 KKASSERT(cpuid < ncpus); 1007 1008 #ifdef notyet 1009 if (map->im_flags & IOAPIC_IMF_CONF) { 1010 if (trig != map->im_trig) { 1011 panic("ioapic_intr_config: trig %s -> %s", 1012 intr_str_trigger(map->im_trig), 1013 intr_str_trigger(trig)); 1014 } 1015 if (pola != map->im_pola) { 1016 panic("ioapic_intr_config: pola %s -> %s", 1017 intr_str_polarity(map->im_pola), 1018 intr_str_polarity(pola)); 1019 } 1020 return; 1021 } 1022 #endif 1023 map->im_flags |= IOAPIC_IMF_CONF; 1024 1025 if (trig == map->im_trig && pola == map->im_pola) 1026 return; 1027 1028 if (bootverbose) { 1029 kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n", 1030 irq, map->im_gsi, 1031 intr_str_trigger(map->im_trig), 1032 intr_str_polarity(map->im_pola), 1033 intr_str_trigger(trig), 1034 intr_str_polarity(pola)); 1035 } 1036 map->im_trig = trig; 1037 map->im_pola = pola; 1038 1039 pin = ioapic_gsi_pin(map->im_gsi); 1040 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1041 1042 info = &ioapic_irqs[irq]; 1043 1044 imen_lock(); 1045 1046 info->io_flags &= ~IOAPIC_IRQI_FLAG_LEVEL; 1047 if (map->im_trig == INTR_TRIGGER_LEVEL) 1048 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 1049 1050 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 1051 map->im_trig, map->im_pola, cpuid); 1052 1053 imen_unlock(); 1054 } 1055 1056 int 1057 ioapic_conf_legacy_extint(int irq) 1058 { 1059 struct ioapic_irqinfo *info; 1060 struct ioapic_irqmap *map; 1061 void *ioaddr; 1062 int pin, error, vec; 1063 1064 /* XXX only irq0 is allowed */ 1065 KKASSERT(irq == 0); 1066 1067 vec = IDT_OFFSET + irq; 1068 1069 if (ioapic_abi_extint_irq == irq) 1070 return 0; 1071 else if (ioapic_abi_extint_irq >= 0) 1072 return EEXIST; 1073 1074 error = icu_ioapic_extint(irq, vec); 1075 if (error) 1076 return error; 1077 1078 /* ExtINT is always targeted to cpu0 */ 1079 map = &ioapic_irqmaps[0][irq]; 1080 1081 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED || 1082 map->im_type == IOAPIC_IMT_LEGACY); 1083 if (map->im_type == IOAPIC_IMT_LEGACY) { 1084 if (map->im_flags & IOAPIC_IMF_CONF) 1085 return EEXIST; 1086 } 1087 ioapic_abi_extint_irq = irq; 1088 1089 map->im_type = IOAPIC_IMT_LEGACY; 1090 map->im_trig = INTR_TRIGGER_EDGE; 1091 map->im_pola = INTR_POLARITY_HIGH; 1092 map->im_flags = IOAPIC_IMF_CONF; 1093 1094 map->im_gsi = ioapic_extpin_gsi(); 1095 KKASSERT(map->im_gsi >= 0); 1096 1097 if (bootverbose) { 1098 kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n", 1099 irq, map->im_gsi, 1100 intr_str_trigger(map->im_trig), 1101 intr_str_polarity(map->im_pola)); 1102 } 1103 1104 pin = ioapic_gsi_pin(map->im_gsi); 1105 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1106 1107 info = &ioapic_irqs[irq]; 1108 1109 imen_lock(); 1110 1111 info->io_addr = ioaddr; 1112 info->io_idx = IOAPIC_REDTBL + (2 * pin); 1113 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 1114 1115 ioapic_extpin_setup(ioaddr, pin, vec); 1116 1117 imen_unlock(); 1118 1119 return 0; 1120 } 1121 1122 static int 1123 ioapic_abi_legacy_intr_cpuid(int irq) 1124 { 1125 const struct ioapic_irqmap *map = NULL; 1126 int cpuid; 1127 1128 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 1129 1130 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1131 map = &ioapic_irqmaps[cpuid][irq]; 1132 if (map->im_type == IOAPIC_IMT_LEGACY) 1133 return cpuid; 1134 } 1135 1136 /* XXX some drivers tries to peek at reserved IRQs */ 1137 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1138 map = &ioapic_irqmaps[cpuid][irq]; 1139 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED); 1140 } 1141 return 0; 1142 } 1143 1144 static int 1145 ioapic_abi_gsi_cpuid(int irq, int gsi) 1146 { 1147 char envpath[32]; 1148 int cpuid = -1; 1149 1150 KKASSERT(gsi >= 0); 1151 1152 if (irq == 0 || gsi == 0) { 1153 KKASSERT(irq >= 0); 1154 if (bootverbose) { 1155 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n", 1156 irq, gsi); 1157 } 1158 return 0; 1159 } 1160 1161 if (irq >= 0 && irq == acpi_sci_irqno()) { 1162 if (bootverbose) { 1163 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n", 1164 irq, gsi); 1165 } 1166 return 0; 1167 } 1168 1169 ksnprintf(envpath, sizeof(envpath), "hw.ioapic.gsi.%d.cpu", gsi); 1170 kgetenv_int(envpath, &cpuid); 1171 1172 if (cpuid < 0) { 1173 if (!ioapic_abi_gsi_balance) { 1174 if (irq >= 0 && bootverbose) { 1175 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 " 1176 "(fixed)\n", irq, gsi); 1177 } 1178 return 0; 1179 } 1180 1181 cpuid = gsi % ncpus; 1182 if (irq >= 0 && bootverbose) { 1183 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n", 1184 irq, gsi, cpuid); 1185 } 1186 } else if (cpuid >= ncpus) { 1187 cpuid = ncpus - 1; 1188 if (irq >= 0 && bootverbose) { 1189 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n", 1190 irq, gsi, cpuid); 1191 } 1192 } else { 1193 if (irq >= 0 && bootverbose) { 1194 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n", 1195 irq, gsi, cpuid); 1196 } 1197 } 1198 return cpuid; 1199 } 1200 1201 static void 1202 ioapic_abi_rman_setup(struct rman *rm) 1203 { 1204 int start, end, i; 1205 1206 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU, 1207 ("invalid rman cpuid %d", rm->rm_cpuid)); 1208 1209 start = end = -1; 1210 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 1211 const struct ioapic_irqmap *map = 1212 &ioapic_irqmaps[rm->rm_cpuid][i]; 1213 1214 if (start < 0) { 1215 if (IOAPIC_IMT_ISHWI(map)) 1216 start = end = i; 1217 } else { 1218 if (IOAPIC_IMT_ISHWI(map)) { 1219 end = i; 1220 } else { 1221 KKASSERT(end >= 0); 1222 if (bootverbose) { 1223 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1224 rm->rm_cpuid, start, end); 1225 } 1226 if (rman_manage_region(rm, start, end)) { 1227 panic("rman_manage_region" 1228 "(cpu%d %d - %d)", rm->rm_cpuid, 1229 start, end); 1230 } 1231 start = end = -1; 1232 } 1233 } 1234 } 1235 if (start >= 0) { 1236 KKASSERT(end >= 0); 1237 if (bootverbose) { 1238 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1239 rm->rm_cpuid, start, end); 1240 } 1241 if (rman_manage_region(rm, start, end)) { 1242 panic("rman_manage_region(cpu%d %d - %d)", 1243 rm->rm_cpuid, start, end); 1244 } 1245 } 1246 } 1247 1248 static int 1249 ioapic_abi_msi_alloc_intern(int type, const char *desc, 1250 int intrs[], int count, int cpuid) 1251 { 1252 int i, error; 1253 1254 KASSERT(cpuid >= 0 && cpuid < ncpus, 1255 ("invalid cpuid %d", cpuid)); 1256 1257 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 1258 KASSERT((count & (count - 1)) == 0, 1259 ("count %d is not power of 2", count)); 1260 1261 lwkt_gettoken(&ioapic_irqmap_tok); 1262 1263 /* 1264 * NOTE: 1265 * Since IDT_OFFSET is 32, which is the maximum valid 'count', 1266 * we do not need to find out the first properly aligned 1267 * interrupt vector. 1268 */ 1269 1270 error = EMSGSIZE; 1271 for (i = ioapic_abi_msi_start; i < IOAPIC_HWI_VECTORS; i += count) { 1272 int j; 1273 1274 if (ioapic_irqmaps[cpuid][i].im_type != IOAPIC_IMT_UNUSED) 1275 continue; 1276 1277 for (j = 1; j < count; ++j) { 1278 if (ioapic_irqmaps[cpuid][i + j].im_type != 1279 IOAPIC_IMT_UNUSED) 1280 break; 1281 } 1282 if (j != count) 1283 continue; 1284 1285 for (j = 0; j < count; ++j) { 1286 struct ioapic_irqmap *map; 1287 int intr = i + j; 1288 1289 map = &ioapic_irqmaps[cpuid][intr]; 1290 KASSERT(map->im_msi_base < 0, 1291 ("intr %d, stale %s-base %d", 1292 intr, desc, map->im_msi_base)); 1293 1294 map->im_type = type; 1295 map->im_msi_base = i; 1296 1297 intrs[j] = intr; 1298 msi_setup(intr, cpuid); 1299 1300 if (bootverbose) { 1301 kprintf("alloc %s intr %d on cpu%d\n", 1302 desc, intr, cpuid); 1303 } 1304 } 1305 error = 0; 1306 break; 1307 } 1308 1309 lwkt_reltoken(&ioapic_irqmap_tok); 1310 1311 return error; 1312 } 1313 1314 static void 1315 ioapic_abi_msi_release_intern(int type, const char *desc, 1316 const int intrs[], int count, int cpuid) 1317 { 1318 int i, msi_base = -1, intr_next = -1, mask; 1319 1320 KASSERT(cpuid >= 0 && cpuid < ncpus, 1321 ("invalid cpuid %d", cpuid)); 1322 1323 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 1324 1325 mask = count - 1; 1326 KASSERT((count & mask) == 0, ("count %d is not power of 2", count)); 1327 1328 lwkt_gettoken(&ioapic_irqmap_tok); 1329 1330 for (i = 0; i < count; ++i) { 1331 struct ioapic_irqmap *map; 1332 int intr = intrs[i]; 1333 1334 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1335 ("invalid intr %d", intr)); 1336 1337 map = &ioapic_irqmaps[cpuid][intr]; 1338 KASSERT(map->im_type == type, 1339 ("trying to release non-%s intr %d, type %d", desc, 1340 intr, map->im_type)); 1341 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1342 ("intr %d, invalid %s-base %d", intr, desc, 1343 map->im_msi_base)); 1344 KASSERT((map->im_msi_base & mask) == 0, 1345 ("intr %d, %s-base %d is not properly aligned %d", 1346 intr, desc, map->im_msi_base, count)); 1347 1348 if (msi_base < 0) { 1349 msi_base = map->im_msi_base; 1350 } else { 1351 KASSERT(map->im_msi_base == msi_base, 1352 ("intr %d, inconsistent %s-base, " 1353 "was %d, now %d", 1354 intr, desc, msi_base, map->im_msi_base)); 1355 } 1356 1357 if (intr_next < intr) 1358 intr_next = intr; 1359 1360 map->im_type = IOAPIC_IMT_UNUSED; 1361 map->im_msi_base = -1; 1362 1363 if (bootverbose) { 1364 kprintf("release %s intr %d on cpu%d\n", 1365 desc, intr, cpuid); 1366 } 1367 } 1368 1369 KKASSERT(intr_next > 0); 1370 KKASSERT(msi_base >= 0); 1371 1372 ++intr_next; 1373 if (intr_next < IOAPIC_HWI_VECTORS) { 1374 const struct ioapic_irqmap *map = 1375 &ioapic_irqmaps[cpuid][intr_next]; 1376 1377 if (map->im_type == type) { 1378 KASSERT(map->im_msi_base != msi_base, 1379 ("more than %d %s was allocated", count, desc)); 1380 } 1381 } 1382 1383 lwkt_reltoken(&ioapic_irqmap_tok); 1384 } 1385 1386 static int 1387 ioapic_abi_msi_alloc(int intrs[], int count, int cpuid) 1388 { 1389 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSI, "MSI", 1390 intrs, count, cpuid); 1391 } 1392 1393 static void 1394 ioapic_abi_msi_release(const int intrs[], int count, int cpuid) 1395 { 1396 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSI, "MSI", 1397 intrs, count, cpuid); 1398 } 1399 1400 static int 1401 ioapic_abi_msix_alloc(int *intr, int cpuid) 1402 { 1403 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSIX, "MSI-X", 1404 intr, 1, cpuid); 1405 } 1406 1407 static void 1408 ioapic_abi_msix_release(int intr, int cpuid) 1409 { 1410 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSIX, "MSI-X", 1411 &intr, 1, cpuid); 1412 } 1413 1414 static void 1415 ioapic_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid) 1416 { 1417 const struct ioapic_irqmap *map; 1418 1419 KASSERT(cpuid >= 0 && cpuid < ncpus, 1420 ("invalid cpuid %d", cpuid)); 1421 1422 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1423 ("invalid intr %d", intr)); 1424 1425 lwkt_gettoken(&ioapic_irqmap_tok); 1426 1427 map = &ioapic_irqmaps[cpuid][intr]; 1428 KASSERT(map->im_type == IOAPIC_IMT_MSI || 1429 map->im_type == IOAPIC_IMT_MSIX, 1430 ("trying to map non-MSI/MSI-X intr %d, type %d", intr, map->im_type)); 1431 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1432 ("intr %d, invalid %s-base %d", intr, 1433 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1434 map->im_msi_base)); 1435 1436 msi_map(map->im_msi_base, addr, data, cpuid); 1437 1438 if (bootverbose) { 1439 kprintf("map %s intr %d on cpu%d\n", 1440 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1441 intr, cpuid); 1442 } 1443 1444 lwkt_reltoken(&ioapic_irqmap_tok); 1445 } 1446