1 /* 2 * Copyright (c) 1991 The Regents of the University of California. 3 * Copyright (c) 1996, by Steve Passe. All rights reserved. 4 * Copyright (c) 2005,2008 The DragonFly Project. All rights reserved. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The DragonFly Project 8 * by Matthew Dillon <dillon@backplane.com> 9 * 10 * This code is derived from software contributed to Berkeley by 11 * William Jolitz. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 3. Neither the name of The DragonFly Project nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific, prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 30 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 31 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 34 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 35 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 36 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 37 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 */ 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/kernel.h> 44 #include <sys/machintr.h> 45 #include <sys/interrupt.h> 46 #include <sys/bus.h> 47 #include <sys/rman.h> 48 #include <sys/thread2.h> 49 50 #include <machine/smp.h> 51 #include <machine/segments.h> 52 #include <machine/md_var.h> 53 #include <machine/intr_machdep.h> 54 #include <machine/globaldata.h> 55 #include <machine/msi_var.h> 56 57 #include <machine_base/isa/isa_intr.h> 58 #include <machine_base/icu/icu.h> 59 #include <machine_base/icu/icu_var.h> 60 #include <machine_base/apic/ioapic.h> 61 #include <machine_base/apic/ioapic_abi.h> 62 #include <machine_base/apic/ioapic_ipl.h> 63 #include <machine_base/apic/apicreg.h> 64 65 #include <dev/acpica/acpi_sci_var.h> 66 67 #define IOAPIC_HWI_VECTORS IDT_HWI_VECTORS 68 69 extern inthand_t 70 IDTVEC(ioapic_intr0), 71 IDTVEC(ioapic_intr1), 72 IDTVEC(ioapic_intr2), 73 IDTVEC(ioapic_intr3), 74 IDTVEC(ioapic_intr4), 75 IDTVEC(ioapic_intr5), 76 IDTVEC(ioapic_intr6), 77 IDTVEC(ioapic_intr7), 78 IDTVEC(ioapic_intr8), 79 IDTVEC(ioapic_intr9), 80 IDTVEC(ioapic_intr10), 81 IDTVEC(ioapic_intr11), 82 IDTVEC(ioapic_intr12), 83 IDTVEC(ioapic_intr13), 84 IDTVEC(ioapic_intr14), 85 IDTVEC(ioapic_intr15), 86 IDTVEC(ioapic_intr16), 87 IDTVEC(ioapic_intr17), 88 IDTVEC(ioapic_intr18), 89 IDTVEC(ioapic_intr19), 90 IDTVEC(ioapic_intr20), 91 IDTVEC(ioapic_intr21), 92 IDTVEC(ioapic_intr22), 93 IDTVEC(ioapic_intr23), 94 IDTVEC(ioapic_intr24), 95 IDTVEC(ioapic_intr25), 96 IDTVEC(ioapic_intr26), 97 IDTVEC(ioapic_intr27), 98 IDTVEC(ioapic_intr28), 99 IDTVEC(ioapic_intr29), 100 IDTVEC(ioapic_intr30), 101 IDTVEC(ioapic_intr31), 102 IDTVEC(ioapic_intr32), 103 IDTVEC(ioapic_intr33), 104 IDTVEC(ioapic_intr34), 105 IDTVEC(ioapic_intr35), 106 IDTVEC(ioapic_intr36), 107 IDTVEC(ioapic_intr37), 108 IDTVEC(ioapic_intr38), 109 IDTVEC(ioapic_intr39), 110 IDTVEC(ioapic_intr40), 111 IDTVEC(ioapic_intr41), 112 IDTVEC(ioapic_intr42), 113 IDTVEC(ioapic_intr43), 114 IDTVEC(ioapic_intr44), 115 IDTVEC(ioapic_intr45), 116 IDTVEC(ioapic_intr46), 117 IDTVEC(ioapic_intr47), 118 IDTVEC(ioapic_intr48), 119 IDTVEC(ioapic_intr49), 120 IDTVEC(ioapic_intr50), 121 IDTVEC(ioapic_intr51), 122 IDTVEC(ioapic_intr52), 123 IDTVEC(ioapic_intr53), 124 IDTVEC(ioapic_intr54), 125 IDTVEC(ioapic_intr55), 126 IDTVEC(ioapic_intr56), 127 IDTVEC(ioapic_intr57), 128 IDTVEC(ioapic_intr58), 129 IDTVEC(ioapic_intr59), 130 IDTVEC(ioapic_intr60), 131 IDTVEC(ioapic_intr61), 132 IDTVEC(ioapic_intr62), 133 IDTVEC(ioapic_intr63), 134 IDTVEC(ioapic_intr64), 135 IDTVEC(ioapic_intr65), 136 IDTVEC(ioapic_intr66), 137 IDTVEC(ioapic_intr67), 138 IDTVEC(ioapic_intr68), 139 IDTVEC(ioapic_intr69), 140 IDTVEC(ioapic_intr70), 141 IDTVEC(ioapic_intr71), 142 IDTVEC(ioapic_intr72), 143 IDTVEC(ioapic_intr73), 144 IDTVEC(ioapic_intr74), 145 IDTVEC(ioapic_intr75), 146 IDTVEC(ioapic_intr76), 147 IDTVEC(ioapic_intr77), 148 IDTVEC(ioapic_intr78), 149 IDTVEC(ioapic_intr79), 150 IDTVEC(ioapic_intr80), 151 IDTVEC(ioapic_intr81), 152 IDTVEC(ioapic_intr82), 153 IDTVEC(ioapic_intr83), 154 IDTVEC(ioapic_intr84), 155 IDTVEC(ioapic_intr85), 156 IDTVEC(ioapic_intr86), 157 IDTVEC(ioapic_intr87), 158 IDTVEC(ioapic_intr88), 159 IDTVEC(ioapic_intr89), 160 IDTVEC(ioapic_intr90), 161 IDTVEC(ioapic_intr91), 162 IDTVEC(ioapic_intr92), 163 IDTVEC(ioapic_intr93), 164 IDTVEC(ioapic_intr94), 165 IDTVEC(ioapic_intr95), 166 IDTVEC(ioapic_intr96), 167 IDTVEC(ioapic_intr97), 168 IDTVEC(ioapic_intr98), 169 IDTVEC(ioapic_intr99), 170 IDTVEC(ioapic_intr100), 171 IDTVEC(ioapic_intr101), 172 IDTVEC(ioapic_intr102), 173 IDTVEC(ioapic_intr103), 174 IDTVEC(ioapic_intr104), 175 IDTVEC(ioapic_intr105), 176 IDTVEC(ioapic_intr106), 177 IDTVEC(ioapic_intr107), 178 IDTVEC(ioapic_intr108), 179 IDTVEC(ioapic_intr109), 180 IDTVEC(ioapic_intr110), 181 IDTVEC(ioapic_intr111), 182 IDTVEC(ioapic_intr112), 183 IDTVEC(ioapic_intr113), 184 IDTVEC(ioapic_intr114), 185 IDTVEC(ioapic_intr115), 186 IDTVEC(ioapic_intr116), 187 IDTVEC(ioapic_intr117), 188 IDTVEC(ioapic_intr118), 189 IDTVEC(ioapic_intr119), 190 IDTVEC(ioapic_intr120), 191 IDTVEC(ioapic_intr121), 192 IDTVEC(ioapic_intr122), 193 IDTVEC(ioapic_intr123), 194 IDTVEC(ioapic_intr124), 195 IDTVEC(ioapic_intr125), 196 IDTVEC(ioapic_intr126), 197 IDTVEC(ioapic_intr127), 198 IDTVEC(ioapic_intr128), 199 IDTVEC(ioapic_intr129), 200 IDTVEC(ioapic_intr130), 201 IDTVEC(ioapic_intr131), 202 IDTVEC(ioapic_intr132), 203 IDTVEC(ioapic_intr133), 204 IDTVEC(ioapic_intr134), 205 IDTVEC(ioapic_intr135), 206 IDTVEC(ioapic_intr136), 207 IDTVEC(ioapic_intr137), 208 IDTVEC(ioapic_intr138), 209 IDTVEC(ioapic_intr139), 210 IDTVEC(ioapic_intr140), 211 IDTVEC(ioapic_intr141), 212 IDTVEC(ioapic_intr142), 213 IDTVEC(ioapic_intr143), 214 IDTVEC(ioapic_intr144), 215 IDTVEC(ioapic_intr145), 216 IDTVEC(ioapic_intr146), 217 IDTVEC(ioapic_intr147), 218 IDTVEC(ioapic_intr148), 219 IDTVEC(ioapic_intr149), 220 IDTVEC(ioapic_intr150), 221 IDTVEC(ioapic_intr151), 222 IDTVEC(ioapic_intr152), 223 IDTVEC(ioapic_intr153), 224 IDTVEC(ioapic_intr154), 225 IDTVEC(ioapic_intr155), 226 IDTVEC(ioapic_intr156), 227 IDTVEC(ioapic_intr157), 228 IDTVEC(ioapic_intr158), 229 IDTVEC(ioapic_intr159), 230 IDTVEC(ioapic_intr160), 231 IDTVEC(ioapic_intr161), 232 IDTVEC(ioapic_intr162), 233 IDTVEC(ioapic_intr163), 234 IDTVEC(ioapic_intr164), 235 IDTVEC(ioapic_intr165), 236 IDTVEC(ioapic_intr166), 237 IDTVEC(ioapic_intr167), 238 IDTVEC(ioapic_intr168), 239 IDTVEC(ioapic_intr169), 240 IDTVEC(ioapic_intr170), 241 IDTVEC(ioapic_intr171), 242 IDTVEC(ioapic_intr172), 243 IDTVEC(ioapic_intr173), 244 IDTVEC(ioapic_intr174), 245 IDTVEC(ioapic_intr175); 246 #if 0 247 IDTVEC(ioapic_intr176), 248 IDTVEC(ioapic_intr177), 249 IDTVEC(ioapic_intr178), 250 IDTVEC(ioapic_intr179), 251 IDTVEC(ioapic_intr180), 252 IDTVEC(ioapic_intr181), 253 IDTVEC(ioapic_intr182), 254 IDTVEC(ioapic_intr183), 255 IDTVEC(ioapic_intr184), 256 IDTVEC(ioapic_intr185), 257 IDTVEC(ioapic_intr186), 258 IDTVEC(ioapic_intr187), 259 IDTVEC(ioapic_intr188), 260 IDTVEC(ioapic_intr189), 261 IDTVEC(ioapic_intr190), 262 IDTVEC(ioapic_intr191); 263 #endif 264 265 static inthand_t *ioapic_intr[IOAPIC_HWI_VECTORS] = { 266 &IDTVEC(ioapic_intr0), 267 &IDTVEC(ioapic_intr1), 268 &IDTVEC(ioapic_intr2), 269 &IDTVEC(ioapic_intr3), 270 &IDTVEC(ioapic_intr4), 271 &IDTVEC(ioapic_intr5), 272 &IDTVEC(ioapic_intr6), 273 &IDTVEC(ioapic_intr7), 274 &IDTVEC(ioapic_intr8), 275 &IDTVEC(ioapic_intr9), 276 &IDTVEC(ioapic_intr10), 277 &IDTVEC(ioapic_intr11), 278 &IDTVEC(ioapic_intr12), 279 &IDTVEC(ioapic_intr13), 280 &IDTVEC(ioapic_intr14), 281 &IDTVEC(ioapic_intr15), 282 &IDTVEC(ioapic_intr16), 283 &IDTVEC(ioapic_intr17), 284 &IDTVEC(ioapic_intr18), 285 &IDTVEC(ioapic_intr19), 286 &IDTVEC(ioapic_intr20), 287 &IDTVEC(ioapic_intr21), 288 &IDTVEC(ioapic_intr22), 289 &IDTVEC(ioapic_intr23), 290 &IDTVEC(ioapic_intr24), 291 &IDTVEC(ioapic_intr25), 292 &IDTVEC(ioapic_intr26), 293 &IDTVEC(ioapic_intr27), 294 &IDTVEC(ioapic_intr28), 295 &IDTVEC(ioapic_intr29), 296 &IDTVEC(ioapic_intr30), 297 &IDTVEC(ioapic_intr31), 298 &IDTVEC(ioapic_intr32), 299 &IDTVEC(ioapic_intr33), 300 &IDTVEC(ioapic_intr34), 301 &IDTVEC(ioapic_intr35), 302 &IDTVEC(ioapic_intr36), 303 &IDTVEC(ioapic_intr37), 304 &IDTVEC(ioapic_intr38), 305 &IDTVEC(ioapic_intr39), 306 &IDTVEC(ioapic_intr40), 307 &IDTVEC(ioapic_intr41), 308 &IDTVEC(ioapic_intr42), 309 &IDTVEC(ioapic_intr43), 310 &IDTVEC(ioapic_intr44), 311 &IDTVEC(ioapic_intr45), 312 &IDTVEC(ioapic_intr46), 313 &IDTVEC(ioapic_intr47), 314 &IDTVEC(ioapic_intr48), 315 &IDTVEC(ioapic_intr49), 316 &IDTVEC(ioapic_intr50), 317 &IDTVEC(ioapic_intr51), 318 &IDTVEC(ioapic_intr52), 319 &IDTVEC(ioapic_intr53), 320 &IDTVEC(ioapic_intr54), 321 &IDTVEC(ioapic_intr55), 322 &IDTVEC(ioapic_intr56), 323 &IDTVEC(ioapic_intr57), 324 &IDTVEC(ioapic_intr58), 325 &IDTVEC(ioapic_intr59), 326 &IDTVEC(ioapic_intr60), 327 &IDTVEC(ioapic_intr61), 328 &IDTVEC(ioapic_intr62), 329 &IDTVEC(ioapic_intr63), 330 &IDTVEC(ioapic_intr64), 331 &IDTVEC(ioapic_intr65), 332 &IDTVEC(ioapic_intr66), 333 &IDTVEC(ioapic_intr67), 334 &IDTVEC(ioapic_intr68), 335 &IDTVEC(ioapic_intr69), 336 &IDTVEC(ioapic_intr70), 337 &IDTVEC(ioapic_intr71), 338 &IDTVEC(ioapic_intr72), 339 &IDTVEC(ioapic_intr73), 340 &IDTVEC(ioapic_intr74), 341 &IDTVEC(ioapic_intr75), 342 &IDTVEC(ioapic_intr76), 343 &IDTVEC(ioapic_intr77), 344 &IDTVEC(ioapic_intr78), 345 &IDTVEC(ioapic_intr79), 346 &IDTVEC(ioapic_intr80), 347 &IDTVEC(ioapic_intr81), 348 &IDTVEC(ioapic_intr82), 349 &IDTVEC(ioapic_intr83), 350 &IDTVEC(ioapic_intr84), 351 &IDTVEC(ioapic_intr85), 352 &IDTVEC(ioapic_intr86), 353 &IDTVEC(ioapic_intr87), 354 &IDTVEC(ioapic_intr88), 355 &IDTVEC(ioapic_intr89), 356 &IDTVEC(ioapic_intr90), 357 &IDTVEC(ioapic_intr91), 358 &IDTVEC(ioapic_intr92), 359 &IDTVEC(ioapic_intr93), 360 &IDTVEC(ioapic_intr94), 361 &IDTVEC(ioapic_intr95), 362 &IDTVEC(ioapic_intr96), 363 &IDTVEC(ioapic_intr97), 364 &IDTVEC(ioapic_intr98), 365 &IDTVEC(ioapic_intr99), 366 &IDTVEC(ioapic_intr100), 367 &IDTVEC(ioapic_intr101), 368 &IDTVEC(ioapic_intr102), 369 &IDTVEC(ioapic_intr103), 370 &IDTVEC(ioapic_intr104), 371 &IDTVEC(ioapic_intr105), 372 &IDTVEC(ioapic_intr106), 373 &IDTVEC(ioapic_intr107), 374 &IDTVEC(ioapic_intr108), 375 &IDTVEC(ioapic_intr109), 376 &IDTVEC(ioapic_intr110), 377 &IDTVEC(ioapic_intr111), 378 &IDTVEC(ioapic_intr112), 379 &IDTVEC(ioapic_intr113), 380 &IDTVEC(ioapic_intr114), 381 &IDTVEC(ioapic_intr115), 382 &IDTVEC(ioapic_intr116), 383 &IDTVEC(ioapic_intr117), 384 &IDTVEC(ioapic_intr118), 385 &IDTVEC(ioapic_intr119), 386 &IDTVEC(ioapic_intr120), 387 &IDTVEC(ioapic_intr121), 388 &IDTVEC(ioapic_intr122), 389 &IDTVEC(ioapic_intr123), 390 &IDTVEC(ioapic_intr124), 391 &IDTVEC(ioapic_intr125), 392 &IDTVEC(ioapic_intr126), 393 &IDTVEC(ioapic_intr127), 394 &IDTVEC(ioapic_intr128), 395 &IDTVEC(ioapic_intr129), 396 &IDTVEC(ioapic_intr130), 397 &IDTVEC(ioapic_intr131), 398 &IDTVEC(ioapic_intr132), 399 &IDTVEC(ioapic_intr133), 400 &IDTVEC(ioapic_intr134), 401 &IDTVEC(ioapic_intr135), 402 &IDTVEC(ioapic_intr136), 403 &IDTVEC(ioapic_intr137), 404 &IDTVEC(ioapic_intr138), 405 &IDTVEC(ioapic_intr139), 406 &IDTVEC(ioapic_intr140), 407 &IDTVEC(ioapic_intr141), 408 &IDTVEC(ioapic_intr142), 409 &IDTVEC(ioapic_intr143), 410 &IDTVEC(ioapic_intr144), 411 &IDTVEC(ioapic_intr145), 412 &IDTVEC(ioapic_intr146), 413 &IDTVEC(ioapic_intr147), 414 &IDTVEC(ioapic_intr148), 415 &IDTVEC(ioapic_intr149), 416 &IDTVEC(ioapic_intr150), 417 &IDTVEC(ioapic_intr151), 418 &IDTVEC(ioapic_intr152), 419 &IDTVEC(ioapic_intr153), 420 &IDTVEC(ioapic_intr154), 421 &IDTVEC(ioapic_intr155), 422 &IDTVEC(ioapic_intr156), 423 &IDTVEC(ioapic_intr157), 424 &IDTVEC(ioapic_intr158), 425 &IDTVEC(ioapic_intr159), 426 &IDTVEC(ioapic_intr160), 427 &IDTVEC(ioapic_intr161), 428 &IDTVEC(ioapic_intr162), 429 &IDTVEC(ioapic_intr163), 430 &IDTVEC(ioapic_intr164), 431 &IDTVEC(ioapic_intr165), 432 &IDTVEC(ioapic_intr166), 433 &IDTVEC(ioapic_intr167), 434 &IDTVEC(ioapic_intr168), 435 &IDTVEC(ioapic_intr169), 436 &IDTVEC(ioapic_intr170), 437 &IDTVEC(ioapic_intr171), 438 &IDTVEC(ioapic_intr172), 439 &IDTVEC(ioapic_intr173), 440 &IDTVEC(ioapic_intr174), 441 &IDTVEC(ioapic_intr175), 442 #if 0 443 &IDTVEC(ioapic_intr176), 444 &IDTVEC(ioapic_intr177), 445 &IDTVEC(ioapic_intr178), 446 &IDTVEC(ioapic_intr179), 447 &IDTVEC(ioapic_intr180), 448 &IDTVEC(ioapic_intr181), 449 &IDTVEC(ioapic_intr182), 450 &IDTVEC(ioapic_intr183), 451 &IDTVEC(ioapic_intr184), 452 &IDTVEC(ioapic_intr185), 453 &IDTVEC(ioapic_intr186), 454 &IDTVEC(ioapic_intr187), 455 &IDTVEC(ioapic_intr188), 456 &IDTVEC(ioapic_intr189), 457 &IDTVEC(ioapic_intr190), 458 &IDTVEC(ioapic_intr191) 459 #endif 460 }; 461 462 #define IOAPIC_HWI_SYSCALL (IDT_OFFSET_SYSCALL - IDT_OFFSET) 463 464 /* 465 * NOTE: Initialized before VM so cannot use kmalloc() for this array. 466 */ 467 static struct ioapic_irqmap { 468 int im_type; /* IOAPIC_IMT_ */ 469 enum intr_trigger im_trig; 470 enum intr_polarity im_pola; 471 int im_gsi; 472 int im_msi_base; 473 uint32_t im_flags; /* IOAPIC_IMF_ */ 474 } ioapic_irqmaps[MAXCPU][IOAPIC_HWI_VECTORS]; 475 476 static struct lwkt_token ioapic_irqmap_tok = 477 LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token); 478 479 #define IOAPIC_IMT_UNUSED 0 480 #define IOAPIC_IMT_RESERVED 1 481 #define IOAPIC_IMT_LEGACY 2 482 #define IOAPIC_IMT_SYSCALL 3 483 #define IOAPIC_IMT_MSI 4 484 #define IOAPIC_IMT_MSIX 5 485 486 #define IOAPIC_IMT_ISHWI(map) ((map)->im_type != IOAPIC_IMT_RESERVED && \ 487 (map)->im_type != IOAPIC_IMT_SYSCALL) 488 489 #define IOAPIC_IMF_CONF 0x1 490 491 extern void IOAPIC_INTREN(int); 492 extern void IOAPIC_INTRDIS(int); 493 494 extern int imcr_present; 495 496 static void ioapic_abi_intr_enable(int); 497 static void ioapic_abi_intr_disable(int); 498 static void ioapic_abi_intr_setup(int, int); 499 static void ioapic_abi_intr_teardown(int); 500 501 static void ioapic_abi_legacy_intr_config(int, 502 enum intr_trigger, enum intr_polarity); 503 static int ioapic_abi_legacy_intr_cpuid(int); 504 static int ioapic_abi_legacy_intr_find(int, 505 enum intr_trigger, enum intr_polarity); 506 static int ioapic_abi_legacy_intr_find_bygsi(int, 507 enum intr_trigger, enum intr_polarity); 508 509 static int ioapic_abi_msi_alloc(int [], int, int); 510 static void ioapic_abi_msi_release(const int [], int, int); 511 static void ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int); 512 static int ioapic_abi_msix_alloc(int *, int); 513 static void ioapic_abi_msix_release(int, int); 514 515 static int ioapic_abi_msi_alloc_intern(int, const char *, 516 int [], int, int); 517 static void ioapic_abi_msi_release_intern(int, const char *, 518 const int [], int, int); 519 520 static void ioapic_abi_finalize(void); 521 static void ioapic_abi_cleanup(void); 522 static void ioapic_abi_setdefault(void); 523 static void ioapic_abi_stabilize(void); 524 static void ioapic_abi_initmap(void); 525 static void ioapic_abi_rman_setup(struct rman *); 526 527 static int ioapic_abi_gsi_cpuid(int, int); 528 static int ioapic_find_unused_irqmap(int); 529 530 struct machintr_abi MachIntrABI_IOAPIC = { 531 MACHINTR_IOAPIC, 532 .intr_disable = ioapic_abi_intr_disable, 533 .intr_enable = ioapic_abi_intr_enable, 534 .intr_setup = ioapic_abi_intr_setup, 535 .intr_teardown = ioapic_abi_intr_teardown, 536 537 .legacy_intr_config = ioapic_abi_legacy_intr_config, 538 .legacy_intr_cpuid = ioapic_abi_legacy_intr_cpuid, 539 .legacy_intr_find = ioapic_abi_legacy_intr_find, 540 .legacy_intr_find_bygsi = ioapic_abi_legacy_intr_find_bygsi, 541 542 .msi_alloc = ioapic_abi_msi_alloc, 543 .msi_release = ioapic_abi_msi_release, 544 .msi_map = ioapic_abi_msi_map, 545 .msix_alloc = ioapic_abi_msix_alloc, 546 .msix_release = ioapic_abi_msix_release, 547 548 .finalize = ioapic_abi_finalize, 549 .cleanup = ioapic_abi_cleanup, 550 .setdefault = ioapic_abi_setdefault, 551 .stabilize = ioapic_abi_stabilize, 552 .initmap = ioapic_abi_initmap, 553 .rman_setup = ioapic_abi_rman_setup 554 }; 555 556 static int ioapic_abi_extint_irq = -1; 557 static int ioapic_abi_legacy_irq_max; 558 static int ioapic_abi_gsi_balance = 1; 559 static int ioapic_abi_msi_start; /* NOTE: for testing only */ 560 561 struct ioapic_irqinfo ioapic_irqs[IOAPIC_HWI_VECTORS]; 562 563 static void 564 ioapic_abi_intr_enable(int irq) 565 { 566 const struct ioapic_irqmap *map; 567 568 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 569 ("ioapic enable, invalid irq %d", irq)); 570 571 map = &ioapic_irqmaps[mycpuid][irq]; 572 KASSERT(IOAPIC_IMT_ISHWI(map), 573 ("ioapic enable, not hwi irq %d, type %d, cpu%d", 574 irq, map->im_type, mycpuid)); 575 if (map->im_type != IOAPIC_IMT_LEGACY) 576 return; 577 578 IOAPIC_INTREN(irq); 579 } 580 581 static void 582 ioapic_abi_intr_disable(int irq) 583 { 584 const struct ioapic_irqmap *map; 585 586 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 587 ("ioapic disable, invalid irq %d", irq)); 588 589 map = &ioapic_irqmaps[mycpuid][irq]; 590 KASSERT(IOAPIC_IMT_ISHWI(map), 591 ("ioapic disable, not hwi irq %d, type %d, cpu%d", 592 irq, map->im_type, mycpuid)); 593 if (map->im_type != IOAPIC_IMT_LEGACY) 594 return; 595 596 IOAPIC_INTRDIS(irq); 597 } 598 599 static void 600 ioapic_abi_finalize(void) 601 { 602 KKASSERT(MachIntrABI.type == MACHINTR_IOAPIC); 603 KKASSERT(ioapic_enable); 604 605 /* 606 * If an IMCR is present, program bit 0 to disconnect the 8259 607 * from the BSP. 608 */ 609 if (imcr_present) { 610 outb(0x22, 0x70); /* select IMCR */ 611 outb(0x23, 0x01); /* disconnect 8259 */ 612 } 613 } 614 615 /* 616 * This routine is called after physical interrupts are enabled but before 617 * the critical section is released. We need to clean out any interrupts 618 * that had already been posted to the cpu. 619 */ 620 static void 621 ioapic_abi_cleanup(void) 622 { 623 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending)); 624 } 625 626 /* Must never be called */ 627 static void 628 ioapic_abi_stabilize(void) 629 { 630 panic("ioapic_stabilize is called"); 631 } 632 633 static void 634 ioapic_abi_intr_setup(int intr, int flags) 635 { 636 const struct ioapic_irqmap *map; 637 int vector, select; 638 uint32_t value; 639 register_t ef; 640 641 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 642 ("ioapic setup, invalid irq %d", intr)); 643 644 map = &ioapic_irqmaps[mycpuid][intr]; 645 KASSERT(IOAPIC_IMT_ISHWI(map), 646 ("ioapic setup, not hwi irq %d, type %d, cpu%d", 647 intr, map->im_type, mycpuid)); 648 if (map->im_type != IOAPIC_IMT_LEGACY) 649 return; 650 651 KASSERT(ioapic_irqs[intr].io_addr != NULL, 652 ("ioapic setup, no GSI information, irq %d", intr)); 653 654 ef = read_rflags(); 655 cpu_disable_intr(); 656 657 vector = IDT_OFFSET + intr; 658 659 /* 660 * Now reprogram the vector in the IO APIC. In order to avoid 661 * losing an EOI for a level interrupt, which is vector based, 662 * make sure that the IO APIC is programmed for edge-triggering 663 * first, then reprogrammed with the new vector. This should 664 * clear the IRR bit. 665 */ 666 imen_lock(); 667 668 select = ioapic_irqs[intr].io_idx; 669 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 670 value |= IOART_INTMSET; 671 672 ioapic_write(ioapic_irqs[intr].io_addr, select, 673 (value & ~APIC_TRIGMOD_MASK)); 674 ioapic_write(ioapic_irqs[intr].io_addr, select, 675 (value & ~IOART_INTVEC) | vector); 676 677 imen_unlock(); 678 679 IOAPIC_INTREN(intr); 680 681 write_rflags(ef); 682 } 683 684 static void 685 ioapic_abi_intr_teardown(int intr) 686 { 687 const struct ioapic_irqmap *map; 688 int vector, select; 689 uint32_t value; 690 register_t ef; 691 692 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 693 ("ioapic teardown, invalid irq %d", intr)); 694 695 map = &ioapic_irqmaps[mycpuid][intr]; 696 KASSERT(IOAPIC_IMT_ISHWI(map), 697 ("ioapic teardown, not hwi irq %d, type %d, cpu%d", 698 intr, map->im_type, mycpuid)); 699 if (map->im_type != IOAPIC_IMT_LEGACY) 700 return; 701 702 KASSERT(ioapic_irqs[intr].io_addr != NULL, 703 ("ioapic teardown, no GSI information, irq %d", intr)); 704 705 ef = read_rflags(); 706 cpu_disable_intr(); 707 708 /* 709 * Teardown an interrupt vector. The vector should already be 710 * installed in the cpu's IDT, but make sure. 711 */ 712 IOAPIC_INTRDIS(intr); 713 714 vector = IDT_OFFSET + intr; 715 716 /* 717 * In order to avoid losing an EOI for a level interrupt, which 718 * is vector based, make sure that the IO APIC is programmed for 719 * edge-triggering first, then reprogrammed with the new vector. 720 * This should clear the IRR bit. 721 */ 722 imen_lock(); 723 724 select = ioapic_irqs[intr].io_idx; 725 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 726 727 ioapic_write(ioapic_irqs[intr].io_addr, select, 728 (value & ~APIC_TRIGMOD_MASK)); 729 ioapic_write(ioapic_irqs[intr].io_addr, select, 730 (value & ~IOART_INTVEC) | vector); 731 732 imen_unlock(); 733 734 write_rflags(ef); 735 } 736 737 static void 738 ioapic_abi_setdefault(void) 739 { 740 int intr; 741 742 for (intr = 0; intr < IOAPIC_HWI_VECTORS; ++intr) { 743 if (intr == IOAPIC_HWI_SYSCALL) 744 continue; 745 setidt_global(IDT_OFFSET + intr, ioapic_intr[intr], 746 SDT_SYSIGT, SEL_KPL, 0); 747 } 748 } 749 750 static void 751 ioapic_abi_initmap(void) 752 { 753 int cpu; 754 755 kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance); 756 757 kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start); 758 ioapic_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */ 759 760 /* 761 * NOTE: ncpus is not ready yet 762 */ 763 for (cpu = 0; cpu < MAXCPU; ++cpu) { 764 int i; 765 766 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 767 ioapic_irqmaps[cpu][i].im_gsi = -1; 768 ioapic_irqmaps[cpu][i].im_msi_base = -1; 769 } 770 ioapic_irqmaps[cpu][IOAPIC_HWI_SYSCALL].im_type = 771 IOAPIC_IMT_SYSCALL; 772 } 773 } 774 775 static int 776 ioapic_find_unused_irqmap(int gsi) 777 { 778 int cpuid, i; 779 780 cpuid = ioapic_abi_gsi_cpuid(-1, gsi); 781 782 for (i = ISA_IRQ_CNT; i < IOAPIC_HWI_VECTORS; ++i) { 783 if (i == acpi_sci_irqno()) 784 continue; 785 if (ioapic_irqmaps[cpuid][i].im_type == IOAPIC_IMT_UNUSED) 786 return i; 787 } 788 return -1; 789 } 790 791 void 792 ioapic_set_legacy_irqmap(int irq, int gsi, enum intr_trigger trig, 793 enum intr_polarity pola) 794 { 795 struct ioapic_irqinfo *info; 796 struct ioapic_irqmap *map; 797 void *ioaddr; 798 int pin, cpuid; 799 800 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 801 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 802 803 KKASSERT(irq >= 0); 804 if (irq >= IOAPIC_HWI_VECTORS) { 805 /* 806 * Some BIOSes seem to assume that all 256 IDT vectors 807 * could be used, while we limit the available IDT 808 * vectors to 192; find an unused IRQ for this GSI. 809 */ 810 irq = ioapic_find_unused_irqmap(gsi); 811 if (irq < 0) { 812 kprintf("failed to find unused irq for gsi %d, " 813 "overflow\n", gsi); 814 return; 815 } 816 } 817 KKASSERT(irq < IOAPIC_HWI_VECTORS); 818 819 cpuid = ioapic_abi_gsi_cpuid(irq, gsi); 820 map = &ioapic_irqmaps[cpuid][irq]; 821 822 if (map->im_type != IOAPIC_IMT_UNUSED) { 823 /* 824 * There are so many IOAPICs, that 1:1 mapping 825 * of GSI and IRQ hits SYSCALL entry. 826 */ 827 irq = ioapic_find_unused_irqmap(gsi); 828 if (irq < 0) { 829 kprintf("failed to find unused irq for gsi %d, " 830 "conflict\n", gsi); 831 return; 832 } 833 KKASSERT(irq < IOAPIC_HWI_VECTORS); 834 835 cpuid = ioapic_abi_gsi_cpuid(irq, gsi); 836 map = &ioapic_irqmaps[cpuid][irq]; 837 } 838 839 if (irq > ioapic_abi_legacy_irq_max) 840 ioapic_abi_legacy_irq_max = irq; 841 842 KKASSERT(map->im_type == IOAPIC_IMT_UNUSED); 843 map->im_type = IOAPIC_IMT_LEGACY; 844 845 map->im_gsi = gsi; 846 map->im_trig = trig; 847 map->im_pola = pola; 848 849 if (bootverbose) { 850 kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n", 851 irq, map->im_gsi, 852 intr_str_trigger(map->im_trig), 853 intr_str_polarity(map->im_pola)); 854 } 855 856 pin = ioapic_gsi_pin(map->im_gsi); 857 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 858 859 info = &ioapic_irqs[irq]; 860 861 imen_lock(); 862 863 info->io_addr = ioaddr; 864 info->io_idx = IOAPIC_REDTBL + (2 * pin); 865 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 866 if (map->im_trig == INTR_TRIGGER_LEVEL) 867 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 868 869 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 870 map->im_trig, map->im_pola, cpuid); 871 872 imen_unlock(); 873 } 874 875 void 876 ioapic_fixup_legacy_irqmaps(void) 877 { 878 int cpu; 879 880 for (cpu = 0; cpu < ncpus; ++cpu) { 881 int i; 882 883 for (i = 0; i < ISA_IRQ_CNT; ++i) { 884 struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][i]; 885 886 if (map->im_type == IOAPIC_IMT_UNUSED) { 887 map->im_type = IOAPIC_IMT_RESERVED; 888 if (bootverbose) { 889 kprintf("IOAPIC: " 890 "cpu%d irq %d reserved\n", cpu, i); 891 } 892 } 893 } 894 } 895 896 ioapic_abi_legacy_irq_max += 1; 897 if (bootverbose) { 898 kprintf("IOAPIC: legacy irq max %d\n", 899 ioapic_abi_legacy_irq_max); 900 } 901 } 902 903 static int 904 ioapic_abi_legacy_intr_find_bygsi(int gsi, enum intr_trigger trig, 905 enum intr_polarity pola) 906 { 907 int cpu; 908 909 #ifdef INVARIANTS 910 if (trig == INTR_TRIGGER_CONFORM) { 911 KKASSERT(pola == INTR_POLARITY_CONFORM); 912 } else { 913 KKASSERT(trig == INTR_TRIGGER_EDGE || 914 trig == INTR_TRIGGER_LEVEL); 915 KKASSERT(pola == INTR_POLARITY_HIGH || 916 pola == INTR_POLARITY_LOW); 917 } 918 #endif 919 920 for (cpu = 0; cpu < ncpus; ++cpu) { 921 int irq; 922 923 for (irq = 0; irq < ioapic_abi_legacy_irq_max; ++irq) { 924 const struct ioapic_irqmap *map = 925 &ioapic_irqmaps[cpu][irq]; 926 927 if (map->im_gsi == gsi) { 928 KKASSERT(map->im_type == IOAPIC_IMT_LEGACY); 929 930 if ((map->im_flags & IOAPIC_IMF_CONF) && 931 trig != INTR_TRIGGER_CONFORM && 932 pola != INTR_POLARITY_CONFORM) { 933 if (map->im_trig != trig || 934 map->im_pola != pola) 935 return -1; 936 } 937 return irq; 938 } 939 } 940 } 941 return -1; 942 } 943 944 static int 945 ioapic_abi_legacy_intr_find(int irq, enum intr_trigger trig, 946 enum intr_polarity pola) 947 { 948 int cpu; 949 950 #ifdef INVARIANTS 951 if (trig == INTR_TRIGGER_CONFORM) { 952 KKASSERT(pola == INTR_POLARITY_CONFORM); 953 } else { 954 KKASSERT(trig == INTR_TRIGGER_EDGE || 955 trig == INTR_TRIGGER_LEVEL); 956 KKASSERT(pola == INTR_POLARITY_HIGH || 957 pola == INTR_POLARITY_LOW); 958 } 959 #endif 960 961 if (irq < 0 || irq >= ioapic_abi_legacy_irq_max) 962 return -1; 963 964 for (cpu = 0; cpu < ncpus; ++cpu) { 965 const struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][irq]; 966 967 if (map->im_type == IOAPIC_IMT_LEGACY) { 968 if ((map->im_flags & IOAPIC_IMF_CONF) && 969 trig != INTR_TRIGGER_CONFORM && 970 pola != INTR_POLARITY_CONFORM) { 971 if (map->im_trig != trig || 972 map->im_pola != pola) 973 return -1; 974 } 975 return irq; 976 } 977 } 978 return -1; 979 } 980 981 static void 982 ioapic_abi_legacy_intr_config(int irq, enum intr_trigger trig, 983 enum intr_polarity pola) 984 { 985 struct ioapic_irqinfo *info; 986 struct ioapic_irqmap *map = NULL; 987 void *ioaddr; 988 int pin, cpuid; 989 990 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 991 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 992 993 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 994 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 995 map = &ioapic_irqmaps[cpuid][irq]; 996 if (map->im_type == IOAPIC_IMT_LEGACY) 997 break; 998 } 999 KKASSERT(cpuid < ncpus); 1000 1001 #ifdef notyet 1002 if (map->im_flags & IOAPIC_IMF_CONF) { 1003 if (trig != map->im_trig) { 1004 panic("ioapic_intr_config: trig %s -> %s", 1005 intr_str_trigger(map->im_trig), 1006 intr_str_trigger(trig)); 1007 } 1008 if (pola != map->im_pola) { 1009 panic("ioapic_intr_config: pola %s -> %s", 1010 intr_str_polarity(map->im_pola), 1011 intr_str_polarity(pola)); 1012 } 1013 return; 1014 } 1015 #endif 1016 map->im_flags |= IOAPIC_IMF_CONF; 1017 1018 if (trig == map->im_trig && pola == map->im_pola) 1019 return; 1020 1021 if (bootverbose) { 1022 kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n", 1023 irq, map->im_gsi, 1024 intr_str_trigger(map->im_trig), 1025 intr_str_polarity(map->im_pola), 1026 intr_str_trigger(trig), 1027 intr_str_polarity(pola)); 1028 } 1029 map->im_trig = trig; 1030 map->im_pola = pola; 1031 1032 pin = ioapic_gsi_pin(map->im_gsi); 1033 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1034 1035 info = &ioapic_irqs[irq]; 1036 1037 imen_lock(); 1038 1039 info->io_flags &= ~IOAPIC_IRQI_FLAG_LEVEL; 1040 if (map->im_trig == INTR_TRIGGER_LEVEL) 1041 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 1042 1043 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 1044 map->im_trig, map->im_pola, cpuid); 1045 1046 imen_unlock(); 1047 } 1048 1049 int 1050 ioapic_conf_legacy_extint(int irq) 1051 { 1052 struct ioapic_irqinfo *info; 1053 struct ioapic_irqmap *map; 1054 void *ioaddr; 1055 int pin, error, vec; 1056 1057 /* XXX only irq0 is allowed */ 1058 KKASSERT(irq == 0); 1059 1060 vec = IDT_OFFSET + irq; 1061 1062 if (ioapic_abi_extint_irq == irq) 1063 return 0; 1064 else if (ioapic_abi_extint_irq >= 0) 1065 return EEXIST; 1066 1067 error = icu_ioapic_extint(irq, vec); 1068 if (error) 1069 return error; 1070 1071 /* ExtINT is always targeted to cpu0 */ 1072 map = &ioapic_irqmaps[0][irq]; 1073 1074 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED || 1075 map->im_type == IOAPIC_IMT_LEGACY); 1076 if (map->im_type == IOAPIC_IMT_LEGACY) { 1077 if (map->im_flags & IOAPIC_IMF_CONF) 1078 return EEXIST; 1079 } 1080 ioapic_abi_extint_irq = irq; 1081 1082 map->im_type = IOAPIC_IMT_LEGACY; 1083 map->im_trig = INTR_TRIGGER_EDGE; 1084 map->im_pola = INTR_POLARITY_HIGH; 1085 map->im_flags = IOAPIC_IMF_CONF; 1086 1087 map->im_gsi = ioapic_extpin_gsi(); 1088 KKASSERT(map->im_gsi >= 0); 1089 1090 if (bootverbose) { 1091 kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n", 1092 irq, map->im_gsi, 1093 intr_str_trigger(map->im_trig), 1094 intr_str_polarity(map->im_pola)); 1095 } 1096 1097 pin = ioapic_gsi_pin(map->im_gsi); 1098 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1099 1100 info = &ioapic_irqs[irq]; 1101 1102 imen_lock(); 1103 1104 info->io_addr = ioaddr; 1105 info->io_idx = IOAPIC_REDTBL + (2 * pin); 1106 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 1107 1108 ioapic_extpin_setup(ioaddr, pin, vec); 1109 1110 imen_unlock(); 1111 1112 return 0; 1113 } 1114 1115 static int 1116 ioapic_abi_legacy_intr_cpuid(int irq) 1117 { 1118 const struct ioapic_irqmap *map = NULL; 1119 int cpuid; 1120 1121 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 1122 1123 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1124 map = &ioapic_irqmaps[cpuid][irq]; 1125 if (map->im_type == IOAPIC_IMT_LEGACY) 1126 return cpuid; 1127 } 1128 1129 /* XXX some drivers tries to peek at reserved IRQs */ 1130 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1131 map = &ioapic_irqmaps[cpuid][irq]; 1132 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED); 1133 } 1134 return 0; 1135 } 1136 1137 static int 1138 ioapic_abi_gsi_cpuid(int irq, int gsi) 1139 { 1140 char envpath[32]; 1141 int cpuid = -1; 1142 1143 KKASSERT(gsi >= 0); 1144 1145 if (irq == 0 || gsi == 0) { 1146 KKASSERT(irq >= 0); 1147 if (bootverbose) { 1148 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n", 1149 irq, gsi); 1150 } 1151 return 0; 1152 } 1153 1154 if (irq >= 0 && irq == acpi_sci_irqno()) { 1155 if (bootverbose) { 1156 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n", 1157 irq, gsi); 1158 } 1159 return 0; 1160 } 1161 1162 ksnprintf(envpath, sizeof(envpath), "hw.ioapic.gsi.%d.cpu", gsi); 1163 kgetenv_int(envpath, &cpuid); 1164 1165 if (cpuid < 0) { 1166 if (!ioapic_abi_gsi_balance) { 1167 if (irq >= 0 && bootverbose) { 1168 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 " 1169 "(fixed)\n", irq, gsi); 1170 } 1171 return 0; 1172 } 1173 1174 cpuid = gsi % ncpus; 1175 if (irq >= 0 && bootverbose) { 1176 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n", 1177 irq, gsi, cpuid); 1178 } 1179 } else if (cpuid >= ncpus) { 1180 cpuid = ncpus - 1; 1181 if (irq >= 0 && bootverbose) { 1182 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n", 1183 irq, gsi, cpuid); 1184 } 1185 } else { 1186 if (irq >= 0 && bootverbose) { 1187 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n", 1188 irq, gsi, cpuid); 1189 } 1190 } 1191 return cpuid; 1192 } 1193 1194 static void 1195 ioapic_abi_rman_setup(struct rman *rm) 1196 { 1197 int start, end, i; 1198 1199 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU, 1200 ("invalid rman cpuid %d", rm->rm_cpuid)); 1201 1202 start = end = -1; 1203 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 1204 const struct ioapic_irqmap *map = 1205 &ioapic_irqmaps[rm->rm_cpuid][i]; 1206 1207 if (start < 0) { 1208 if (IOAPIC_IMT_ISHWI(map)) 1209 start = end = i; 1210 } else { 1211 if (IOAPIC_IMT_ISHWI(map)) { 1212 end = i; 1213 } else { 1214 KKASSERT(end >= 0); 1215 if (bootverbose) { 1216 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1217 rm->rm_cpuid, start, end); 1218 } 1219 if (rman_manage_region(rm, start, end)) { 1220 panic("rman_manage_region" 1221 "(cpu%d %d - %d)", rm->rm_cpuid, 1222 start, end); 1223 } 1224 start = end = -1; 1225 } 1226 } 1227 } 1228 if (start >= 0) { 1229 KKASSERT(end >= 0); 1230 if (bootverbose) { 1231 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1232 rm->rm_cpuid, start, end); 1233 } 1234 if (rman_manage_region(rm, start, end)) { 1235 panic("rman_manage_region(cpu%d %d - %d)", 1236 rm->rm_cpuid, start, end); 1237 } 1238 } 1239 } 1240 1241 static int 1242 ioapic_abi_msi_alloc_intern(int type, const char *desc, 1243 int intrs[], int count, int cpuid) 1244 { 1245 int i, error; 1246 1247 KASSERT(cpuid >= 0 && cpuid < ncpus, 1248 ("invalid cpuid %d", cpuid)); 1249 1250 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 1251 KASSERT((count & (count - 1)) == 0, 1252 ("count %d is not power of 2", count)); 1253 1254 lwkt_gettoken(&ioapic_irqmap_tok); 1255 1256 /* 1257 * NOTE: 1258 * Since IDT_OFFSET is 32, which is the maximum valid 'count', 1259 * we do not need to find out the first properly aligned 1260 * interrupt vector. 1261 */ 1262 1263 error = EMSGSIZE; 1264 for (i = ioapic_abi_msi_start; i < IOAPIC_HWI_VECTORS; i += count) { 1265 int j; 1266 1267 if (ioapic_irqmaps[cpuid][i].im_type != IOAPIC_IMT_UNUSED) 1268 continue; 1269 1270 for (j = 1; j < count; ++j) { 1271 if (ioapic_irqmaps[cpuid][i + j].im_type != 1272 IOAPIC_IMT_UNUSED) 1273 break; 1274 } 1275 if (j != count) 1276 continue; 1277 1278 for (j = 0; j < count; ++j) { 1279 struct ioapic_irqmap *map; 1280 int intr = i + j; 1281 1282 map = &ioapic_irqmaps[cpuid][intr]; 1283 KASSERT(map->im_msi_base < 0, 1284 ("intr %d, stale %s-base %d", 1285 intr, desc, map->im_msi_base)); 1286 1287 map->im_type = type; 1288 map->im_msi_base = i; 1289 1290 intrs[j] = intr; 1291 msi_setup(intr, cpuid); 1292 1293 if (bootverbose) { 1294 kprintf("alloc %s intr %d on cpu%d\n", 1295 desc, intr, cpuid); 1296 } 1297 } 1298 error = 0; 1299 break; 1300 } 1301 1302 lwkt_reltoken(&ioapic_irqmap_tok); 1303 1304 return error; 1305 } 1306 1307 static void 1308 ioapic_abi_msi_release_intern(int type, const char *desc, 1309 const int intrs[], int count, int cpuid) 1310 { 1311 int i, msi_base = -1, intr_next = -1, mask; 1312 1313 KASSERT(cpuid >= 0 && cpuid < ncpus, 1314 ("invalid cpuid %d", cpuid)); 1315 1316 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 1317 1318 mask = count - 1; 1319 KASSERT((count & mask) == 0, ("count %d is not power of 2", count)); 1320 1321 lwkt_gettoken(&ioapic_irqmap_tok); 1322 1323 for (i = 0; i < count; ++i) { 1324 struct ioapic_irqmap *map; 1325 int intr = intrs[i]; 1326 1327 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1328 ("invalid intr %d", intr)); 1329 1330 map = &ioapic_irqmaps[cpuid][intr]; 1331 KASSERT(map->im_type == type, 1332 ("trying to release non-%s intr %d, type %d", desc, 1333 intr, map->im_type)); 1334 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1335 ("intr %d, invalid %s-base %d", intr, desc, 1336 map->im_msi_base)); 1337 KASSERT((map->im_msi_base & mask) == 0, 1338 ("intr %d, %s-base %d is not properly aligned %d", 1339 intr, desc, map->im_msi_base, count)); 1340 1341 if (msi_base < 0) { 1342 msi_base = map->im_msi_base; 1343 } else { 1344 KASSERT(map->im_msi_base == msi_base, 1345 ("intr %d, inconsistent %s-base, " 1346 "was %d, now %d", 1347 intr, desc, msi_base, map->im_msi_base)); 1348 } 1349 1350 if (intr_next < intr) 1351 intr_next = intr; 1352 1353 map->im_type = IOAPIC_IMT_UNUSED; 1354 map->im_msi_base = -1; 1355 1356 if (bootverbose) { 1357 kprintf("release %s intr %d on cpu%d\n", 1358 desc, intr, cpuid); 1359 } 1360 } 1361 1362 KKASSERT(intr_next > 0); 1363 KKASSERT(msi_base >= 0); 1364 1365 ++intr_next; 1366 if (intr_next < IOAPIC_HWI_VECTORS) { 1367 const struct ioapic_irqmap *map = 1368 &ioapic_irqmaps[cpuid][intr_next]; 1369 1370 if (map->im_type == type) { 1371 KASSERT(map->im_msi_base != msi_base, 1372 ("more than %d %s was allocated", count, desc)); 1373 } 1374 } 1375 1376 lwkt_reltoken(&ioapic_irqmap_tok); 1377 } 1378 1379 static int 1380 ioapic_abi_msi_alloc(int intrs[], int count, int cpuid) 1381 { 1382 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSI, "MSI", 1383 intrs, count, cpuid); 1384 } 1385 1386 static void 1387 ioapic_abi_msi_release(const int intrs[], int count, int cpuid) 1388 { 1389 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSI, "MSI", 1390 intrs, count, cpuid); 1391 } 1392 1393 static int 1394 ioapic_abi_msix_alloc(int *intr, int cpuid) 1395 { 1396 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSIX, "MSI-X", 1397 intr, 1, cpuid); 1398 } 1399 1400 static void 1401 ioapic_abi_msix_release(int intr, int cpuid) 1402 { 1403 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSIX, "MSI-X", 1404 &intr, 1, cpuid); 1405 } 1406 1407 static void 1408 ioapic_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid) 1409 { 1410 const struct ioapic_irqmap *map; 1411 1412 KASSERT(cpuid >= 0 && cpuid < ncpus, 1413 ("invalid cpuid %d", cpuid)); 1414 1415 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1416 ("invalid intr %d", intr)); 1417 1418 lwkt_gettoken(&ioapic_irqmap_tok); 1419 1420 map = &ioapic_irqmaps[cpuid][intr]; 1421 KASSERT(map->im_type == IOAPIC_IMT_MSI || 1422 map->im_type == IOAPIC_IMT_MSIX, 1423 ("trying to map non-MSI/MSI-X intr %d, type %d", intr, map->im_type)); 1424 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1425 ("intr %d, invalid %s-base %d", intr, 1426 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1427 map->im_msi_base)); 1428 1429 msi_map(map->im_msi_base, addr, data, cpuid); 1430 1431 if (bootverbose) { 1432 kprintf("map %s intr %d on cpu%d\n", 1433 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1434 intr, cpuid); 1435 } 1436 1437 lwkt_reltoken(&ioapic_irqmap_tok); 1438 } 1439