1 /* 2 * Copyright (c) 1991 The Regents of the University of California. 3 * Copyright (c) 1996, by Steve Passe. All rights reserved. 4 * Copyright (c) 2005,2008 The DragonFly Project. All rights reserved. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The DragonFly Project 8 * by Matthew Dillon <dillon@backplane.com> 9 * 10 * This code is derived from software contributed to Berkeley by 11 * William Jolitz. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 3. Neither the name of The DragonFly Project nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific, prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 30 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 31 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 34 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 35 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 36 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 37 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * $DragonFly: src/sys/platform/pc64/apic/apic_abi.c,v 1.1 2008/08/29 17:07:12 dillon Exp $ 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/machintr.h> 47 #include <sys/interrupt.h> 48 #include <sys/bus.h> 49 #include <sys/rman.h> 50 #include <sys/thread2.h> 51 52 #include <machine/smp.h> 53 #include <machine/segments.h> 54 #include <machine/md_var.h> 55 #include <machine/intr_machdep.h> 56 #include <machine/globaldata.h> 57 #include <machine/msi_var.h> 58 59 #include <machine_base/isa/isa_intr.h> 60 #include <machine_base/icu/icu.h> 61 #include <machine_base/icu/icu_var.h> 62 #include <machine_base/apic/ioapic.h> 63 #include <machine_base/apic/ioapic_abi.h> 64 #include <machine_base/apic/ioapic_ipl.h> 65 #include <machine_base/apic/apicreg.h> 66 67 #include <dev/acpica5/acpi_sci_var.h> 68 69 #define IOAPIC_HWI_VECTORS IDT_HWI_VECTORS 70 71 extern inthand_t 72 IDTVEC(ioapic_intr0), 73 IDTVEC(ioapic_intr1), 74 IDTVEC(ioapic_intr2), 75 IDTVEC(ioapic_intr3), 76 IDTVEC(ioapic_intr4), 77 IDTVEC(ioapic_intr5), 78 IDTVEC(ioapic_intr6), 79 IDTVEC(ioapic_intr7), 80 IDTVEC(ioapic_intr8), 81 IDTVEC(ioapic_intr9), 82 IDTVEC(ioapic_intr10), 83 IDTVEC(ioapic_intr11), 84 IDTVEC(ioapic_intr12), 85 IDTVEC(ioapic_intr13), 86 IDTVEC(ioapic_intr14), 87 IDTVEC(ioapic_intr15), 88 IDTVEC(ioapic_intr16), 89 IDTVEC(ioapic_intr17), 90 IDTVEC(ioapic_intr18), 91 IDTVEC(ioapic_intr19), 92 IDTVEC(ioapic_intr20), 93 IDTVEC(ioapic_intr21), 94 IDTVEC(ioapic_intr22), 95 IDTVEC(ioapic_intr23), 96 IDTVEC(ioapic_intr24), 97 IDTVEC(ioapic_intr25), 98 IDTVEC(ioapic_intr26), 99 IDTVEC(ioapic_intr27), 100 IDTVEC(ioapic_intr28), 101 IDTVEC(ioapic_intr29), 102 IDTVEC(ioapic_intr30), 103 IDTVEC(ioapic_intr31), 104 IDTVEC(ioapic_intr32), 105 IDTVEC(ioapic_intr33), 106 IDTVEC(ioapic_intr34), 107 IDTVEC(ioapic_intr35), 108 IDTVEC(ioapic_intr36), 109 IDTVEC(ioapic_intr37), 110 IDTVEC(ioapic_intr38), 111 IDTVEC(ioapic_intr39), 112 IDTVEC(ioapic_intr40), 113 IDTVEC(ioapic_intr41), 114 IDTVEC(ioapic_intr42), 115 IDTVEC(ioapic_intr43), 116 IDTVEC(ioapic_intr44), 117 IDTVEC(ioapic_intr45), 118 IDTVEC(ioapic_intr46), 119 IDTVEC(ioapic_intr47), 120 IDTVEC(ioapic_intr48), 121 IDTVEC(ioapic_intr49), 122 IDTVEC(ioapic_intr50), 123 IDTVEC(ioapic_intr51), 124 IDTVEC(ioapic_intr52), 125 IDTVEC(ioapic_intr53), 126 IDTVEC(ioapic_intr54), 127 IDTVEC(ioapic_intr55), 128 IDTVEC(ioapic_intr56), 129 IDTVEC(ioapic_intr57), 130 IDTVEC(ioapic_intr58), 131 IDTVEC(ioapic_intr59), 132 IDTVEC(ioapic_intr60), 133 IDTVEC(ioapic_intr61), 134 IDTVEC(ioapic_intr62), 135 IDTVEC(ioapic_intr63), 136 IDTVEC(ioapic_intr64), 137 IDTVEC(ioapic_intr65), 138 IDTVEC(ioapic_intr66), 139 IDTVEC(ioapic_intr67), 140 IDTVEC(ioapic_intr68), 141 IDTVEC(ioapic_intr69), 142 IDTVEC(ioapic_intr70), 143 IDTVEC(ioapic_intr71), 144 IDTVEC(ioapic_intr72), 145 IDTVEC(ioapic_intr73), 146 IDTVEC(ioapic_intr74), 147 IDTVEC(ioapic_intr75), 148 IDTVEC(ioapic_intr76), 149 IDTVEC(ioapic_intr77), 150 IDTVEC(ioapic_intr78), 151 IDTVEC(ioapic_intr79), 152 IDTVEC(ioapic_intr80), 153 IDTVEC(ioapic_intr81), 154 IDTVEC(ioapic_intr82), 155 IDTVEC(ioapic_intr83), 156 IDTVEC(ioapic_intr84), 157 IDTVEC(ioapic_intr85), 158 IDTVEC(ioapic_intr86), 159 IDTVEC(ioapic_intr87), 160 IDTVEC(ioapic_intr88), 161 IDTVEC(ioapic_intr89), 162 IDTVEC(ioapic_intr90), 163 IDTVEC(ioapic_intr91), 164 IDTVEC(ioapic_intr92), 165 IDTVEC(ioapic_intr93), 166 IDTVEC(ioapic_intr94), 167 IDTVEC(ioapic_intr95), 168 IDTVEC(ioapic_intr96), 169 IDTVEC(ioapic_intr97), 170 IDTVEC(ioapic_intr98), 171 IDTVEC(ioapic_intr99), 172 IDTVEC(ioapic_intr100), 173 IDTVEC(ioapic_intr101), 174 IDTVEC(ioapic_intr102), 175 IDTVEC(ioapic_intr103), 176 IDTVEC(ioapic_intr104), 177 IDTVEC(ioapic_intr105), 178 IDTVEC(ioapic_intr106), 179 IDTVEC(ioapic_intr107), 180 IDTVEC(ioapic_intr108), 181 IDTVEC(ioapic_intr109), 182 IDTVEC(ioapic_intr110), 183 IDTVEC(ioapic_intr111), 184 IDTVEC(ioapic_intr112), 185 IDTVEC(ioapic_intr113), 186 IDTVEC(ioapic_intr114), 187 IDTVEC(ioapic_intr115), 188 IDTVEC(ioapic_intr116), 189 IDTVEC(ioapic_intr117), 190 IDTVEC(ioapic_intr118), 191 IDTVEC(ioapic_intr119), 192 IDTVEC(ioapic_intr120), 193 IDTVEC(ioapic_intr121), 194 IDTVEC(ioapic_intr122), 195 IDTVEC(ioapic_intr123), 196 IDTVEC(ioapic_intr124), 197 IDTVEC(ioapic_intr125), 198 IDTVEC(ioapic_intr126), 199 IDTVEC(ioapic_intr127), 200 IDTVEC(ioapic_intr128), 201 IDTVEC(ioapic_intr129), 202 IDTVEC(ioapic_intr130), 203 IDTVEC(ioapic_intr131), 204 IDTVEC(ioapic_intr132), 205 IDTVEC(ioapic_intr133), 206 IDTVEC(ioapic_intr134), 207 IDTVEC(ioapic_intr135), 208 IDTVEC(ioapic_intr136), 209 IDTVEC(ioapic_intr137), 210 IDTVEC(ioapic_intr138), 211 IDTVEC(ioapic_intr139), 212 IDTVEC(ioapic_intr140), 213 IDTVEC(ioapic_intr141), 214 IDTVEC(ioapic_intr142), 215 IDTVEC(ioapic_intr143), 216 IDTVEC(ioapic_intr144), 217 IDTVEC(ioapic_intr145), 218 IDTVEC(ioapic_intr146), 219 IDTVEC(ioapic_intr147), 220 IDTVEC(ioapic_intr148), 221 IDTVEC(ioapic_intr149), 222 IDTVEC(ioapic_intr150), 223 IDTVEC(ioapic_intr151), 224 IDTVEC(ioapic_intr152), 225 IDTVEC(ioapic_intr153), 226 IDTVEC(ioapic_intr154), 227 IDTVEC(ioapic_intr155), 228 IDTVEC(ioapic_intr156), 229 IDTVEC(ioapic_intr157), 230 IDTVEC(ioapic_intr158), 231 IDTVEC(ioapic_intr159), 232 IDTVEC(ioapic_intr160), 233 IDTVEC(ioapic_intr161), 234 IDTVEC(ioapic_intr162), 235 IDTVEC(ioapic_intr163), 236 IDTVEC(ioapic_intr164), 237 IDTVEC(ioapic_intr165), 238 IDTVEC(ioapic_intr166), 239 IDTVEC(ioapic_intr167), 240 IDTVEC(ioapic_intr168), 241 IDTVEC(ioapic_intr169), 242 IDTVEC(ioapic_intr170), 243 IDTVEC(ioapic_intr171), 244 IDTVEC(ioapic_intr172), 245 IDTVEC(ioapic_intr173), 246 IDTVEC(ioapic_intr174), 247 IDTVEC(ioapic_intr175), 248 IDTVEC(ioapic_intr176), 249 IDTVEC(ioapic_intr177), 250 IDTVEC(ioapic_intr178), 251 IDTVEC(ioapic_intr179), 252 IDTVEC(ioapic_intr180), 253 IDTVEC(ioapic_intr181), 254 IDTVEC(ioapic_intr182), 255 IDTVEC(ioapic_intr183), 256 IDTVEC(ioapic_intr184), 257 IDTVEC(ioapic_intr185), 258 IDTVEC(ioapic_intr186), 259 IDTVEC(ioapic_intr187), 260 IDTVEC(ioapic_intr188), 261 IDTVEC(ioapic_intr189), 262 IDTVEC(ioapic_intr190), 263 IDTVEC(ioapic_intr191); 264 265 static inthand_t *ioapic_intr[IOAPIC_HWI_VECTORS] = { 266 &IDTVEC(ioapic_intr0), 267 &IDTVEC(ioapic_intr1), 268 &IDTVEC(ioapic_intr2), 269 &IDTVEC(ioapic_intr3), 270 &IDTVEC(ioapic_intr4), 271 &IDTVEC(ioapic_intr5), 272 &IDTVEC(ioapic_intr6), 273 &IDTVEC(ioapic_intr7), 274 &IDTVEC(ioapic_intr8), 275 &IDTVEC(ioapic_intr9), 276 &IDTVEC(ioapic_intr10), 277 &IDTVEC(ioapic_intr11), 278 &IDTVEC(ioapic_intr12), 279 &IDTVEC(ioapic_intr13), 280 &IDTVEC(ioapic_intr14), 281 &IDTVEC(ioapic_intr15), 282 &IDTVEC(ioapic_intr16), 283 &IDTVEC(ioapic_intr17), 284 &IDTVEC(ioapic_intr18), 285 &IDTVEC(ioapic_intr19), 286 &IDTVEC(ioapic_intr20), 287 &IDTVEC(ioapic_intr21), 288 &IDTVEC(ioapic_intr22), 289 &IDTVEC(ioapic_intr23), 290 &IDTVEC(ioapic_intr24), 291 &IDTVEC(ioapic_intr25), 292 &IDTVEC(ioapic_intr26), 293 &IDTVEC(ioapic_intr27), 294 &IDTVEC(ioapic_intr28), 295 &IDTVEC(ioapic_intr29), 296 &IDTVEC(ioapic_intr30), 297 &IDTVEC(ioapic_intr31), 298 &IDTVEC(ioapic_intr32), 299 &IDTVEC(ioapic_intr33), 300 &IDTVEC(ioapic_intr34), 301 &IDTVEC(ioapic_intr35), 302 &IDTVEC(ioapic_intr36), 303 &IDTVEC(ioapic_intr37), 304 &IDTVEC(ioapic_intr38), 305 &IDTVEC(ioapic_intr39), 306 &IDTVEC(ioapic_intr40), 307 &IDTVEC(ioapic_intr41), 308 &IDTVEC(ioapic_intr42), 309 &IDTVEC(ioapic_intr43), 310 &IDTVEC(ioapic_intr44), 311 &IDTVEC(ioapic_intr45), 312 &IDTVEC(ioapic_intr46), 313 &IDTVEC(ioapic_intr47), 314 &IDTVEC(ioapic_intr48), 315 &IDTVEC(ioapic_intr49), 316 &IDTVEC(ioapic_intr50), 317 &IDTVEC(ioapic_intr51), 318 &IDTVEC(ioapic_intr52), 319 &IDTVEC(ioapic_intr53), 320 &IDTVEC(ioapic_intr54), 321 &IDTVEC(ioapic_intr55), 322 &IDTVEC(ioapic_intr56), 323 &IDTVEC(ioapic_intr57), 324 &IDTVEC(ioapic_intr58), 325 &IDTVEC(ioapic_intr59), 326 &IDTVEC(ioapic_intr60), 327 &IDTVEC(ioapic_intr61), 328 &IDTVEC(ioapic_intr62), 329 &IDTVEC(ioapic_intr63), 330 &IDTVEC(ioapic_intr64), 331 &IDTVEC(ioapic_intr65), 332 &IDTVEC(ioapic_intr66), 333 &IDTVEC(ioapic_intr67), 334 &IDTVEC(ioapic_intr68), 335 &IDTVEC(ioapic_intr69), 336 &IDTVEC(ioapic_intr70), 337 &IDTVEC(ioapic_intr71), 338 &IDTVEC(ioapic_intr72), 339 &IDTVEC(ioapic_intr73), 340 &IDTVEC(ioapic_intr74), 341 &IDTVEC(ioapic_intr75), 342 &IDTVEC(ioapic_intr76), 343 &IDTVEC(ioapic_intr77), 344 &IDTVEC(ioapic_intr78), 345 &IDTVEC(ioapic_intr79), 346 &IDTVEC(ioapic_intr80), 347 &IDTVEC(ioapic_intr81), 348 &IDTVEC(ioapic_intr82), 349 &IDTVEC(ioapic_intr83), 350 &IDTVEC(ioapic_intr84), 351 &IDTVEC(ioapic_intr85), 352 &IDTVEC(ioapic_intr86), 353 &IDTVEC(ioapic_intr87), 354 &IDTVEC(ioapic_intr88), 355 &IDTVEC(ioapic_intr89), 356 &IDTVEC(ioapic_intr90), 357 &IDTVEC(ioapic_intr91), 358 &IDTVEC(ioapic_intr92), 359 &IDTVEC(ioapic_intr93), 360 &IDTVEC(ioapic_intr94), 361 &IDTVEC(ioapic_intr95), 362 &IDTVEC(ioapic_intr96), 363 &IDTVEC(ioapic_intr97), 364 &IDTVEC(ioapic_intr98), 365 &IDTVEC(ioapic_intr99), 366 &IDTVEC(ioapic_intr100), 367 &IDTVEC(ioapic_intr101), 368 &IDTVEC(ioapic_intr102), 369 &IDTVEC(ioapic_intr103), 370 &IDTVEC(ioapic_intr104), 371 &IDTVEC(ioapic_intr105), 372 &IDTVEC(ioapic_intr106), 373 &IDTVEC(ioapic_intr107), 374 &IDTVEC(ioapic_intr108), 375 &IDTVEC(ioapic_intr109), 376 &IDTVEC(ioapic_intr110), 377 &IDTVEC(ioapic_intr111), 378 &IDTVEC(ioapic_intr112), 379 &IDTVEC(ioapic_intr113), 380 &IDTVEC(ioapic_intr114), 381 &IDTVEC(ioapic_intr115), 382 &IDTVEC(ioapic_intr116), 383 &IDTVEC(ioapic_intr117), 384 &IDTVEC(ioapic_intr118), 385 &IDTVEC(ioapic_intr119), 386 &IDTVEC(ioapic_intr120), 387 &IDTVEC(ioapic_intr121), 388 &IDTVEC(ioapic_intr122), 389 &IDTVEC(ioapic_intr123), 390 &IDTVEC(ioapic_intr124), 391 &IDTVEC(ioapic_intr125), 392 &IDTVEC(ioapic_intr126), 393 &IDTVEC(ioapic_intr127), 394 &IDTVEC(ioapic_intr128), 395 &IDTVEC(ioapic_intr129), 396 &IDTVEC(ioapic_intr130), 397 &IDTVEC(ioapic_intr131), 398 &IDTVEC(ioapic_intr132), 399 &IDTVEC(ioapic_intr133), 400 &IDTVEC(ioapic_intr134), 401 &IDTVEC(ioapic_intr135), 402 &IDTVEC(ioapic_intr136), 403 &IDTVEC(ioapic_intr137), 404 &IDTVEC(ioapic_intr138), 405 &IDTVEC(ioapic_intr139), 406 &IDTVEC(ioapic_intr140), 407 &IDTVEC(ioapic_intr141), 408 &IDTVEC(ioapic_intr142), 409 &IDTVEC(ioapic_intr143), 410 &IDTVEC(ioapic_intr144), 411 &IDTVEC(ioapic_intr145), 412 &IDTVEC(ioapic_intr146), 413 &IDTVEC(ioapic_intr147), 414 &IDTVEC(ioapic_intr148), 415 &IDTVEC(ioapic_intr149), 416 &IDTVEC(ioapic_intr150), 417 &IDTVEC(ioapic_intr151), 418 &IDTVEC(ioapic_intr152), 419 &IDTVEC(ioapic_intr153), 420 &IDTVEC(ioapic_intr154), 421 &IDTVEC(ioapic_intr155), 422 &IDTVEC(ioapic_intr156), 423 &IDTVEC(ioapic_intr157), 424 &IDTVEC(ioapic_intr158), 425 &IDTVEC(ioapic_intr159), 426 &IDTVEC(ioapic_intr160), 427 &IDTVEC(ioapic_intr161), 428 &IDTVEC(ioapic_intr162), 429 &IDTVEC(ioapic_intr163), 430 &IDTVEC(ioapic_intr164), 431 &IDTVEC(ioapic_intr165), 432 &IDTVEC(ioapic_intr166), 433 &IDTVEC(ioapic_intr167), 434 &IDTVEC(ioapic_intr168), 435 &IDTVEC(ioapic_intr169), 436 &IDTVEC(ioapic_intr170), 437 &IDTVEC(ioapic_intr171), 438 &IDTVEC(ioapic_intr172), 439 &IDTVEC(ioapic_intr173), 440 &IDTVEC(ioapic_intr174), 441 &IDTVEC(ioapic_intr175), 442 &IDTVEC(ioapic_intr176), 443 &IDTVEC(ioapic_intr177), 444 &IDTVEC(ioapic_intr178), 445 &IDTVEC(ioapic_intr179), 446 &IDTVEC(ioapic_intr180), 447 &IDTVEC(ioapic_intr181), 448 &IDTVEC(ioapic_intr182), 449 &IDTVEC(ioapic_intr183), 450 &IDTVEC(ioapic_intr184), 451 &IDTVEC(ioapic_intr185), 452 &IDTVEC(ioapic_intr186), 453 &IDTVEC(ioapic_intr187), 454 &IDTVEC(ioapic_intr188), 455 &IDTVEC(ioapic_intr189), 456 &IDTVEC(ioapic_intr190), 457 &IDTVEC(ioapic_intr191) 458 }; 459 460 #define IOAPIC_HWI_SYSCALL (IDT_OFFSET_SYSCALL - IDT_OFFSET) 461 462 static struct ioapic_irqmap { 463 int im_type; /* IOAPIC_IMT_ */ 464 enum intr_trigger im_trig; 465 enum intr_polarity im_pola; 466 int im_gsi; 467 int im_msi_base; 468 uint32_t im_flags; /* IOAPIC_IMF_ */ 469 } ioapic_irqmaps[MAXCPU][IOAPIC_HWI_VECTORS]; 470 471 static struct lwkt_token ioapic_irqmap_tok = 472 LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token); 473 474 #define IOAPIC_IMT_UNUSED 0 475 #define IOAPIC_IMT_RESERVED 1 476 #define IOAPIC_IMT_LEGACY 2 477 #define IOAPIC_IMT_SYSCALL 3 478 #define IOAPIC_IMT_MSI 4 479 #define IOAPIC_IMT_MSIX 5 480 481 #define IOAPIC_IMT_ISHWI(map) ((map)->im_type != IOAPIC_IMT_RESERVED && \ 482 (map)->im_type != IOAPIC_IMT_SYSCALL) 483 484 #define IOAPIC_IMF_CONF 0x1 485 486 extern void IOAPIC_INTREN(int); 487 extern void IOAPIC_INTRDIS(int); 488 489 extern int imcr_present; 490 491 static void ioapic_abi_intr_enable(int); 492 static void ioapic_abi_intr_disable(int); 493 static void ioapic_abi_intr_setup(int, int); 494 static void ioapic_abi_intr_teardown(int); 495 496 static void ioapic_abi_legacy_intr_config(int, 497 enum intr_trigger, enum intr_polarity); 498 static int ioapic_abi_legacy_intr_cpuid(int); 499 500 static int ioapic_abi_msi_alloc(int [], int, int); 501 static void ioapic_abi_msi_release(const int [], int, int); 502 static void ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int); 503 static int ioapic_abi_msix_alloc(int *, int); 504 static void ioapic_abi_msix_release(int, int); 505 506 static int ioapic_abi_msi_alloc_intern(int, const char *, 507 int [], int, int); 508 static void ioapic_abi_msi_release_intern(int, const char *, 509 const int [], int, int); 510 511 static void ioapic_abi_finalize(void); 512 static void ioapic_abi_cleanup(void); 513 static void ioapic_abi_setdefault(void); 514 static void ioapic_abi_stabilize(void); 515 static void ioapic_abi_initmap(void); 516 static void ioapic_abi_rman_setup(struct rman *); 517 518 static int ioapic_abi_gsi_cpuid(int, int); 519 520 struct machintr_abi MachIntrABI_IOAPIC = { 521 MACHINTR_IOAPIC, 522 .intr_disable = ioapic_abi_intr_disable, 523 .intr_enable = ioapic_abi_intr_enable, 524 .intr_setup = ioapic_abi_intr_setup, 525 .intr_teardown = ioapic_abi_intr_teardown, 526 527 .legacy_intr_config = ioapic_abi_legacy_intr_config, 528 .legacy_intr_cpuid = ioapic_abi_legacy_intr_cpuid, 529 530 .msi_alloc = ioapic_abi_msi_alloc, 531 .msi_release = ioapic_abi_msi_release, 532 .msi_map = ioapic_abi_msi_map, 533 .msix_alloc = ioapic_abi_msix_alloc, 534 .msix_release = ioapic_abi_msix_release, 535 536 .finalize = ioapic_abi_finalize, 537 .cleanup = ioapic_abi_cleanup, 538 .setdefault = ioapic_abi_setdefault, 539 .stabilize = ioapic_abi_stabilize, 540 .initmap = ioapic_abi_initmap, 541 .rman_setup = ioapic_abi_rman_setup 542 }; 543 544 static int ioapic_abi_extint_irq = -1; 545 static int ioapic_abi_legacy_irq_max; 546 static int ioapic_abi_gsi_balance; 547 static int ioapic_abi_msi_start; /* NOTE: for testing only */ 548 549 struct ioapic_irqinfo ioapic_irqs[IOAPIC_HWI_VECTORS]; 550 551 static void 552 ioapic_abi_intr_enable(int irq) 553 { 554 const struct ioapic_irqmap *map; 555 556 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 557 ("ioapic enable, invalid irq %d\n", irq)); 558 559 map = &ioapic_irqmaps[mycpuid][irq]; 560 KASSERT(IOAPIC_IMT_ISHWI(map), 561 ("ioapic enable, not hwi irq %d, type %d, cpu%d\n", 562 irq, map->im_type, mycpuid)); 563 if (map->im_type != IOAPIC_IMT_LEGACY) 564 return; 565 566 IOAPIC_INTREN(irq); 567 } 568 569 static void 570 ioapic_abi_intr_disable(int irq) 571 { 572 const struct ioapic_irqmap *map; 573 574 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 575 ("ioapic disable, invalid irq %d\n", irq)); 576 577 map = &ioapic_irqmaps[mycpuid][irq]; 578 KASSERT(IOAPIC_IMT_ISHWI(map), 579 ("ioapic disable, not hwi irq %d, type %d, cpu%d\n", 580 irq, map->im_type, mycpuid)); 581 if (map->im_type != IOAPIC_IMT_LEGACY) 582 return; 583 584 IOAPIC_INTRDIS(irq); 585 } 586 587 static void 588 ioapic_abi_finalize(void) 589 { 590 KKASSERT(MachIntrABI.type == MACHINTR_IOAPIC); 591 KKASSERT(ioapic_enable); 592 593 /* 594 * If an IMCR is present, program bit 0 to disconnect the 8259 595 * from the BSP. 596 */ 597 if (imcr_present) { 598 outb(0x22, 0x70); /* select IMCR */ 599 outb(0x23, 0x01); /* disconnect 8259 */ 600 } 601 } 602 603 /* 604 * This routine is called after physical interrupts are enabled but before 605 * the critical section is released. We need to clean out any interrupts 606 * that had already been posted to the cpu. 607 */ 608 static void 609 ioapic_abi_cleanup(void) 610 { 611 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending)); 612 } 613 614 /* Must never be called */ 615 static void 616 ioapic_abi_stabilize(void) 617 { 618 panic("ioapic_stabilize is called\n"); 619 } 620 621 static void 622 ioapic_abi_intr_setup(int intr, int flags) 623 { 624 const struct ioapic_irqmap *map; 625 int vector, select; 626 uint32_t value; 627 register_t ef; 628 629 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 630 ("ioapic setup, invalid irq %d\n", intr)); 631 632 map = &ioapic_irqmaps[mycpuid][intr]; 633 KASSERT(IOAPIC_IMT_ISHWI(map), 634 ("ioapic setup, not hwi irq %d, type %d, cpu%d", 635 intr, map->im_type, mycpuid)); 636 if (map->im_type != IOAPIC_IMT_LEGACY) 637 return; 638 639 KASSERT(ioapic_irqs[intr].io_addr != NULL, 640 ("ioapic setup, no GSI information, irq %d\n", intr)); 641 642 ef = read_rflags(); 643 cpu_disable_intr(); 644 645 vector = IDT_OFFSET + intr; 646 647 /* 648 * Now reprogram the vector in the IO APIC. In order to avoid 649 * losing an EOI for a level interrupt, which is vector based, 650 * make sure that the IO APIC is programmed for edge-triggering 651 * first, then reprogrammed with the new vector. This should 652 * clear the IRR bit. 653 */ 654 imen_lock(); 655 656 select = ioapic_irqs[intr].io_idx; 657 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 658 value |= IOART_INTMSET; 659 660 ioapic_write(ioapic_irqs[intr].io_addr, select, 661 (value & ~APIC_TRIGMOD_MASK)); 662 ioapic_write(ioapic_irqs[intr].io_addr, select, 663 (value & ~IOART_INTVEC) | vector); 664 665 imen_unlock(); 666 667 IOAPIC_INTREN(intr); 668 669 write_rflags(ef); 670 } 671 672 static void 673 ioapic_abi_intr_teardown(int intr) 674 { 675 const struct ioapic_irqmap *map; 676 int vector, select; 677 uint32_t value; 678 register_t ef; 679 680 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 681 ("ioapic teardown, invalid irq %d\n", intr)); 682 683 map = &ioapic_irqmaps[mycpuid][intr]; 684 KASSERT(IOAPIC_IMT_ISHWI(map), 685 ("ioapic teardown, not hwi irq %d, type %d, cpu%d", 686 intr, map->im_type, mycpuid)); 687 if (map->im_type != IOAPIC_IMT_LEGACY) 688 return; 689 690 KASSERT(ioapic_irqs[intr].io_addr != NULL, 691 ("ioapic teardown, no GSI information, irq %d\n", intr)); 692 693 ef = read_rflags(); 694 cpu_disable_intr(); 695 696 /* 697 * Teardown an interrupt vector. The vector should already be 698 * installed in the cpu's IDT, but make sure. 699 */ 700 IOAPIC_INTRDIS(intr); 701 702 vector = IDT_OFFSET + intr; 703 704 /* 705 * In order to avoid losing an EOI for a level interrupt, which 706 * is vector based, make sure that the IO APIC is programmed for 707 * edge-triggering first, then reprogrammed with the new vector. 708 * This should clear the IRR bit. 709 */ 710 imen_lock(); 711 712 select = ioapic_irqs[intr].io_idx; 713 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 714 715 ioapic_write(ioapic_irqs[intr].io_addr, select, 716 (value & ~APIC_TRIGMOD_MASK)); 717 ioapic_write(ioapic_irqs[intr].io_addr, select, 718 (value & ~IOART_INTVEC) | vector); 719 720 imen_unlock(); 721 722 write_rflags(ef); 723 } 724 725 static void 726 ioapic_abi_setdefault(void) 727 { 728 int intr; 729 730 for (intr = 0; intr < IOAPIC_HWI_VECTORS; ++intr) { 731 if (intr == IOAPIC_HWI_SYSCALL) 732 continue; 733 setidt_global(IDT_OFFSET + intr, ioapic_intr[intr], 734 SDT_SYSIGT, SEL_KPL, 0); 735 } 736 } 737 738 static void 739 ioapic_abi_initmap(void) 740 { 741 int cpu; 742 743 kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance); 744 745 kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start); 746 ioapic_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */ 747 748 /* 749 * NOTE: ncpus is not ready yet 750 */ 751 for (cpu = 0; cpu < MAXCPU; ++cpu) { 752 int i; 753 754 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 755 ioapic_irqmaps[cpu][i].im_gsi = -1; 756 ioapic_irqmaps[cpu][i].im_msi_base = -1; 757 } 758 ioapic_irqmaps[cpu][IOAPIC_HWI_SYSCALL].im_type = 759 IOAPIC_IMT_SYSCALL; 760 } 761 } 762 763 void 764 ioapic_set_legacy_irqmap(int irq, int gsi, enum intr_trigger trig, 765 enum intr_polarity pola) 766 { 767 struct ioapic_irqinfo *info; 768 struct ioapic_irqmap *map; 769 void *ioaddr; 770 int pin, cpuid; 771 772 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 773 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 774 775 KKASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS); 776 if (irq > ioapic_abi_legacy_irq_max) 777 ioapic_abi_legacy_irq_max = irq; 778 779 cpuid = ioapic_abi_gsi_cpuid(irq, gsi); 780 781 map = &ioapic_irqmaps[cpuid][irq]; 782 783 KKASSERT(map->im_type == IOAPIC_IMT_UNUSED); 784 map->im_type = IOAPIC_IMT_LEGACY; 785 786 map->im_gsi = gsi; 787 map->im_trig = trig; 788 map->im_pola = pola; 789 790 if (bootverbose) { 791 kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n", 792 irq, map->im_gsi, 793 intr_str_trigger(map->im_trig), 794 intr_str_polarity(map->im_pola)); 795 } 796 797 pin = ioapic_gsi_pin(map->im_gsi); 798 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 799 800 info = &ioapic_irqs[irq]; 801 802 imen_lock(); 803 804 info->io_addr = ioaddr; 805 info->io_idx = IOAPIC_REDTBL + (2 * pin); 806 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 807 if (map->im_trig == INTR_TRIGGER_LEVEL) 808 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 809 810 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 811 map->im_trig, map->im_pola, cpuid); 812 813 imen_unlock(); 814 } 815 816 void 817 ioapic_fixup_legacy_irqmaps(void) 818 { 819 int cpu; 820 821 for (cpu = 0; cpu < ncpus; ++cpu) { 822 int i; 823 824 for (i = 0; i < ISA_IRQ_CNT; ++i) { 825 struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][i]; 826 827 if (map->im_type == IOAPIC_IMT_UNUSED) { 828 map->im_type = IOAPIC_IMT_RESERVED; 829 if (bootverbose) { 830 kprintf("IOAPIC: " 831 "cpu%d irq %d reserved\n", cpu, i); 832 } 833 } 834 } 835 } 836 837 ioapic_abi_legacy_irq_max += 1; 838 if (bootverbose) { 839 kprintf("IOAPIC: legacy irq max %d\n", 840 ioapic_abi_legacy_irq_max); 841 } 842 } 843 844 int 845 ioapic_find_legacy_by_gsi(int gsi, enum intr_trigger trig, 846 enum intr_polarity pola) 847 { 848 int cpu; 849 850 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 851 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 852 853 for (cpu = 0; cpu < ncpus; ++cpu) { 854 int irq; 855 856 for (irq = 0; irq < ioapic_abi_legacy_irq_max; ++irq) { 857 const struct ioapic_irqmap *map = 858 &ioapic_irqmaps[cpu][irq]; 859 860 if (map->im_gsi == gsi) { 861 KKASSERT(map->im_type == IOAPIC_IMT_LEGACY); 862 863 if (map->im_flags & IOAPIC_IMF_CONF) { 864 if (map->im_trig != trig || 865 map->im_pola != pola) 866 return -1; 867 } 868 return irq; 869 } 870 } 871 } 872 return -1; 873 } 874 875 int 876 ioapic_find_legacy_by_irq(int irq, enum intr_trigger trig, 877 enum intr_polarity pola) 878 { 879 int cpu; 880 881 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 882 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 883 884 if (irq < 0 || irq >= ioapic_abi_legacy_irq_max) 885 return -1; 886 887 for (cpu = 0; cpu < ncpus; ++cpu) { 888 const struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][irq]; 889 890 if (map->im_type == IOAPIC_IMT_LEGACY) { 891 if (map->im_flags & IOAPIC_IMF_CONF) { 892 if (map->im_trig != trig || 893 map->im_pola != pola) 894 return -1; 895 } 896 return irq; 897 } 898 } 899 return -1; 900 } 901 902 static void 903 ioapic_abi_legacy_intr_config(int irq, enum intr_trigger trig, 904 enum intr_polarity pola) 905 { 906 struct ioapic_irqinfo *info; 907 struct ioapic_irqmap *map = NULL; 908 void *ioaddr; 909 int pin, cpuid; 910 911 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 912 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 913 914 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 915 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 916 map = &ioapic_irqmaps[cpuid][irq]; 917 if (map->im_type == IOAPIC_IMT_LEGACY) 918 break; 919 } 920 KKASSERT(cpuid < ncpus); 921 922 #ifdef notyet 923 if (map->im_flags & IOAPIC_IMF_CONF) { 924 if (trig != map->im_trig) { 925 panic("ioapic_intr_config: trig %s -> %s\n", 926 intr_str_trigger(map->im_trig), 927 intr_str_trigger(trig)); 928 } 929 if (pola != map->im_pola) { 930 panic("ioapic_intr_config: pola %s -> %s\n", 931 intr_str_polarity(map->im_pola), 932 intr_str_polarity(pola)); 933 } 934 return; 935 } 936 #endif 937 map->im_flags |= IOAPIC_IMF_CONF; 938 939 if (trig == map->im_trig && pola == map->im_pola) 940 return; 941 942 if (bootverbose) { 943 kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n", 944 irq, map->im_gsi, 945 intr_str_trigger(map->im_trig), 946 intr_str_polarity(map->im_pola), 947 intr_str_trigger(trig), 948 intr_str_polarity(pola)); 949 } 950 map->im_trig = trig; 951 map->im_pola = pola; 952 953 pin = ioapic_gsi_pin(map->im_gsi); 954 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 955 956 info = &ioapic_irqs[irq]; 957 958 imen_lock(); 959 960 info->io_flags &= ~IOAPIC_IRQI_FLAG_LEVEL; 961 if (map->im_trig == INTR_TRIGGER_LEVEL) 962 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 963 964 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 965 map->im_trig, map->im_pola, cpuid); 966 967 imen_unlock(); 968 } 969 970 int 971 ioapic_conf_legacy_extint(int irq) 972 { 973 struct ioapic_irqinfo *info; 974 struct ioapic_irqmap *map; 975 void *ioaddr; 976 int pin, error, vec; 977 978 /* XXX only irq0 is allowed */ 979 KKASSERT(irq == 0); 980 981 vec = IDT_OFFSET + irq; 982 983 if (ioapic_abi_extint_irq == irq) 984 return 0; 985 else if (ioapic_abi_extint_irq >= 0) 986 return EEXIST; 987 988 error = icu_ioapic_extint(irq, vec); 989 if (error) 990 return error; 991 992 /* ExtINT is always targeted to cpu0 */ 993 map = &ioapic_irqmaps[0][irq]; 994 995 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED || 996 map->im_type == IOAPIC_IMT_LEGACY); 997 if (map->im_type == IOAPIC_IMT_LEGACY) { 998 if (map->im_flags & IOAPIC_IMF_CONF) 999 return EEXIST; 1000 } 1001 ioapic_abi_extint_irq = irq; 1002 1003 map->im_type = IOAPIC_IMT_LEGACY; 1004 map->im_trig = INTR_TRIGGER_EDGE; 1005 map->im_pola = INTR_POLARITY_HIGH; 1006 map->im_flags = IOAPIC_IMF_CONF; 1007 1008 map->im_gsi = ioapic_extpin_gsi(); 1009 KKASSERT(map->im_gsi >= 0); 1010 1011 if (bootverbose) { 1012 kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n", 1013 irq, map->im_gsi, 1014 intr_str_trigger(map->im_trig), 1015 intr_str_polarity(map->im_pola)); 1016 } 1017 1018 pin = ioapic_gsi_pin(map->im_gsi); 1019 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1020 1021 info = &ioapic_irqs[irq]; 1022 1023 imen_lock(); 1024 1025 info->io_addr = ioaddr; 1026 info->io_idx = IOAPIC_REDTBL + (2 * pin); 1027 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 1028 1029 ioapic_extpin_setup(ioaddr, pin, vec); 1030 1031 imen_unlock(); 1032 1033 return 0; 1034 } 1035 1036 static int 1037 ioapic_abi_legacy_intr_cpuid(int irq) 1038 { 1039 const struct ioapic_irqmap *map = NULL; 1040 int cpuid; 1041 1042 KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max); 1043 1044 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1045 map = &ioapic_irqmaps[cpuid][irq]; 1046 if (map->im_type == IOAPIC_IMT_LEGACY) 1047 return cpuid; 1048 } 1049 1050 /* XXX some drivers tries to peek at reserved IRQs */ 1051 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1052 map = &ioapic_irqmaps[cpuid][irq]; 1053 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED); 1054 } 1055 return 0; 1056 } 1057 1058 static int 1059 ioapic_abi_gsi_cpuid(int irq, int gsi) 1060 { 1061 char envpath[32]; 1062 int cpuid = -1; 1063 1064 KKASSERT(gsi >= 0); 1065 1066 if (irq == 0 || gsi == 0) { 1067 if (bootverbose) { 1068 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n", 1069 irq, gsi); 1070 } 1071 return 0; 1072 } 1073 1074 if (irq == acpi_sci_irqno()) { 1075 if (bootverbose) { 1076 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n", 1077 irq, gsi); 1078 } 1079 return 0; 1080 } 1081 1082 ksnprintf(envpath, sizeof(envpath), "hw.ioapic.gsi.%d.cpu", gsi); 1083 kgetenv_int(envpath, &cpuid); 1084 1085 if (cpuid < 0) { 1086 if (!ioapic_abi_gsi_balance) { 1087 if (bootverbose) { 1088 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 " 1089 "(fixed)\n", irq, gsi); 1090 } 1091 return 0; 1092 } 1093 1094 cpuid = gsi % ncpus; 1095 if (bootverbose) { 1096 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n", 1097 irq, gsi, cpuid); 1098 } 1099 } else if (cpuid >= ncpus) { 1100 cpuid = ncpus - 1; 1101 if (bootverbose) { 1102 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n", 1103 irq, gsi, cpuid); 1104 } 1105 } else { 1106 if (bootverbose) { 1107 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n", 1108 irq, gsi, cpuid); 1109 } 1110 } 1111 return cpuid; 1112 } 1113 1114 static void 1115 ioapic_abi_rman_setup(struct rman *rm) 1116 { 1117 int start, end, i; 1118 1119 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU, 1120 ("invalid rman cpuid %d", rm->rm_cpuid)); 1121 1122 start = end = -1; 1123 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 1124 const struct ioapic_irqmap *map = 1125 &ioapic_irqmaps[rm->rm_cpuid][i]; 1126 1127 if (start < 0) { 1128 if (IOAPIC_IMT_ISHWI(map)) 1129 start = end = i; 1130 } else { 1131 if (IOAPIC_IMT_ISHWI(map)) { 1132 end = i; 1133 } else { 1134 KKASSERT(end >= 0); 1135 if (bootverbose) { 1136 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1137 rm->rm_cpuid, start, end); 1138 } 1139 if (rman_manage_region(rm, start, end)) { 1140 panic("rman_manage_region" 1141 "(cpu%d %d - %d)", rm->rm_cpuid, 1142 start, end); 1143 } 1144 start = end = -1; 1145 } 1146 } 1147 } 1148 if (start >= 0) { 1149 KKASSERT(end >= 0); 1150 if (bootverbose) { 1151 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1152 rm->rm_cpuid, start, end); 1153 } 1154 if (rman_manage_region(rm, start, end)) { 1155 panic("rman_manage_region(cpu%d %d - %d)", 1156 rm->rm_cpuid, start, end); 1157 } 1158 } 1159 } 1160 1161 static int 1162 ioapic_abi_msi_alloc_intern(int type, const char *desc, 1163 int intrs[], int count, int cpuid) 1164 { 1165 int i, error; 1166 1167 KASSERT(cpuid >= 0 && cpuid < ncpus, 1168 ("invalid cpuid %d", cpuid)); 1169 1170 KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count)); 1171 KASSERT((count & (count - 1)) == 0, 1172 ("count %d is not power of 2\n", count)); 1173 1174 lwkt_gettoken(&ioapic_irqmap_tok); 1175 1176 /* 1177 * NOTE: 1178 * Since IDT_OFFSET is 32, which is the maximum valid 'count', 1179 * we do not need to find out the first properly aligned 1180 * interrupt vector. 1181 */ 1182 1183 error = EMSGSIZE; 1184 for (i = ioapic_abi_msi_start; i < IOAPIC_HWI_VECTORS; i += count) { 1185 int j; 1186 1187 if (ioapic_irqmaps[cpuid][i].im_type != IOAPIC_IMT_UNUSED) 1188 continue; 1189 1190 for (j = 1; j < count; ++j) { 1191 if (ioapic_irqmaps[cpuid][i + j].im_type != 1192 IOAPIC_IMT_UNUSED) 1193 break; 1194 } 1195 if (j != count) 1196 continue; 1197 1198 for (j = 0; j < count; ++j) { 1199 struct ioapic_irqmap *map; 1200 int intr = i + j; 1201 1202 map = &ioapic_irqmaps[cpuid][intr]; 1203 KASSERT(map->im_msi_base < 0, 1204 ("intr %d, stale %s-base %d\n", 1205 intr, desc, map->im_msi_base)); 1206 1207 map->im_type = type; 1208 map->im_msi_base = i; 1209 1210 intrs[j] = intr; 1211 msi_setup(intr, cpuid); 1212 1213 if (bootverbose) { 1214 kprintf("alloc %s intr %d on cpu%d\n", 1215 desc, intr, cpuid); 1216 } 1217 } 1218 error = 0; 1219 break; 1220 } 1221 1222 lwkt_reltoken(&ioapic_irqmap_tok); 1223 1224 return error; 1225 } 1226 1227 static void 1228 ioapic_abi_msi_release_intern(int type, const char *desc, 1229 const int intrs[], int count, int cpuid) 1230 { 1231 int i, msi_base = -1, intr_next = -1, mask; 1232 1233 KASSERT(cpuid >= 0 && cpuid < ncpus, 1234 ("invalid cpuid %d", cpuid)); 1235 1236 KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count)); 1237 1238 mask = count - 1; 1239 KASSERT((count & mask) == 0, ("count %d is not power of 2\n", count)); 1240 1241 lwkt_gettoken(&ioapic_irqmap_tok); 1242 1243 for (i = 0; i < count; ++i) { 1244 struct ioapic_irqmap *map; 1245 int intr = intrs[i]; 1246 1247 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1248 ("invalid intr %d\n", intr)); 1249 1250 map = &ioapic_irqmaps[cpuid][intr]; 1251 KASSERT(map->im_type == type, 1252 ("try release non-%s intr %d, type %d\n", desc, 1253 intr, map->im_type)); 1254 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1255 ("intr %d, invalid %s-base %d\n", intr, desc, 1256 map->im_msi_base)); 1257 KASSERT((map->im_msi_base & mask) == 0, 1258 ("intr %d, %s-base %d is not proper aligned %d\n", 1259 intr, desc, map->im_msi_base, count)); 1260 1261 if (msi_base < 0) { 1262 msi_base = map->im_msi_base; 1263 } else { 1264 KASSERT(map->im_msi_base == msi_base, 1265 ("intr %d, inconsistent %s-base, " 1266 "was %d, now %d\n", 1267 intr, desc, msi_base, map->im_msi_base)); 1268 } 1269 1270 if (intr_next < intr) 1271 intr_next = intr; 1272 1273 map->im_type = IOAPIC_IMT_UNUSED; 1274 map->im_msi_base = -1; 1275 1276 if (bootverbose) { 1277 kprintf("release %s intr %d on cpu%d\n", 1278 desc, intr, cpuid); 1279 } 1280 } 1281 1282 KKASSERT(intr_next > 0); 1283 KKASSERT(msi_base >= 0); 1284 1285 ++intr_next; 1286 if (intr_next < IOAPIC_HWI_VECTORS) { 1287 const struct ioapic_irqmap *map = 1288 &ioapic_irqmaps[cpuid][intr_next]; 1289 1290 if (map->im_type == type) { 1291 KASSERT(map->im_msi_base != msi_base, 1292 ("more than %d %s was allocated\n", count, desc)); 1293 } 1294 } 1295 1296 lwkt_reltoken(&ioapic_irqmap_tok); 1297 } 1298 1299 static int 1300 ioapic_abi_msi_alloc(int intrs[], int count, int cpuid) 1301 { 1302 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSI, "MSI", 1303 intrs, count, cpuid); 1304 } 1305 1306 static void 1307 ioapic_abi_msi_release(const int intrs[], int count, int cpuid) 1308 { 1309 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSI, "MSI", 1310 intrs, count, cpuid); 1311 } 1312 1313 static int 1314 ioapic_abi_msix_alloc(int *intr, int cpuid) 1315 { 1316 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSIX, "MSI-X", 1317 intr, 1, cpuid); 1318 } 1319 1320 static void 1321 ioapic_abi_msix_release(int intr, int cpuid) 1322 { 1323 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSIX, "MSI-X", 1324 &intr, 1, cpuid); 1325 } 1326 1327 static void 1328 ioapic_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid) 1329 { 1330 const struct ioapic_irqmap *map; 1331 1332 KASSERT(cpuid >= 0 && cpuid < ncpus, 1333 ("invalid cpuid %d", cpuid)); 1334 1335 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1336 ("invalid intr %d\n", intr)); 1337 1338 lwkt_gettoken(&ioapic_irqmap_tok); 1339 1340 map = &ioapic_irqmaps[cpuid][intr]; 1341 KASSERT(map->im_type == IOAPIC_IMT_MSI || 1342 map->im_type == IOAPIC_IMT_MSIX, 1343 ("try map non-MSI/MSI-X intr %d, type %d\n", intr, map->im_type)); 1344 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1345 ("intr %d, invalid %s-base %d\n", intr, 1346 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1347 map->im_msi_base)); 1348 1349 msi_map(map->im_msi_base, addr, data, cpuid); 1350 1351 if (bootverbose) { 1352 kprintf("map %s intr %d on cpu%d\n", 1353 map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X", 1354 intr, cpuid); 1355 } 1356 1357 lwkt_reltoken(&ioapic_irqmap_tok); 1358 } 1359