1 /* 2 * Copyright (c) 1991 The Regents of the University of California. 3 * Copyright (c) 1996, by Steve Passe. All rights reserved. 4 * Copyright (c) 2005,2008 The DragonFly Project. All rights reserved. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The DragonFly Project 8 * by Matthew Dillon <dillon@backplane.com> 9 * 10 * This code is derived from software contributed to Berkeley by 11 * William Jolitz. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 3. Neither the name of The DragonFly Project nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific, prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 30 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 31 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 34 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 35 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 36 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 37 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * $DragonFly: src/sys/platform/pc64/apic/apic_abi.c,v 1.1 2008/08/29 17:07:12 dillon Exp $ 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/machintr.h> 47 #include <sys/interrupt.h> 48 #include <sys/bus.h> 49 #include <sys/rman.h> 50 #include <sys/thread2.h> 51 52 #include <machine/smp.h> 53 #include <machine/segments.h> 54 #include <machine/md_var.h> 55 #include <machine/intr_machdep.h> 56 #include <machine/globaldata.h> 57 #include <machine/msi_var.h> 58 59 #include <machine_base/isa/isa_intr.h> 60 #include <machine_base/icu/icu.h> 61 #include <machine_base/icu/icu_var.h> 62 #include <machine_base/apic/ioapic.h> 63 #include <machine_base/apic/ioapic_abi.h> 64 #include <machine_base/apic/ioapic_ipl.h> 65 #include <machine_base/apic/apicreg.h> 66 67 #include <dev/acpica5/acpi_sci_var.h> 68 69 #define IOAPIC_HWI_VECTORS IDT_HWI_VECTORS 70 71 extern inthand_t 72 IDTVEC(ioapic_intr0), 73 IDTVEC(ioapic_intr1), 74 IDTVEC(ioapic_intr2), 75 IDTVEC(ioapic_intr3), 76 IDTVEC(ioapic_intr4), 77 IDTVEC(ioapic_intr5), 78 IDTVEC(ioapic_intr6), 79 IDTVEC(ioapic_intr7), 80 IDTVEC(ioapic_intr8), 81 IDTVEC(ioapic_intr9), 82 IDTVEC(ioapic_intr10), 83 IDTVEC(ioapic_intr11), 84 IDTVEC(ioapic_intr12), 85 IDTVEC(ioapic_intr13), 86 IDTVEC(ioapic_intr14), 87 IDTVEC(ioapic_intr15), 88 IDTVEC(ioapic_intr16), 89 IDTVEC(ioapic_intr17), 90 IDTVEC(ioapic_intr18), 91 IDTVEC(ioapic_intr19), 92 IDTVEC(ioapic_intr20), 93 IDTVEC(ioapic_intr21), 94 IDTVEC(ioapic_intr22), 95 IDTVEC(ioapic_intr23), 96 IDTVEC(ioapic_intr24), 97 IDTVEC(ioapic_intr25), 98 IDTVEC(ioapic_intr26), 99 IDTVEC(ioapic_intr27), 100 IDTVEC(ioapic_intr28), 101 IDTVEC(ioapic_intr29), 102 IDTVEC(ioapic_intr30), 103 IDTVEC(ioapic_intr31), 104 IDTVEC(ioapic_intr32), 105 IDTVEC(ioapic_intr33), 106 IDTVEC(ioapic_intr34), 107 IDTVEC(ioapic_intr35), 108 IDTVEC(ioapic_intr36), 109 IDTVEC(ioapic_intr37), 110 IDTVEC(ioapic_intr38), 111 IDTVEC(ioapic_intr39), 112 IDTVEC(ioapic_intr40), 113 IDTVEC(ioapic_intr41), 114 IDTVEC(ioapic_intr42), 115 IDTVEC(ioapic_intr43), 116 IDTVEC(ioapic_intr44), 117 IDTVEC(ioapic_intr45), 118 IDTVEC(ioapic_intr46), 119 IDTVEC(ioapic_intr47), 120 IDTVEC(ioapic_intr48), 121 IDTVEC(ioapic_intr49), 122 IDTVEC(ioapic_intr50), 123 IDTVEC(ioapic_intr51), 124 IDTVEC(ioapic_intr52), 125 IDTVEC(ioapic_intr53), 126 IDTVEC(ioapic_intr54), 127 IDTVEC(ioapic_intr55), 128 IDTVEC(ioapic_intr56), 129 IDTVEC(ioapic_intr57), 130 IDTVEC(ioapic_intr58), 131 IDTVEC(ioapic_intr59), 132 IDTVEC(ioapic_intr60), 133 IDTVEC(ioapic_intr61), 134 IDTVEC(ioapic_intr62), 135 IDTVEC(ioapic_intr63), 136 IDTVEC(ioapic_intr64), 137 IDTVEC(ioapic_intr65), 138 IDTVEC(ioapic_intr66), 139 IDTVEC(ioapic_intr67), 140 IDTVEC(ioapic_intr68), 141 IDTVEC(ioapic_intr69), 142 IDTVEC(ioapic_intr70), 143 IDTVEC(ioapic_intr71), 144 IDTVEC(ioapic_intr72), 145 IDTVEC(ioapic_intr73), 146 IDTVEC(ioapic_intr74), 147 IDTVEC(ioapic_intr75), 148 IDTVEC(ioapic_intr76), 149 IDTVEC(ioapic_intr77), 150 IDTVEC(ioapic_intr78), 151 IDTVEC(ioapic_intr79), 152 IDTVEC(ioapic_intr80), 153 IDTVEC(ioapic_intr81), 154 IDTVEC(ioapic_intr82), 155 IDTVEC(ioapic_intr83), 156 IDTVEC(ioapic_intr84), 157 IDTVEC(ioapic_intr85), 158 IDTVEC(ioapic_intr86), 159 IDTVEC(ioapic_intr87), 160 IDTVEC(ioapic_intr88), 161 IDTVEC(ioapic_intr89), 162 IDTVEC(ioapic_intr90), 163 IDTVEC(ioapic_intr91), 164 IDTVEC(ioapic_intr92), 165 IDTVEC(ioapic_intr93), 166 IDTVEC(ioapic_intr94), 167 IDTVEC(ioapic_intr95), 168 IDTVEC(ioapic_intr96), 169 IDTVEC(ioapic_intr97), 170 IDTVEC(ioapic_intr98), 171 IDTVEC(ioapic_intr99), 172 IDTVEC(ioapic_intr100), 173 IDTVEC(ioapic_intr101), 174 IDTVEC(ioapic_intr102), 175 IDTVEC(ioapic_intr103), 176 IDTVEC(ioapic_intr104), 177 IDTVEC(ioapic_intr105), 178 IDTVEC(ioapic_intr106), 179 IDTVEC(ioapic_intr107), 180 IDTVEC(ioapic_intr108), 181 IDTVEC(ioapic_intr109), 182 IDTVEC(ioapic_intr110), 183 IDTVEC(ioapic_intr111), 184 IDTVEC(ioapic_intr112), 185 IDTVEC(ioapic_intr113), 186 IDTVEC(ioapic_intr114), 187 IDTVEC(ioapic_intr115), 188 IDTVEC(ioapic_intr116), 189 IDTVEC(ioapic_intr117), 190 IDTVEC(ioapic_intr118), 191 IDTVEC(ioapic_intr119), 192 IDTVEC(ioapic_intr120), 193 IDTVEC(ioapic_intr121), 194 IDTVEC(ioapic_intr122), 195 IDTVEC(ioapic_intr123), 196 IDTVEC(ioapic_intr124), 197 IDTVEC(ioapic_intr125), 198 IDTVEC(ioapic_intr126), 199 IDTVEC(ioapic_intr127), 200 IDTVEC(ioapic_intr128), 201 IDTVEC(ioapic_intr129), 202 IDTVEC(ioapic_intr130), 203 IDTVEC(ioapic_intr131), 204 IDTVEC(ioapic_intr132), 205 IDTVEC(ioapic_intr133), 206 IDTVEC(ioapic_intr134), 207 IDTVEC(ioapic_intr135), 208 IDTVEC(ioapic_intr136), 209 IDTVEC(ioapic_intr137), 210 IDTVEC(ioapic_intr138), 211 IDTVEC(ioapic_intr139), 212 IDTVEC(ioapic_intr140), 213 IDTVEC(ioapic_intr141), 214 IDTVEC(ioapic_intr142), 215 IDTVEC(ioapic_intr143), 216 IDTVEC(ioapic_intr144), 217 IDTVEC(ioapic_intr145), 218 IDTVEC(ioapic_intr146), 219 IDTVEC(ioapic_intr147), 220 IDTVEC(ioapic_intr148), 221 IDTVEC(ioapic_intr149), 222 IDTVEC(ioapic_intr150), 223 IDTVEC(ioapic_intr151), 224 IDTVEC(ioapic_intr152), 225 IDTVEC(ioapic_intr153), 226 IDTVEC(ioapic_intr154), 227 IDTVEC(ioapic_intr155), 228 IDTVEC(ioapic_intr156), 229 IDTVEC(ioapic_intr157), 230 IDTVEC(ioapic_intr158), 231 IDTVEC(ioapic_intr159), 232 IDTVEC(ioapic_intr160), 233 IDTVEC(ioapic_intr161), 234 IDTVEC(ioapic_intr162), 235 IDTVEC(ioapic_intr163), 236 IDTVEC(ioapic_intr164), 237 IDTVEC(ioapic_intr165), 238 IDTVEC(ioapic_intr166), 239 IDTVEC(ioapic_intr167), 240 IDTVEC(ioapic_intr168), 241 IDTVEC(ioapic_intr169), 242 IDTVEC(ioapic_intr170), 243 IDTVEC(ioapic_intr171), 244 IDTVEC(ioapic_intr172), 245 IDTVEC(ioapic_intr173), 246 IDTVEC(ioapic_intr174), 247 IDTVEC(ioapic_intr175), 248 IDTVEC(ioapic_intr176), 249 IDTVEC(ioapic_intr177), 250 IDTVEC(ioapic_intr178), 251 IDTVEC(ioapic_intr179), 252 IDTVEC(ioapic_intr180), 253 IDTVEC(ioapic_intr181), 254 IDTVEC(ioapic_intr182), 255 IDTVEC(ioapic_intr183), 256 IDTVEC(ioapic_intr184), 257 IDTVEC(ioapic_intr185), 258 IDTVEC(ioapic_intr186), 259 IDTVEC(ioapic_intr187), 260 IDTVEC(ioapic_intr188), 261 IDTVEC(ioapic_intr189), 262 IDTVEC(ioapic_intr190), 263 IDTVEC(ioapic_intr191); 264 265 static inthand_t *ioapic_intr[IOAPIC_HWI_VECTORS] = { 266 &IDTVEC(ioapic_intr0), 267 &IDTVEC(ioapic_intr1), 268 &IDTVEC(ioapic_intr2), 269 &IDTVEC(ioapic_intr3), 270 &IDTVEC(ioapic_intr4), 271 &IDTVEC(ioapic_intr5), 272 &IDTVEC(ioapic_intr6), 273 &IDTVEC(ioapic_intr7), 274 &IDTVEC(ioapic_intr8), 275 &IDTVEC(ioapic_intr9), 276 &IDTVEC(ioapic_intr10), 277 &IDTVEC(ioapic_intr11), 278 &IDTVEC(ioapic_intr12), 279 &IDTVEC(ioapic_intr13), 280 &IDTVEC(ioapic_intr14), 281 &IDTVEC(ioapic_intr15), 282 &IDTVEC(ioapic_intr16), 283 &IDTVEC(ioapic_intr17), 284 &IDTVEC(ioapic_intr18), 285 &IDTVEC(ioapic_intr19), 286 &IDTVEC(ioapic_intr20), 287 &IDTVEC(ioapic_intr21), 288 &IDTVEC(ioapic_intr22), 289 &IDTVEC(ioapic_intr23), 290 &IDTVEC(ioapic_intr24), 291 &IDTVEC(ioapic_intr25), 292 &IDTVEC(ioapic_intr26), 293 &IDTVEC(ioapic_intr27), 294 &IDTVEC(ioapic_intr28), 295 &IDTVEC(ioapic_intr29), 296 &IDTVEC(ioapic_intr30), 297 &IDTVEC(ioapic_intr31), 298 &IDTVEC(ioapic_intr32), 299 &IDTVEC(ioapic_intr33), 300 &IDTVEC(ioapic_intr34), 301 &IDTVEC(ioapic_intr35), 302 &IDTVEC(ioapic_intr36), 303 &IDTVEC(ioapic_intr37), 304 &IDTVEC(ioapic_intr38), 305 &IDTVEC(ioapic_intr39), 306 &IDTVEC(ioapic_intr40), 307 &IDTVEC(ioapic_intr41), 308 &IDTVEC(ioapic_intr42), 309 &IDTVEC(ioapic_intr43), 310 &IDTVEC(ioapic_intr44), 311 &IDTVEC(ioapic_intr45), 312 &IDTVEC(ioapic_intr46), 313 &IDTVEC(ioapic_intr47), 314 &IDTVEC(ioapic_intr48), 315 &IDTVEC(ioapic_intr49), 316 &IDTVEC(ioapic_intr50), 317 &IDTVEC(ioapic_intr51), 318 &IDTVEC(ioapic_intr52), 319 &IDTVEC(ioapic_intr53), 320 &IDTVEC(ioapic_intr54), 321 &IDTVEC(ioapic_intr55), 322 &IDTVEC(ioapic_intr56), 323 &IDTVEC(ioapic_intr57), 324 &IDTVEC(ioapic_intr58), 325 &IDTVEC(ioapic_intr59), 326 &IDTVEC(ioapic_intr60), 327 &IDTVEC(ioapic_intr61), 328 &IDTVEC(ioapic_intr62), 329 &IDTVEC(ioapic_intr63), 330 &IDTVEC(ioapic_intr64), 331 &IDTVEC(ioapic_intr65), 332 &IDTVEC(ioapic_intr66), 333 &IDTVEC(ioapic_intr67), 334 &IDTVEC(ioapic_intr68), 335 &IDTVEC(ioapic_intr69), 336 &IDTVEC(ioapic_intr70), 337 &IDTVEC(ioapic_intr71), 338 &IDTVEC(ioapic_intr72), 339 &IDTVEC(ioapic_intr73), 340 &IDTVEC(ioapic_intr74), 341 &IDTVEC(ioapic_intr75), 342 &IDTVEC(ioapic_intr76), 343 &IDTVEC(ioapic_intr77), 344 &IDTVEC(ioapic_intr78), 345 &IDTVEC(ioapic_intr79), 346 &IDTVEC(ioapic_intr80), 347 &IDTVEC(ioapic_intr81), 348 &IDTVEC(ioapic_intr82), 349 &IDTVEC(ioapic_intr83), 350 &IDTVEC(ioapic_intr84), 351 &IDTVEC(ioapic_intr85), 352 &IDTVEC(ioapic_intr86), 353 &IDTVEC(ioapic_intr87), 354 &IDTVEC(ioapic_intr88), 355 &IDTVEC(ioapic_intr89), 356 &IDTVEC(ioapic_intr90), 357 &IDTVEC(ioapic_intr91), 358 &IDTVEC(ioapic_intr92), 359 &IDTVEC(ioapic_intr93), 360 &IDTVEC(ioapic_intr94), 361 &IDTVEC(ioapic_intr95), 362 &IDTVEC(ioapic_intr96), 363 &IDTVEC(ioapic_intr97), 364 &IDTVEC(ioapic_intr98), 365 &IDTVEC(ioapic_intr99), 366 &IDTVEC(ioapic_intr100), 367 &IDTVEC(ioapic_intr101), 368 &IDTVEC(ioapic_intr102), 369 &IDTVEC(ioapic_intr103), 370 &IDTVEC(ioapic_intr104), 371 &IDTVEC(ioapic_intr105), 372 &IDTVEC(ioapic_intr106), 373 &IDTVEC(ioapic_intr107), 374 &IDTVEC(ioapic_intr108), 375 &IDTVEC(ioapic_intr109), 376 &IDTVEC(ioapic_intr110), 377 &IDTVEC(ioapic_intr111), 378 &IDTVEC(ioapic_intr112), 379 &IDTVEC(ioapic_intr113), 380 &IDTVEC(ioapic_intr114), 381 &IDTVEC(ioapic_intr115), 382 &IDTVEC(ioapic_intr116), 383 &IDTVEC(ioapic_intr117), 384 &IDTVEC(ioapic_intr118), 385 &IDTVEC(ioapic_intr119), 386 &IDTVEC(ioapic_intr120), 387 &IDTVEC(ioapic_intr121), 388 &IDTVEC(ioapic_intr122), 389 &IDTVEC(ioapic_intr123), 390 &IDTVEC(ioapic_intr124), 391 &IDTVEC(ioapic_intr125), 392 &IDTVEC(ioapic_intr126), 393 &IDTVEC(ioapic_intr127), 394 &IDTVEC(ioapic_intr128), 395 &IDTVEC(ioapic_intr129), 396 &IDTVEC(ioapic_intr130), 397 &IDTVEC(ioapic_intr131), 398 &IDTVEC(ioapic_intr132), 399 &IDTVEC(ioapic_intr133), 400 &IDTVEC(ioapic_intr134), 401 &IDTVEC(ioapic_intr135), 402 &IDTVEC(ioapic_intr136), 403 &IDTVEC(ioapic_intr137), 404 &IDTVEC(ioapic_intr138), 405 &IDTVEC(ioapic_intr139), 406 &IDTVEC(ioapic_intr140), 407 &IDTVEC(ioapic_intr141), 408 &IDTVEC(ioapic_intr142), 409 &IDTVEC(ioapic_intr143), 410 &IDTVEC(ioapic_intr144), 411 &IDTVEC(ioapic_intr145), 412 &IDTVEC(ioapic_intr146), 413 &IDTVEC(ioapic_intr147), 414 &IDTVEC(ioapic_intr148), 415 &IDTVEC(ioapic_intr149), 416 &IDTVEC(ioapic_intr150), 417 &IDTVEC(ioapic_intr151), 418 &IDTVEC(ioapic_intr152), 419 &IDTVEC(ioapic_intr153), 420 &IDTVEC(ioapic_intr154), 421 &IDTVEC(ioapic_intr155), 422 &IDTVEC(ioapic_intr156), 423 &IDTVEC(ioapic_intr157), 424 &IDTVEC(ioapic_intr158), 425 &IDTVEC(ioapic_intr159), 426 &IDTVEC(ioapic_intr160), 427 &IDTVEC(ioapic_intr161), 428 &IDTVEC(ioapic_intr162), 429 &IDTVEC(ioapic_intr163), 430 &IDTVEC(ioapic_intr164), 431 &IDTVEC(ioapic_intr165), 432 &IDTVEC(ioapic_intr166), 433 &IDTVEC(ioapic_intr167), 434 &IDTVEC(ioapic_intr168), 435 &IDTVEC(ioapic_intr169), 436 &IDTVEC(ioapic_intr170), 437 &IDTVEC(ioapic_intr171), 438 &IDTVEC(ioapic_intr172), 439 &IDTVEC(ioapic_intr173), 440 &IDTVEC(ioapic_intr174), 441 &IDTVEC(ioapic_intr175), 442 &IDTVEC(ioapic_intr176), 443 &IDTVEC(ioapic_intr177), 444 &IDTVEC(ioapic_intr178), 445 &IDTVEC(ioapic_intr179), 446 &IDTVEC(ioapic_intr180), 447 &IDTVEC(ioapic_intr181), 448 &IDTVEC(ioapic_intr182), 449 &IDTVEC(ioapic_intr183), 450 &IDTVEC(ioapic_intr184), 451 &IDTVEC(ioapic_intr185), 452 &IDTVEC(ioapic_intr186), 453 &IDTVEC(ioapic_intr187), 454 &IDTVEC(ioapic_intr188), 455 &IDTVEC(ioapic_intr189), 456 &IDTVEC(ioapic_intr190), 457 &IDTVEC(ioapic_intr191) 458 }; 459 460 #define IOAPIC_HWI_SYSCALL (IDT_OFFSET_SYSCALL - IDT_OFFSET) 461 462 static struct ioapic_irqmap { 463 int im_type; /* IOAPIC_IMT_ */ 464 enum intr_trigger im_trig; 465 enum intr_polarity im_pola; 466 int im_gsi; 467 int im_msi_base; 468 uint32_t im_flags; /* IOAPIC_IMF_ */ 469 } ioapic_irqmaps[MAXCPU][IOAPIC_HWI_VECTORS]; 470 471 static struct lwkt_token ioapic_irqmap_tok = 472 LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token); 473 474 #define IOAPIC_IMT_UNUSED 0 475 #define IOAPIC_IMT_RESERVED 1 476 #define IOAPIC_IMT_LINE 2 477 #define IOAPIC_IMT_SYSCALL 3 478 #define IOAPIC_IMT_MSI 4 479 480 #define IOAPIC_IMT_ISHWI(map) ((map)->im_type != IOAPIC_IMT_RESERVED && \ 481 (map)->im_type != IOAPIC_IMT_SYSCALL) 482 483 #define IOAPIC_IMF_CONF 0x1 484 485 extern void IOAPIC_INTREN(int); 486 extern void IOAPIC_INTRDIS(int); 487 488 extern int imcr_present; 489 490 static void ioapic_abi_intr_enable(int); 491 static void ioapic_abi_intr_disable(int); 492 static void ioapic_abi_intr_setup(int, int); 493 static void ioapic_abi_intr_teardown(int); 494 static void ioapic_abi_intr_config(int, 495 enum intr_trigger, enum intr_polarity); 496 static int ioapic_abi_intr_cpuid(int); 497 498 static int ioapic_abi_msi_alloc(int [], int, int); 499 static void ioapic_abi_msi_release(const int [], int, int); 500 static void ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int); 501 502 static void ioapic_abi_finalize(void); 503 static void ioapic_abi_cleanup(void); 504 static void ioapic_abi_setdefault(void); 505 static void ioapic_abi_stabilize(void); 506 static void ioapic_abi_initmap(void); 507 static void ioapic_abi_rman_setup(struct rman *); 508 509 static int ioapic_abi_gsi_cpuid(int, int); 510 511 struct machintr_abi MachIntrABI_IOAPIC = { 512 MACHINTR_IOAPIC, 513 .intr_disable = ioapic_abi_intr_disable, 514 .intr_enable = ioapic_abi_intr_enable, 515 .intr_setup = ioapic_abi_intr_setup, 516 .intr_teardown = ioapic_abi_intr_teardown, 517 .intr_config = ioapic_abi_intr_config, 518 .intr_cpuid = ioapic_abi_intr_cpuid, 519 520 .msi_alloc = ioapic_abi_msi_alloc, 521 .msi_release = ioapic_abi_msi_release, 522 .msi_map = ioapic_abi_msi_map, 523 524 .finalize = ioapic_abi_finalize, 525 .cleanup = ioapic_abi_cleanup, 526 .setdefault = ioapic_abi_setdefault, 527 .stabilize = ioapic_abi_stabilize, 528 .initmap = ioapic_abi_initmap, 529 .rman_setup = ioapic_abi_rman_setup 530 }; 531 532 static int ioapic_abi_extint_irq = -1; 533 static int ioapic_abi_line_irq_max; 534 static int ioapic_abi_gsi_balance; 535 static int ioapic_abi_msi_start; /* NOTE: for testing only */ 536 537 struct ioapic_irqinfo ioapic_irqs[IOAPIC_HWI_VECTORS]; 538 539 static void 540 ioapic_abi_intr_enable(int irq) 541 { 542 const struct ioapic_irqmap *map; 543 544 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 545 ("ioapic enable, invalid irq %d\n", irq)); 546 547 map = &ioapic_irqmaps[mycpuid][irq]; 548 KASSERT(IOAPIC_IMT_ISHWI(map), 549 ("ioapic enable, not hwi irq %d, type %d, cpu%d\n", 550 irq, map->im_type, mycpuid)); 551 if (map->im_type != IOAPIC_IMT_LINE) 552 return; 553 554 IOAPIC_INTREN(irq); 555 } 556 557 static void 558 ioapic_abi_intr_disable(int irq) 559 { 560 const struct ioapic_irqmap *map; 561 562 KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS, 563 ("ioapic disable, invalid irq %d\n", irq)); 564 565 map = &ioapic_irqmaps[mycpuid][irq]; 566 KASSERT(IOAPIC_IMT_ISHWI(map), 567 ("ioapic disable, not hwi irq %d, type %d, cpu%d\n", 568 irq, map->im_type, mycpuid)); 569 if (map->im_type != IOAPIC_IMT_LINE) 570 return; 571 572 IOAPIC_INTRDIS(irq); 573 } 574 575 static void 576 ioapic_abi_finalize(void) 577 { 578 KKASSERT(MachIntrABI.type == MACHINTR_IOAPIC); 579 KKASSERT(ioapic_enable); 580 581 /* 582 * If an IMCR is present, program bit 0 to disconnect the 8259 583 * from the BSP. 584 */ 585 if (imcr_present) { 586 outb(0x22, 0x70); /* select IMCR */ 587 outb(0x23, 0x01); /* disconnect 8259 */ 588 } 589 } 590 591 /* 592 * This routine is called after physical interrupts are enabled but before 593 * the critical section is released. We need to clean out any interrupts 594 * that had already been posted to the cpu. 595 */ 596 static void 597 ioapic_abi_cleanup(void) 598 { 599 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending)); 600 } 601 602 /* Must never be called */ 603 static void 604 ioapic_abi_stabilize(void) 605 { 606 panic("ioapic_stabilize is called\n"); 607 } 608 609 static void 610 ioapic_abi_intr_setup(int intr, int flags) 611 { 612 const struct ioapic_irqmap *map; 613 int vector, select; 614 uint32_t value; 615 register_t ef; 616 617 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 618 ("ioapic setup, invalid irq %d\n", intr)); 619 620 map = &ioapic_irqmaps[mycpuid][intr]; 621 KASSERT(IOAPIC_IMT_ISHWI(map), 622 ("ioapic setup, not hwi irq %d, type %d, cpu%d", 623 intr, map->im_type, mycpuid)); 624 if (map->im_type != IOAPIC_IMT_LINE) 625 return; 626 627 KASSERT(ioapic_irqs[intr].io_addr != NULL, 628 ("ioapic setup, no GSI information, irq %d\n", intr)); 629 630 ef = read_rflags(); 631 cpu_disable_intr(); 632 633 vector = IDT_OFFSET + intr; 634 635 /* 636 * Now reprogram the vector in the IO APIC. In order to avoid 637 * losing an EOI for a level interrupt, which is vector based, 638 * make sure that the IO APIC is programmed for edge-triggering 639 * first, then reprogrammed with the new vector. This should 640 * clear the IRR bit. 641 */ 642 imen_lock(); 643 644 select = ioapic_irqs[intr].io_idx; 645 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 646 value |= IOART_INTMSET; 647 648 ioapic_write(ioapic_irqs[intr].io_addr, select, 649 (value & ~APIC_TRIGMOD_MASK)); 650 ioapic_write(ioapic_irqs[intr].io_addr, select, 651 (value & ~IOART_INTVEC) | vector); 652 653 imen_unlock(); 654 655 IOAPIC_INTREN(intr); 656 657 write_rflags(ef); 658 } 659 660 static void 661 ioapic_abi_intr_teardown(int intr) 662 { 663 const struct ioapic_irqmap *map; 664 int vector, select; 665 uint32_t value; 666 register_t ef; 667 668 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 669 ("ioapic teardown, invalid irq %d\n", intr)); 670 671 map = &ioapic_irqmaps[mycpuid][intr]; 672 KASSERT(IOAPIC_IMT_ISHWI(map), 673 ("ioapic teardown, not hwi irq %d, type %d, cpu%d", 674 intr, map->im_type, mycpuid)); 675 if (map->im_type != IOAPIC_IMT_LINE) 676 return; 677 678 KASSERT(ioapic_irqs[intr].io_addr != NULL, 679 ("ioapic teardown, no GSI information, irq %d\n", intr)); 680 681 ef = read_rflags(); 682 cpu_disable_intr(); 683 684 /* 685 * Teardown an interrupt vector. The vector should already be 686 * installed in the cpu's IDT, but make sure. 687 */ 688 IOAPIC_INTRDIS(intr); 689 690 vector = IDT_OFFSET + intr; 691 692 /* 693 * In order to avoid losing an EOI for a level interrupt, which 694 * is vector based, make sure that the IO APIC is programmed for 695 * edge-triggering first, then reprogrammed with the new vector. 696 * This should clear the IRR bit. 697 */ 698 imen_lock(); 699 700 select = ioapic_irqs[intr].io_idx; 701 value = ioapic_read(ioapic_irqs[intr].io_addr, select); 702 703 ioapic_write(ioapic_irqs[intr].io_addr, select, 704 (value & ~APIC_TRIGMOD_MASK)); 705 ioapic_write(ioapic_irqs[intr].io_addr, select, 706 (value & ~IOART_INTVEC) | vector); 707 708 imen_unlock(); 709 710 write_rflags(ef); 711 } 712 713 static void 714 ioapic_abi_setdefault(void) 715 { 716 int intr; 717 718 for (intr = 0; intr < IOAPIC_HWI_VECTORS; ++intr) { 719 if (intr == IOAPIC_HWI_SYSCALL) 720 continue; 721 setidt_global(IDT_OFFSET + intr, ioapic_intr[intr], 722 SDT_SYSIGT, SEL_KPL, 0); 723 } 724 } 725 726 static void 727 ioapic_abi_initmap(void) 728 { 729 int cpu; 730 731 kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance); 732 733 kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start); 734 ioapic_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */ 735 736 /* 737 * NOTE: ncpus is not ready yet 738 */ 739 for (cpu = 0; cpu < MAXCPU; ++cpu) { 740 int i; 741 742 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 743 ioapic_irqmaps[cpu][i].im_gsi = -1; 744 ioapic_irqmaps[cpu][i].im_msi_base = -1; 745 } 746 ioapic_irqmaps[cpu][IOAPIC_HWI_SYSCALL].im_type = 747 IOAPIC_IMT_SYSCALL; 748 } 749 } 750 751 void 752 ioapic_abi_set_irqmap(int irq, int gsi, enum intr_trigger trig, 753 enum intr_polarity pola) 754 { 755 struct ioapic_irqinfo *info; 756 struct ioapic_irqmap *map; 757 void *ioaddr; 758 int pin, cpuid; 759 760 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 761 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 762 763 KKASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS); 764 if (irq > ioapic_abi_line_irq_max) 765 ioapic_abi_line_irq_max = irq; 766 767 cpuid = ioapic_abi_gsi_cpuid(irq, gsi); 768 769 map = &ioapic_irqmaps[cpuid][irq]; 770 771 KKASSERT(map->im_type == IOAPIC_IMT_UNUSED); 772 map->im_type = IOAPIC_IMT_LINE; 773 774 map->im_gsi = gsi; 775 map->im_trig = trig; 776 map->im_pola = pola; 777 778 if (bootverbose) { 779 kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n", 780 irq, map->im_gsi, 781 intr_str_trigger(map->im_trig), 782 intr_str_polarity(map->im_pola)); 783 } 784 785 pin = ioapic_gsi_pin(map->im_gsi); 786 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 787 788 info = &ioapic_irqs[irq]; 789 790 imen_lock(); 791 792 info->io_addr = ioaddr; 793 info->io_idx = IOAPIC_REDTBL + (2 * pin); 794 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 795 if (map->im_trig == INTR_TRIGGER_LEVEL) 796 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 797 798 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 799 map->im_trig, map->im_pola, cpuid); 800 801 imen_unlock(); 802 } 803 804 void 805 ioapic_abi_fixup_irqmap(void) 806 { 807 int cpu; 808 809 for (cpu = 0; cpu < ncpus; ++cpu) { 810 int i; 811 812 for (i = 0; i < ISA_IRQ_CNT; ++i) { 813 struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][i]; 814 815 if (map->im_type == IOAPIC_IMT_UNUSED) { 816 map->im_type = IOAPIC_IMT_RESERVED; 817 if (bootverbose) { 818 kprintf("IOAPIC: " 819 "cpu%d irq %d reserved\n", cpu, i); 820 } 821 } 822 } 823 } 824 825 ioapic_abi_line_irq_max += 1; 826 if (bootverbose) 827 kprintf("IOAPIC: line irq max %d\n", ioapic_abi_line_irq_max); 828 } 829 830 int 831 ioapic_abi_find_gsi(int gsi, enum intr_trigger trig, enum intr_polarity pola) 832 { 833 int cpu; 834 835 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 836 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 837 838 for (cpu = 0; cpu < ncpus; ++cpu) { 839 int irq; 840 841 for (irq = 0; irq < ioapic_abi_line_irq_max; ++irq) { 842 const struct ioapic_irqmap *map = 843 &ioapic_irqmaps[cpu][irq]; 844 845 if (map->im_gsi == gsi) { 846 KKASSERT(map->im_type == IOAPIC_IMT_LINE); 847 848 if (map->im_flags & IOAPIC_IMF_CONF) { 849 if (map->im_trig != trig || 850 map->im_pola != pola) 851 return -1; 852 } 853 return irq; 854 } 855 } 856 } 857 return -1; 858 } 859 860 int 861 ioapic_abi_find_irq(int irq, enum intr_trigger trig, enum intr_polarity pola) 862 { 863 int cpu; 864 865 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 866 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 867 868 if (irq < 0 || irq >= ioapic_abi_line_irq_max) 869 return -1; 870 871 for (cpu = 0; cpu < ncpus; ++cpu) { 872 const struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][irq]; 873 874 if (map->im_type == IOAPIC_IMT_LINE) { 875 if (map->im_flags & IOAPIC_IMF_CONF) { 876 if (map->im_trig != trig || 877 map->im_pola != pola) 878 return -1; 879 } 880 return irq; 881 } 882 } 883 return -1; 884 } 885 886 static void 887 ioapic_abi_intr_config(int irq, enum intr_trigger trig, enum intr_polarity pola) 888 { 889 struct ioapic_irqinfo *info; 890 struct ioapic_irqmap *map = NULL; 891 void *ioaddr; 892 int pin, cpuid; 893 894 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 895 KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW); 896 897 KKASSERT(irq >= 0 && irq < ioapic_abi_line_irq_max); 898 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 899 map = &ioapic_irqmaps[cpuid][irq]; 900 if (map->im_type == IOAPIC_IMT_LINE) 901 break; 902 } 903 KKASSERT(cpuid < ncpus); 904 905 #ifdef notyet 906 if (map->im_flags & IOAPIC_IMF_CONF) { 907 if (trig != map->im_trig) { 908 panic("ioapic_intr_config: trig %s -> %s\n", 909 intr_str_trigger(map->im_trig), 910 intr_str_trigger(trig)); 911 } 912 if (pola != map->im_pola) { 913 panic("ioapic_intr_config: pola %s -> %s\n", 914 intr_str_polarity(map->im_pola), 915 intr_str_polarity(pola)); 916 } 917 return; 918 } 919 #endif 920 map->im_flags |= IOAPIC_IMF_CONF; 921 922 if (trig == map->im_trig && pola == map->im_pola) 923 return; 924 925 if (bootverbose) { 926 kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n", 927 irq, map->im_gsi, 928 intr_str_trigger(map->im_trig), 929 intr_str_polarity(map->im_pola), 930 intr_str_trigger(trig), 931 intr_str_polarity(pola)); 932 } 933 map->im_trig = trig; 934 map->im_pola = pola; 935 936 pin = ioapic_gsi_pin(map->im_gsi); 937 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 938 939 info = &ioapic_irqs[irq]; 940 941 imen_lock(); 942 943 info->io_flags &= ~IOAPIC_IRQI_FLAG_LEVEL; 944 if (map->im_trig == INTR_TRIGGER_LEVEL) 945 info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL; 946 947 ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq, 948 map->im_trig, map->im_pola, cpuid); 949 950 imen_unlock(); 951 } 952 953 int 954 ioapic_abi_extint_irqmap(int irq) 955 { 956 struct ioapic_irqinfo *info; 957 struct ioapic_irqmap *map; 958 void *ioaddr; 959 int pin, error, vec; 960 961 /* XXX only irq0 is allowed */ 962 KKASSERT(irq == 0); 963 964 vec = IDT_OFFSET + irq; 965 966 if (ioapic_abi_extint_irq == irq) 967 return 0; 968 else if (ioapic_abi_extint_irq >= 0) 969 return EEXIST; 970 971 error = icu_ioapic_extint(irq, vec); 972 if (error) 973 return error; 974 975 /* ExtINT is always targeted to cpu0 */ 976 map = &ioapic_irqmaps[0][irq]; 977 978 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED || 979 map->im_type == IOAPIC_IMT_LINE); 980 if (map->im_type == IOAPIC_IMT_LINE) { 981 if (map->im_flags & IOAPIC_IMF_CONF) 982 return EEXIST; 983 } 984 ioapic_abi_extint_irq = irq; 985 986 map->im_type = IOAPIC_IMT_LINE; 987 map->im_trig = INTR_TRIGGER_EDGE; 988 map->im_pola = INTR_POLARITY_HIGH; 989 map->im_flags = IOAPIC_IMF_CONF; 990 991 map->im_gsi = ioapic_extpin_gsi(); 992 KKASSERT(map->im_gsi >= 0); 993 994 if (bootverbose) { 995 kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n", 996 irq, map->im_gsi, 997 intr_str_trigger(map->im_trig), 998 intr_str_polarity(map->im_pola)); 999 } 1000 1001 pin = ioapic_gsi_pin(map->im_gsi); 1002 ioaddr = ioapic_gsi_ioaddr(map->im_gsi); 1003 1004 info = &ioapic_irqs[irq]; 1005 1006 imen_lock(); 1007 1008 info->io_addr = ioaddr; 1009 info->io_idx = IOAPIC_REDTBL + (2 * pin); 1010 info->io_flags = IOAPIC_IRQI_FLAG_MASKED; 1011 1012 ioapic_extpin_setup(ioaddr, pin, vec); 1013 1014 imen_unlock(); 1015 1016 return 0; 1017 } 1018 1019 static int 1020 ioapic_abi_intr_cpuid(int irq) 1021 { 1022 const struct ioapic_irqmap *map = NULL; 1023 int cpuid; 1024 1025 KKASSERT(irq >= 0 && irq < ioapic_abi_line_irq_max); 1026 1027 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1028 map = &ioapic_irqmaps[cpuid][irq]; 1029 if (map->im_type == IOAPIC_IMT_LINE) 1030 return cpuid; 1031 } 1032 1033 /* XXX some drivers tries to peek at reserved IRQs */ 1034 for (cpuid = 0; cpuid < ncpus; ++cpuid) { 1035 map = &ioapic_irqmaps[cpuid][irq]; 1036 KKASSERT(map->im_type == IOAPIC_IMT_RESERVED); 1037 } 1038 return 0; 1039 } 1040 1041 static int 1042 ioapic_abi_gsi_cpuid(int irq, int gsi) 1043 { 1044 char envpath[32]; 1045 int cpuid = -1; 1046 1047 KKASSERT(gsi >= 0); 1048 1049 if (irq == 0 || gsi == 0) { 1050 if (bootverbose) { 1051 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n", 1052 irq, gsi); 1053 } 1054 return 0; 1055 } 1056 1057 if (irq == acpi_sci_irqno()) { 1058 if (bootverbose) { 1059 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n", 1060 irq, gsi); 1061 } 1062 return 0; 1063 } 1064 1065 ksnprintf(envpath, sizeof(envpath), "hw.ioapic.gsi.%d.cpu", gsi); 1066 kgetenv_int(envpath, &cpuid); 1067 1068 if (cpuid < 0) { 1069 if (!ioapic_abi_gsi_balance) { 1070 if (bootverbose) { 1071 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 " 1072 "(fixed)\n", irq, gsi); 1073 } 1074 return 0; 1075 } 1076 1077 cpuid = gsi % ncpus; 1078 if (bootverbose) { 1079 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n", 1080 irq, gsi, cpuid); 1081 } 1082 } else if (cpuid >= ncpus) { 1083 cpuid = ncpus - 1; 1084 if (bootverbose) { 1085 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n", 1086 irq, gsi, cpuid); 1087 } 1088 } else { 1089 if (bootverbose) { 1090 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n", 1091 irq, gsi, cpuid); 1092 } 1093 } 1094 return cpuid; 1095 } 1096 1097 static void 1098 ioapic_abi_rman_setup(struct rman *rm) 1099 { 1100 int start, end, i; 1101 1102 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU, 1103 ("invalid rman cpuid %d", rm->rm_cpuid)); 1104 1105 start = end = -1; 1106 for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) { 1107 const struct ioapic_irqmap *map = 1108 &ioapic_irqmaps[rm->rm_cpuid][i]; 1109 1110 if (start < 0) { 1111 if (IOAPIC_IMT_ISHWI(map)) 1112 start = end = i; 1113 } else { 1114 if (IOAPIC_IMT_ISHWI(map)) { 1115 end = i; 1116 } else { 1117 KKASSERT(end >= 0); 1118 if (bootverbose) { 1119 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1120 rm->rm_cpuid, start, end); 1121 } 1122 if (rman_manage_region(rm, start, end)) { 1123 panic("rman_manage_region" 1124 "(cpu%d %d - %d)", rm->rm_cpuid, 1125 start, end); 1126 } 1127 start = end = -1; 1128 } 1129 } 1130 } 1131 if (start >= 0) { 1132 KKASSERT(end >= 0); 1133 if (bootverbose) { 1134 kprintf("IOAPIC: rman cpu%d %d - %d\n", 1135 rm->rm_cpuid, start, end); 1136 } 1137 if (rman_manage_region(rm, start, end)) { 1138 panic("rman_manage_region(cpu%d %d - %d)", 1139 rm->rm_cpuid, start, end); 1140 } 1141 } 1142 } 1143 1144 static int 1145 ioapic_abi_msi_alloc(int intrs[], int count, int cpuid) 1146 { 1147 int i, error; 1148 1149 KASSERT(cpuid >= 0 && cpuid < ncpus, 1150 ("invalid cpuid %d", cpuid)); 1151 1152 KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count)); 1153 KASSERT((count & (count - 1)) == 0, 1154 ("count %d is not power of 2\n", count)); 1155 1156 lwkt_gettoken(&ioapic_irqmap_tok); 1157 1158 /* 1159 * NOTE: 1160 * Since IDT_OFFSET is 32, which is the maximum valid 'count', 1161 * we do not need to find out the first properly aligned 1162 * interrupt vector. 1163 */ 1164 1165 error = EMSGSIZE; 1166 for (i = ioapic_abi_msi_start; i < IOAPIC_HWI_VECTORS; i += count) { 1167 int j; 1168 1169 if (ioapic_irqmaps[cpuid][i].im_type != IOAPIC_IMT_UNUSED) 1170 continue; 1171 1172 for (j = 1; j < count; ++j) { 1173 if (ioapic_irqmaps[cpuid][i + j].im_type != 1174 IOAPIC_IMT_UNUSED) 1175 break; 1176 } 1177 if (j != count) 1178 continue; 1179 1180 for (j = 0; j < count; ++j) { 1181 struct ioapic_irqmap *map; 1182 int intr = i + j; 1183 1184 map = &ioapic_irqmaps[cpuid][intr]; 1185 KASSERT(map->im_msi_base < 0, 1186 ("intr %d, stale MSI-base %d\n", 1187 intr, map->im_msi_base)); 1188 1189 map->im_type = IOAPIC_IMT_MSI; 1190 map->im_msi_base = i; 1191 1192 intrs[j] = intr; 1193 msi_setup(intr, cpuid); 1194 1195 if (bootverbose) { 1196 kprintf("alloc MSI intr %d on cpu%d\n", 1197 intr, cpuid); 1198 } 1199 } 1200 error = 0; 1201 break; 1202 } 1203 1204 lwkt_reltoken(&ioapic_irqmap_tok); 1205 1206 return error; 1207 } 1208 1209 static void 1210 ioapic_abi_msi_release(const int intrs[], int count, int cpuid) 1211 { 1212 int i, msi_base = -1, intr_next = -1, mask; 1213 1214 KASSERT(cpuid >= 0 && cpuid < ncpus, 1215 ("invalid cpuid %d", cpuid)); 1216 1217 KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count)); 1218 1219 mask = count - 1; 1220 KASSERT((count & mask) == 0, ("count %d is not power of 2\n", count)); 1221 1222 lwkt_gettoken(&ioapic_irqmap_tok); 1223 1224 for (i = 0; i < count; ++i) { 1225 struct ioapic_irqmap *map; 1226 int intr = intrs[i]; 1227 1228 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1229 ("invalid intr %d\n", intr)); 1230 1231 map = &ioapic_irqmaps[cpuid][intr]; 1232 KASSERT(map->im_type == IOAPIC_IMT_MSI, 1233 ("try release non-MSI intr %d, type %d\n", 1234 intr, map->im_type)); 1235 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1236 ("intr %d, invalid MSI-base %d\n", intr, map->im_msi_base)); 1237 KASSERT((map->im_msi_base & mask) == 0, 1238 ("intr %d, MSI-base %d is not proper aligned %d\n", 1239 intr, map->im_msi_base, count)); 1240 1241 if (msi_base < 0) { 1242 msi_base = map->im_msi_base; 1243 } else { 1244 KASSERT(map->im_msi_base == msi_base, 1245 ("intr %d, inconsistent MSI-base, " 1246 "was %d, now %d\n", 1247 intr, msi_base, map->im_msi_base)); 1248 } 1249 1250 if (intr_next < intr) 1251 intr_next = intr; 1252 1253 map->im_type = IOAPIC_IMT_UNUSED; 1254 map->im_msi_base = -1; 1255 1256 if (bootverbose) 1257 kprintf("release MSI intr %d on cpu%d\n", intr, cpuid); 1258 } 1259 1260 KKASSERT(intr_next > 0); 1261 KKASSERT(msi_base >= 0); 1262 1263 ++intr_next; 1264 if (intr_next < IOAPIC_HWI_VECTORS) { 1265 const struct ioapic_irqmap *map = 1266 &ioapic_irqmaps[cpuid][intr_next]; 1267 1268 if (map->im_type == IOAPIC_IMT_MSI) { 1269 KASSERT(map->im_msi_base != msi_base, 1270 ("more than %d MSI was allocated\n", count)); 1271 } 1272 } 1273 1274 lwkt_reltoken(&ioapic_irqmap_tok); 1275 } 1276 1277 static void 1278 ioapic_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid) 1279 { 1280 const struct ioapic_irqmap *map; 1281 1282 KASSERT(cpuid >= 0 && cpuid < ncpus, 1283 ("invalid cpuid %d", cpuid)); 1284 1285 KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS, 1286 ("invalid intr %d\n", intr)); 1287 1288 lwkt_gettoken(&ioapic_irqmap_tok); 1289 1290 map = &ioapic_irqmaps[cpuid][intr]; 1291 KASSERT(map->im_type == IOAPIC_IMT_MSI, 1292 ("try map non-MSI intr %d, type %d\n", intr, map->im_type)); 1293 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 1294 ("intr %d, invalid MSI-base %d\n", intr, map->im_msi_base)); 1295 1296 msi_map(map->im_msi_base, addr, data, cpuid); 1297 1298 if (bootverbose) 1299 kprintf("map MSI intr %d on cpu%d\n", intr, cpuid); 1300 1301 lwkt_reltoken(&ioapic_irqmap_tok); 1302 } 1303