1/* 2 * from: vector.s, 386BSD 0.1 unknown origin 3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $ 4 */ 5 6#if 0 7#include "opt_auto_eoi.h" 8#endif 9 10#include <machine/asmacros.h> 11#include <machine/lock.h> 12#include <machine/psl.h> 13#include <machine/trap.h> 14#include <machine/segments.h> 15 16#include <machine_base/icu/icu.h> 17#include <bus/isa/isa.h> 18 19#include "assym.s" 20 21#include "apicreg.h" 22#include <machine_base/apic/ioapic_ipl.h> 23#include <machine/intr_machdep.h> 24 25#ifdef foo 26/* convert an absolute IRQ# into bitmask */ 27#define IRQ_LBIT(irq_num) (1UL << (irq_num & 0x3f)) 28#endif 29 30#define IRQ_SBITS(irq_num) ((irq_num) & 0x3f) 31 32/* convert an absolute IRQ# into gd_ipending index */ 33#define IRQ_LIDX(irq_num) ((irq_num) >> 6) 34 35#ifdef SMP 36#define MPLOCKED lock ; 37#else 38#define MPLOCKED 39#endif 40 41#define APIC_PUSH_FRAME \ 42 PUSH_FRAME ; /* 15 regs + space for 5 extras */ \ 43 movq $0,TF_XFLAGS(%rsp) ; \ 44 movq $0,TF_TRAPNO(%rsp) ; \ 45 movq $0,TF_ADDR(%rsp) ; \ 46 movq $0,TF_FLAGS(%rsp) ; \ 47 movq $0,TF_ERR(%rsp) ; \ 48 cld ; \ 49 50/* 51 * JG stale? Warning: POP_FRAME can only be used if there is no chance of a 52 * segment register being changed (e.g. by procfs), which is why syscalls 53 * have to use doreti. 54 */ 55#define APIC_POP_FRAME \ 56 POP_FRAME ; \ 57 58#define IOAPICADDR(irq_num) \ 59 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR 60#define REDIRIDX(irq_num) \ 61 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX 62#define IOAPICFLAGS(irq_num) \ 63 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS 64 65#define MASK_IRQ(irq_num) \ 66 IOAPIC_IMASK_LOCK ; /* into critical reg */ \ 67 testl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ 68 jne 7f ; /* masked, don't mask */ \ 69 orl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ 70 /* set the mask bit */ \ 71 movq IOAPICADDR(irq_num), %rcx ; /* ioapic addr */ \ 72 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 73 movl %eax, (%rcx) ; /* write the index */ \ 74 orl $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */ \ 757: ; /* already masked */ \ 76 IOAPIC_IMASK_UNLOCK ; \ 77 78/* 79 * Test to see whether we are handling an edge or level triggered INT. 80 * Level-triggered INTs must still be masked as we don't clear the source, 81 * and the EOI cycle would cause redundant INTs to occur. 82 */ 83#define MASK_LEVEL_IRQ(irq_num) \ 84 testl $IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \ 85 jz 9f ; /* edge, don't mask */ \ 86 MASK_IRQ(irq_num) ; \ 879: ; \ 88 89/* 90 * Test to see if the source is currntly masked, clear if so. 91 */ 92#define UNMASK_IRQ(irq_num) \ 93 cmpl $0,%eax ; \ 94 jnz 8f ; \ 95 IOAPIC_IMASK_LOCK ; /* into critical reg */ \ 96 testl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ 97 je 7f ; /* bit clear, not masked */ \ 98 andl $~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ 99 /* clear mask bit */ \ 100 movq IOAPICADDR(irq_num),%rcx ; /* ioapic addr */ \ 101 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 102 movl %eax,(%rcx) ; /* write the index */ \ 103 andl $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \ 1047: ; \ 105 IOAPIC_IMASK_UNLOCK ; \ 1068: ; \ 107 108/* 109 * Interrupt call handlers run in the following sequence: 110 * 111 * - Push the trap frame required by doreti 112 * - Mask the interrupt and reenable its source 113 * - If we cannot take the interrupt set its ipending bit and 114 * doreti. 115 * - If we can take the interrupt clear its ipending bit, 116 * call the handler, then unmask and doreti. 117 * 118 * YYY can cache gd base opitner instead of using hidden %fs prefixes. 119 */ 120 121#define INTR_HANDLER(irq_num) \ 122 .text ; \ 123 SUPERALIGN_TEXT ; \ 124IDTVEC(ioapic_intr##irq_num) ; \ 125 APIC_PUSH_FRAME ; \ 126 FAKE_MCOUNT(TF_RIP(%rsp)) ; \ 127 MASK_LEVEL_IRQ(irq_num) ; \ 128 movq lapic, %rax ; \ 129 movl $0, LA_EOI(%rax) ; \ 130 movq PCPU(curthread),%rbx ; \ 131 testl $-1,TD_NEST_COUNT(%rbx) ; \ 132 jne 1f ; \ 133 testl $-1,TD_CRITCOUNT(%rbx) ; \ 134 je 2f ; \ 1351: ; \ 136 /* in critical section, make interrupt pending */ \ 137 /* set the pending bit and return, leave interrupt masked */ \ 138 movq $1,%rcx ; \ 139 shlq $IRQ_SBITS(irq_num),%rcx ; \ 140 movq $IRQ_LIDX(irq_num),%rdx ; \ 141 orq %rcx,PCPU_E8(ipending,%rdx) ; \ 142 orl $RQF_INTPEND,PCPU(reqflags) ; \ 143 jmp 5f ; \ 1442: ; \ 145 /* clear pending bit, run handler */ \ 146 movq $1,%rcx ; \ 147 shlq $IRQ_SBITS(irq_num),%rcx ; \ 148 notq %rcx ; \ 149 movq $IRQ_LIDX(irq_num),%rdx ; \ 150 andq %rcx,PCPU_E8(ipending,%rdx) ; \ 151 pushq $irq_num ; /* trapframe -> intrframe */ \ 152 movq %rsp, %rdi ; /* pass frame by reference */ \ 153 incl TD_CRITCOUNT(%rbx) ; \ 154 sti ; \ 155 call ithread_fast_handler ; /* returns 0 to unmask */ \ 156 decl TD_CRITCOUNT(%rbx) ; \ 157 addq $8, %rsp ; /* intrframe -> trapframe */ \ 158 UNMASK_IRQ(irq_num) ; \ 1595: ; \ 160 MEXITCOUNT ; \ 161 jmp doreti ; \ 162 163/* 164 * Handle "spurious INTerrupts". 165 * 166 * NOTE: This is different than the "spurious INTerrupt" generated by an 167 * 8259 PIC for missing INTs. See the APIC documentation for details. 168 * This routine should NOT do an 'EOI' cycle. 169 * 170 * NOTE: Even though we don't do anything here we must still swapgs if 171 * coming from a user frame in case the iretq faults... just use 172 * the nominal APIC_PUSH_FRAME sequence to get it done. 173 */ 174 .text 175 SUPERALIGN_TEXT 176 .globl Xspuriousint 177Xspuriousint: 178 APIC_PUSH_FRAME 179 /* No EOI cycle used here */ 180 FAKE_MCOUNT(TF_RIP(%rsp)) 181 MEXITCOUNT 182 APIC_POP_FRAME 183 jmp doreti_iret 184 185#ifdef SMP 186 187/* 188 * Handle TLB shootdowns. 189 * 190 * NOTE: interrupts are left disabled. 191 */ 192 .text 193 SUPERALIGN_TEXT 194 .globl Xinvltlb 195Xinvltlb: 196 APIC_PUSH_FRAME 197 movq lapic, %rax 198 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 199 FAKE_MCOUNT(TF_RIP(%rsp)) 200 subq $8,%rsp /* make same as interrupt frame */ 201 movq %rsp,%rdi /* pass frame by reference */ 202 call smp_invltlb_intr 203 addq $8,%rsp /* turn into trapframe */ 204 MEXITCOUNT 205 APIC_POP_FRAME 206 jmp doreti_iret 207 208/* 209 * Executed by a CPU when it receives an Xcpustop IPI from another CPU, 210 * 211 * - We cannot call doreti 212 * - Signals its receipt. 213 * - Waits for permission to restart. 214 * - Processing pending IPIQ events while waiting. 215 * - Signals its restart. 216 */ 217 218 .text 219 SUPERALIGN_TEXT 220 .globl Xcpustop 221Xcpustop: 222 APIC_PUSH_FRAME 223 movq lapic, %rax 224 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 225 226 movl PCPU(cpuid), %eax 227 imull $PCB_SIZE, %eax 228 leaq CNAME(stoppcbs), %rdi 229 addq %rax, %rdi 230 call CNAME(savectx) /* Save process context */ 231 232 movslq PCPU(cpuid), %rax 233 234 /* 235 * Indicate that we have stopped and loop waiting for permission 236 * to start again. We must still process IPI events while in a 237 * stopped state. 238 * 239 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts 240 * (e.g. Xtimer, Xinvltlb). 241 */ 242 MPLOCKED 243 btsq %rax, stopped_cpus /* stopped_cpus |= (1<<id) */ 244 sti 2451: 246 andl $~RQF_IPIQ,PCPU(reqflags) 247 pushq %rax 248 call lwkt_smp_stopped 249 popq %rax 250 pause 251 btq %rax, started_cpus /* while (!(started_cpus & (1<<id))) */ 252 jnc 1b 253 254 MPLOCKED 255 btrq %rax, started_cpus /* started_cpus &= ~(1<<id) */ 256 MPLOCKED 257 btrq %rax, stopped_cpus /* stopped_cpus &= ~(1<<id) */ 258 259 testq %rax, %rax 260 jnz 2f 261 262 movq CNAME(cpustop_restartfunc), %rax 263 testq %rax, %rax 264 jz 2f 265 movq $0, CNAME(cpustop_restartfunc) /* One-shot */ 266 267 call *%rax 2682: 269 MEXITCOUNT 270 APIC_POP_FRAME 271 jmp doreti_iret 272 273 /* 274 * For now just have one ipiq IPI, but what we really want is 275 * to have one for each source cpu to the APICs don't get stalled 276 * backlogging the requests. 277 */ 278 .text 279 SUPERALIGN_TEXT 280 .globl Xipiq 281Xipiq: 282 APIC_PUSH_FRAME 283 movq lapic, %rax 284 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 285 FAKE_MCOUNT(TF_RIP(%rsp)) 286 287 incl PCPU(cnt) + V_IPI 288 movq PCPU(curthread),%rbx 289 testl $-1,TD_CRITCOUNT(%rbx) 290 jne 1f 291 subq $8,%rsp /* make same as interrupt frame */ 292 movq %rsp,%rdi /* pass frame by reference */ 293 incl PCPU(intr_nesting_level) 294 incl TD_CRITCOUNT(%rbx) 295 sti 296 call lwkt_process_ipiq_frame 297 decl TD_CRITCOUNT(%rbx) 298 decl PCPU(intr_nesting_level) 299 addq $8,%rsp /* turn into trapframe */ 300 MEXITCOUNT 301 jmp doreti 3021: 303 orl $RQF_IPIQ,PCPU(reqflags) 304 MEXITCOUNT 305 APIC_POP_FRAME 306 jmp doreti_iret 307 308#endif /* SMP */ 309 310 .text 311 SUPERALIGN_TEXT 312 .globl Xtimer 313Xtimer: 314 APIC_PUSH_FRAME 315 movq lapic, %rax 316 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 317 FAKE_MCOUNT(TF_RIP(%rsp)) 318 319 subq $8,%rsp /* make same as interrupt frame */ 320 movq %rsp,%rdi /* pass frame by reference */ 321 call lapic_timer_always 322 addq $8,%rsp /* turn into trapframe */ 323 324 incl PCPU(cnt) + V_TIMER 325 movq TF_RIP(%rsp),%rbx /* sample addr before checking crit */ 326 movq %rbx,PCPU(sample_pc) 327 movq PCPU(curthread),%rbx 328 testl $-1,TD_CRITCOUNT(%rbx) 329 jne 1f 330 testl $-1,TD_NEST_COUNT(%rbx) 331 jne 1f 332 subq $8,%rsp /* make same as interrupt frame */ 333 movq %rsp,%rdi /* pass frame by reference */ 334 incl PCPU(intr_nesting_level) 335 incl TD_CRITCOUNT(%rbx) 336 sti 337 call lapic_timer_process_frame 338 decl TD_CRITCOUNT(%rbx) 339 decl PCPU(intr_nesting_level) 340 addq $8,%rsp /* turn into trapframe */ 341 MEXITCOUNT 342 jmp doreti 3431: 344 orl $RQF_TIMER,PCPU(reqflags) 345 MEXITCOUNT 346 APIC_POP_FRAME 347 jmp doreti_iret 348 349MCOUNT_LABEL(bintr) 350 INTR_HANDLER(0) 351 INTR_HANDLER(1) 352 INTR_HANDLER(2) 353 INTR_HANDLER(3) 354 INTR_HANDLER(4) 355 INTR_HANDLER(5) 356 INTR_HANDLER(6) 357 INTR_HANDLER(7) 358 INTR_HANDLER(8) 359 INTR_HANDLER(9) 360 INTR_HANDLER(10) 361 INTR_HANDLER(11) 362 INTR_HANDLER(12) 363 INTR_HANDLER(13) 364 INTR_HANDLER(14) 365 INTR_HANDLER(15) 366 INTR_HANDLER(16) 367 INTR_HANDLER(17) 368 INTR_HANDLER(18) 369 INTR_HANDLER(19) 370 INTR_HANDLER(20) 371 INTR_HANDLER(21) 372 INTR_HANDLER(22) 373 INTR_HANDLER(23) 374 INTR_HANDLER(24) 375 INTR_HANDLER(25) 376 INTR_HANDLER(26) 377 INTR_HANDLER(27) 378 INTR_HANDLER(28) 379 INTR_HANDLER(29) 380 INTR_HANDLER(30) 381 INTR_HANDLER(31) 382 INTR_HANDLER(32) 383 INTR_HANDLER(33) 384 INTR_HANDLER(34) 385 INTR_HANDLER(35) 386 INTR_HANDLER(36) 387 INTR_HANDLER(37) 388 INTR_HANDLER(38) 389 INTR_HANDLER(39) 390 INTR_HANDLER(40) 391 INTR_HANDLER(41) 392 INTR_HANDLER(42) 393 INTR_HANDLER(43) 394 INTR_HANDLER(44) 395 INTR_HANDLER(45) 396 INTR_HANDLER(46) 397 INTR_HANDLER(47) 398 INTR_HANDLER(48) 399 INTR_HANDLER(49) 400 INTR_HANDLER(50) 401 INTR_HANDLER(51) 402 INTR_HANDLER(52) 403 INTR_HANDLER(53) 404 INTR_HANDLER(54) 405 INTR_HANDLER(55) 406 INTR_HANDLER(56) 407 INTR_HANDLER(57) 408 INTR_HANDLER(58) 409 INTR_HANDLER(59) 410 INTR_HANDLER(60) 411 INTR_HANDLER(61) 412 INTR_HANDLER(62) 413 INTR_HANDLER(63) 414 INTR_HANDLER(64) 415 INTR_HANDLER(65) 416 INTR_HANDLER(66) 417 INTR_HANDLER(67) 418 INTR_HANDLER(68) 419 INTR_HANDLER(69) 420 INTR_HANDLER(70) 421 INTR_HANDLER(71) 422 INTR_HANDLER(72) 423 INTR_HANDLER(73) 424 INTR_HANDLER(74) 425 INTR_HANDLER(75) 426 INTR_HANDLER(76) 427 INTR_HANDLER(77) 428 INTR_HANDLER(78) 429 INTR_HANDLER(79) 430 INTR_HANDLER(80) 431 INTR_HANDLER(81) 432 INTR_HANDLER(82) 433 INTR_HANDLER(83) 434 INTR_HANDLER(84) 435 INTR_HANDLER(85) 436 INTR_HANDLER(86) 437 INTR_HANDLER(87) 438 INTR_HANDLER(88) 439 INTR_HANDLER(89) 440 INTR_HANDLER(90) 441 INTR_HANDLER(91) 442 INTR_HANDLER(92) 443 INTR_HANDLER(93) 444 INTR_HANDLER(94) 445 INTR_HANDLER(95) 446 INTR_HANDLER(96) 447 INTR_HANDLER(97) 448 INTR_HANDLER(98) 449 INTR_HANDLER(99) 450 INTR_HANDLER(100) 451 INTR_HANDLER(101) 452 INTR_HANDLER(102) 453 INTR_HANDLER(103) 454 INTR_HANDLER(104) 455 INTR_HANDLER(105) 456 INTR_HANDLER(106) 457 INTR_HANDLER(107) 458 INTR_HANDLER(108) 459 INTR_HANDLER(109) 460 INTR_HANDLER(110) 461 INTR_HANDLER(111) 462 INTR_HANDLER(112) 463 INTR_HANDLER(113) 464 INTR_HANDLER(114) 465 INTR_HANDLER(115) 466 INTR_HANDLER(116) 467 INTR_HANDLER(117) 468 INTR_HANDLER(118) 469 INTR_HANDLER(119) 470 INTR_HANDLER(120) 471 INTR_HANDLER(121) 472 INTR_HANDLER(122) 473 INTR_HANDLER(123) 474 INTR_HANDLER(124) 475 INTR_HANDLER(125) 476 INTR_HANDLER(126) 477 INTR_HANDLER(127) 478 INTR_HANDLER(128) 479 INTR_HANDLER(129) 480 INTR_HANDLER(130) 481 INTR_HANDLER(131) 482 INTR_HANDLER(132) 483 INTR_HANDLER(133) 484 INTR_HANDLER(134) 485 INTR_HANDLER(135) 486 INTR_HANDLER(136) 487 INTR_HANDLER(137) 488 INTR_HANDLER(138) 489 INTR_HANDLER(139) 490 INTR_HANDLER(140) 491 INTR_HANDLER(141) 492 INTR_HANDLER(142) 493 INTR_HANDLER(143) 494 INTR_HANDLER(144) 495 INTR_HANDLER(145) 496 INTR_HANDLER(146) 497 INTR_HANDLER(147) 498 INTR_HANDLER(148) 499 INTR_HANDLER(149) 500 INTR_HANDLER(150) 501 INTR_HANDLER(151) 502 INTR_HANDLER(152) 503 INTR_HANDLER(153) 504 INTR_HANDLER(154) 505 INTR_HANDLER(155) 506 INTR_HANDLER(156) 507 INTR_HANDLER(157) 508 INTR_HANDLER(158) 509 INTR_HANDLER(159) 510 INTR_HANDLER(160) 511 INTR_HANDLER(161) 512 INTR_HANDLER(162) 513 INTR_HANDLER(163) 514 INTR_HANDLER(164) 515 INTR_HANDLER(165) 516 INTR_HANDLER(166) 517 INTR_HANDLER(167) 518 INTR_HANDLER(168) 519 INTR_HANDLER(169) 520 INTR_HANDLER(170) 521 INTR_HANDLER(171) 522 INTR_HANDLER(172) 523 INTR_HANDLER(173) 524 INTR_HANDLER(174) 525 INTR_HANDLER(175) 526 INTR_HANDLER(176) 527 INTR_HANDLER(177) 528 INTR_HANDLER(178) 529 INTR_HANDLER(179) 530 INTR_HANDLER(180) 531 INTR_HANDLER(181) 532 INTR_HANDLER(182) 533 INTR_HANDLER(183) 534 INTR_HANDLER(184) 535 INTR_HANDLER(185) 536 INTR_HANDLER(186) 537 INTR_HANDLER(187) 538 INTR_HANDLER(188) 539 INTR_HANDLER(189) 540 INTR_HANDLER(190) 541 INTR_HANDLER(191) 542MCOUNT_LABEL(eintr) 543 544 .data 545 546/* variables used by stop_cpus()/restart_cpus()/Xcpustop */ 547 .globl stopped_cpus, started_cpus 548stopped_cpus: 549 .quad 0 550started_cpus: 551 .quad 0 552 553 .globl CNAME(cpustop_restartfunc) 554CNAME(cpustop_restartfunc): 555 .quad 0 556 557 .text 558 559