1/* 2 * from: vector.s, 386BSD 0.1 unknown origin 3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $ 4 */ 5 6#if 0 7#include "opt_auto_eoi.h" 8#endif 9 10#include <machine/asmacros.h> 11#include <machine/lock.h> 12#include <machine/psl.h> 13#include <machine/trap.h> 14#include <machine/segments.h> 15 16#include <machine_base/icu/icu.h> 17#include <bus/isa/isa.h> 18 19#include "assym.s" 20 21#include "apicreg.h" 22#include "apic_ipl.h" 23#include <machine/smp.h> 24#include <machine_base/isa/intr_machdep.h> 25 26/* convert an absolute IRQ# into a bitmask */ 27#define IRQ_LBIT(irq_num) (1 << (irq_num)) 28 29/* make an index into the IO APIC from the IRQ# */ 30#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) 31 32#ifdef SMP 33#define MPLOCKED lock ; 34#else 35#define MPLOCKED 36#endif 37 38#define APIC_PUSH_FRAME \ 39 PUSH_FRAME ; /* 15 regs + space for 5 extras */ \ 40 movq $0,TF_XFLAGS(%rsp) ; \ 41 movq $0,TF_TRAPNO(%rsp) ; \ 42 movq $0,TF_ADDR(%rsp) ; \ 43 movq $0,TF_FLAGS(%rsp) ; \ 44 movq $0,TF_ERR(%rsp) ; \ 45 cld ; \ 46 47/* 48 * JG stale? Warning: POP_FRAME can only be used if there is no chance of a 49 * segment register being changed (e.g. by procfs), which is why syscalls 50 * have to use doreti. 51 */ 52#define APIC_POP_FRAME POP_FRAME 53 54#define IOAPICADDR(irq_num) \ 55 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR 56#define REDIRIDX(irq_num) \ 57 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX 58#define IOAPICFLAGS(irq_num) \ 59 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS 60 61#define MASK_IRQ(irq_num) \ 62 APIC_IMASK_LOCK ; /* into critical reg */ \ 63 testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ 64 jne 7f ; /* masked, don't mask */ \ 65 orl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ 66 /* set the mask bit */ \ 67 movq IOAPICADDR(irq_num), %rcx ; /* ioapic addr */ \ 68 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 69 movl %eax, (%rcx) ; /* write the index */ \ 70 orl $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */ \ 717: ; /* already masked */ \ 72 APIC_IMASK_UNLOCK ; \ 73 74/* 75 * Test to see whether we are handling an edge or level triggered INT. 76 * Level-triggered INTs must still be masked as we don't clear the source, 77 * and the EOI cycle would cause redundant INTs to occur. 78 */ 79#define MASK_LEVEL_IRQ(irq_num) \ 80 testl $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \ 81 jz 9f ; /* edge, don't mask */ \ 82 MASK_IRQ(irq_num) ; \ 839: ; \ 84 85/* 86 * Test to see if the source is currntly masked, clear if so. 87 */ 88#define UNMASK_IRQ(irq_num) \ 89 cmpl $0,%eax ; \ 90 jnz 8f ; \ 91 APIC_IMASK_LOCK ; /* into critical reg */ \ 92 testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ 93 je 7f ; /* bit clear, not masked */ \ 94 andl $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \ 95 /* clear mask bit */ \ 96 movq IOAPICADDR(irq_num),%rcx ; /* ioapic addr */ \ 97 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 98 movl %eax,(%rcx) ; /* write the index */ \ 99 andl $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \ 1007: ; \ 101 APIC_IMASK_UNLOCK ; \ 1028: ; \ 103 104#ifdef SMP /* APIC-IO */ 105 106/* 107 * Interrupt call handlers run in the following sequence: 108 * 109 * - Push the trap frame required by doreti 110 * - Mask the interrupt and reenable its source 111 * - If we cannot take the interrupt set its fpending bit and 112 * doreti. 113 * - If we can take the interrupt clear its fpending bit, 114 * call the handler, then unmask and doreti. 115 * 116 * YYY can cache gd base opitner instead of using hidden %fs prefixes. 117 */ 118 119#define INTR_HANDLER(irq_num) \ 120 .text ; \ 121 SUPERALIGN_TEXT ; \ 122IDTVEC(apic_intr##irq_num) ; \ 123 APIC_PUSH_FRAME ; \ 124 FAKE_MCOUNT(TF_RIP(%rsp)) ; \ 125 MASK_LEVEL_IRQ(irq_num) ; \ 126 movq lapic, %rax ; \ 127 movl $0, LA_EOI(%rax) ; \ 128 movq PCPU(curthread),%rbx ; \ 129 testl $-1,TD_NEST_COUNT(%rbx) ; \ 130 jne 1f ; \ 131 testl $-1,TD_CRITCOUNT(%rbx) ; \ 132 je 2f ; \ 1331: ; \ 134 /* in critical section, make interrupt pending */ \ 135 /* set the pending bit and return, leave interrupt masked */ \ 136 orl $IRQ_LBIT(irq_num),PCPU(fpending) ; \ 137 orl $RQF_INTPEND,PCPU(reqflags) ; \ 138 jmp 5f ; \ 1392: ; \ 140 /* clear pending bit, run handler */ \ 141 andl $~IRQ_LBIT(irq_num),PCPU(fpending) ; \ 142 pushq $irq_num ; /* trapframe -> intrframe */ \ 143 movq %rsp, %rdi ; /* pass frame by reference */ \ 144 incl TD_CRITCOUNT(%rbx) ; \ 145 sti ; \ 146 call ithread_fast_handler ; /* returns 0 to unmask */ \ 147 decl TD_CRITCOUNT(%rbx) ; \ 148 addq $8, %rsp ; /* intrframe -> trapframe */ \ 149 UNMASK_IRQ(irq_num) ; \ 1505: ; \ 151 MEXITCOUNT ; \ 152 jmp doreti ; \ 153 154#endif 155 156/* 157 * Handle "spurious INTerrupts". 158 * Notes: 159 * This is different than the "spurious INTerrupt" generated by an 160 * 8259 PIC for missing INTs. See the APIC documentation for details. 161 * This routine should NOT do an 'EOI' cycle. 162 */ 163 .text 164 SUPERALIGN_TEXT 165 .globl Xspuriousint 166Xspuriousint: 167 168 /* No EOI cycle used here */ 169 170 jmp doreti_iret 171 172 173/* 174 * Handle TLB shootdowns. 175 * 176 * NOTE: interrupts are left disabled. 177 */ 178 .text 179 SUPERALIGN_TEXT 180 .globl Xinvltlb 181Xinvltlb: 182 APIC_PUSH_FRAME 183 movq lapic, %rax 184 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 185 FAKE_MCOUNT(TF_RIP(%rsp)) 186 subq $8,%rsp /* make same as interrupt frame */ 187 movq %rsp,%rdi /* pass frame by reference */ 188 call smp_invltlb_intr 189 addq $8,%rsp /* turn into trapframe */ 190 MEXITCOUNT 191 APIC_POP_FRAME 192 jmp doreti_iret 193 194/* 195 * Executed by a CPU when it receives an Xcpustop IPI from another CPU, 196 * 197 * - We cannot call doreti 198 * - Signals its receipt. 199 * - Waits for permission to restart. 200 * - Processing pending IPIQ events while waiting. 201 * - Signals its restart. 202 */ 203 204 .text 205 SUPERALIGN_TEXT 206 .globl Xcpustop 207Xcpustop: 208 APIC_PUSH_FRAME 209 movq lapic, %rax 210 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 211 212 movl PCPU(cpuid), %eax 213 imull $PCB_SIZE, %eax 214 leaq CNAME(stoppcbs), %rdi 215 addq %rax, %rdi 216 call CNAME(savectx) /* Save process context */ 217 218 movslq PCPU(cpuid), %rax 219 220 /* 221 * Indicate that we have stopped and loop waiting for permission 222 * to start again. We must still process IPI events while in a 223 * stopped state. 224 * 225 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts 226 * (e.g. Xtimer, Xinvltlb). 227 */ 228 MPLOCKED 229 btsq %rax, stopped_cpus /* stopped_cpus |= (1<<id) */ 230 sti 2311: 232 andl $~RQF_IPIQ,PCPU(reqflags) 233 pushq %rax 234 call lwkt_smp_stopped 235 popq %rax 236 pause 237 btq %rax, started_cpus /* while (!(started_cpus & (1<<id))) */ 238 jnc 1b 239 240 MPLOCKED 241 btrq %rax, started_cpus /* started_cpus &= ~(1<<id) */ 242 MPLOCKED 243 btrq %rax, stopped_cpus /* stopped_cpus &= ~(1<<id) */ 244 245 test %eax, %eax 246 jnz 2f 247 248 movq CNAME(cpustop_restartfunc), %rax 249 test %rax, %rax 250 jz 2f 251 movq $0, CNAME(cpustop_restartfunc) /* One-shot */ 252 253 call *%rax 2542: 255 MEXITCOUNT 256 APIC_POP_FRAME 257 jmp doreti_iret 258 259 /* 260 * For now just have one ipiq IPI, but what we really want is 261 * to have one for each source cpu to the APICs don't get stalled 262 * backlogging the requests. 263 */ 264 .text 265 SUPERALIGN_TEXT 266 .globl Xipiq 267Xipiq: 268 APIC_PUSH_FRAME 269 movq lapic, %rax 270 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 271 FAKE_MCOUNT(TF_RIP(%rsp)) 272 273 incl PCPU(cnt) + V_IPI 274 movq PCPU(curthread),%rbx 275 testl $-1,TD_CRITCOUNT(%rbx) 276 jne 1f 277 subq $8,%rsp /* make same as interrupt frame */ 278 movq %rsp,%rdi /* pass frame by reference */ 279 incl PCPU(intr_nesting_level) 280 incl TD_CRITCOUNT(%rbx) 281 sti 282 call lwkt_process_ipiq_frame 283 decl TD_CRITCOUNT(%rbx) 284 decl PCPU(intr_nesting_level) 285 addq $8,%rsp /* turn into trapframe */ 286 MEXITCOUNT 287 jmp doreti 2881: 289 orl $RQF_IPIQ,PCPU(reqflags) 290 MEXITCOUNT 291 APIC_POP_FRAME 292 jmp doreti_iret 293 294 .text 295 SUPERALIGN_TEXT 296 .globl Xtimer 297Xtimer: 298 APIC_PUSH_FRAME 299 movq lapic, %rax 300 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 301 FAKE_MCOUNT(TF_RIP(%rsp)) 302 303 subq $8,%rsp /* make same as interrupt frame */ 304 movq %rsp,%rdi /* pass frame by reference */ 305 call lapic_timer_always 306 addq $8,%rsp /* turn into trapframe */ 307 308 incl PCPU(cnt) + V_TIMER 309 movq PCPU(curthread),%rbx 310 testl $-1,TD_CRITCOUNT(%rbx) 311 jne 1f 312 testl $-1,TD_NEST_COUNT(%rbx) 313 jne 1f 314 subq $8,%rsp /* make same as interrupt frame */ 315 movq %rsp,%rdi /* pass frame by reference */ 316 incl PCPU(intr_nesting_level) 317 incl TD_CRITCOUNT(%rbx) 318 sti 319 call lapic_timer_process_frame 320 decl TD_CRITCOUNT(%rbx) 321 decl PCPU(intr_nesting_level) 322 addq $8,%rsp /* turn into trapframe */ 323 MEXITCOUNT 324 jmp doreti 3251: 326 orl $RQF_TIMER,PCPU(reqflags) 327 MEXITCOUNT 328 APIC_POP_FRAME 329 jmp doreti_iret 330 331#ifdef SMP /* APIC-IO */ 332 333MCOUNT_LABEL(bintr) 334 INTR_HANDLER(0) 335 INTR_HANDLER(1) 336 INTR_HANDLER(2) 337 INTR_HANDLER(3) 338 INTR_HANDLER(4) 339 INTR_HANDLER(5) 340 INTR_HANDLER(6) 341 INTR_HANDLER(7) 342 INTR_HANDLER(8) 343 INTR_HANDLER(9) 344 INTR_HANDLER(10) 345 INTR_HANDLER(11) 346 INTR_HANDLER(12) 347 INTR_HANDLER(13) 348 INTR_HANDLER(14) 349 INTR_HANDLER(15) 350 INTR_HANDLER(16) 351 INTR_HANDLER(17) 352 INTR_HANDLER(18) 353 INTR_HANDLER(19) 354 INTR_HANDLER(20) 355 INTR_HANDLER(21) 356 INTR_HANDLER(22) 357 INTR_HANDLER(23) 358MCOUNT_LABEL(eintr) 359 360#endif 361 362 .data 363 364/* variables used by stop_cpus()/restart_cpus()/Xcpustop */ 365 .globl stopped_cpus, started_cpus 366stopped_cpus: 367 .quad 0 368started_cpus: 369 .quad 0 370 371 .globl CNAME(cpustop_restartfunc) 372CNAME(cpustop_restartfunc): 373 .quad 0 374 375 .text 376 377