1/* 2 * from: vector.s, 386BSD 0.1 unknown origin 3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $ 4 */ 5 6#if 0 7#include "opt_auto_eoi.h" 8#endif 9 10#include <machine/asmacros.h> 11#include <machine/lock.h> 12#include <machine/psl.h> 13#include <machine/trap.h> 14#include <machine/segments.h> 15 16#include <machine_base/icu/icu.h> 17#include <bus/isa/isa.h> 18 19#include "assym.s" 20 21#include "apicreg.h" 22#include "apic_ipl.h" 23#include <machine/smp.h> 24#include <machine_base/isa/intr_machdep.h> 25 26/* convert an absolute IRQ# into a bitmask */ 27#define IRQ_LBIT(irq_num) (1 << (irq_num)) 28 29/* make an index into the IO APIC from the IRQ# */ 30#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) 31 32#ifdef SMP 33#define MPLOCKED lock ; 34#else 35#define MPLOCKED 36#endif 37 38#define APIC_PUSH_FRAME \ 39 PUSH_FRAME ; /* 15 regs + space for 5 extras */ \ 40 movq $0,TF_XFLAGS(%rsp) ; \ 41 movq $0,TF_TRAPNO(%rsp) ; \ 42 movq $0,TF_ADDR(%rsp) ; \ 43 movq $0,TF_FLAGS(%rsp) ; \ 44 movq $0,TF_ERR(%rsp) ; \ 45 cld ; \ 46 47/* 48 * JG stale? Warning: POP_FRAME can only be used if there is no chance of a 49 * segment register being changed (e.g. by procfs), which is why syscalls 50 * have to use doreti. 51 */ 52#define APIC_POP_FRAME POP_FRAME 53 54/* sizeof(struct apic_intmapinfo) == 24 */ 55#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 24 * (irq_num) + 8 56#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 24 * (irq_num) + 16 57 58#define MASK_IRQ(irq_num) \ 59 APIC_IMASK_LOCK ; /* into critical reg */ \ 60 testl $IRQ_LBIT(irq_num), apic_imen ; \ 61 jne 7f ; /* masked, don't mask */ \ 62 orl $IRQ_LBIT(irq_num), apic_imen ; /* set the mask bit */ \ 63 movq IOAPICADDR(irq_num), %rcx ; /* ioapic addr */ \ 64 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 65 movl %eax, (%rcx) ; /* write the index */ \ 66 movl IOAPIC_WINDOW(%rcx), %eax ; /* current value */ \ 67 orl $IOART_INTMASK, %eax ; /* set the mask */ \ 68 movl %eax, IOAPIC_WINDOW(%rcx) ; /* new value */ \ 697: ; /* already masked */ \ 70 APIC_IMASK_UNLOCK ; \ 71 72/* 73 * Test to see whether we are handling an edge or level triggered INT. 74 * Level-triggered INTs must still be masked as we don't clear the source, 75 * and the EOI cycle would cause redundant INTs to occur. 76 */ 77#define MASK_LEVEL_IRQ(irq_num) \ 78 testl $IRQ_LBIT(irq_num), apic_pin_trigger ; \ 79 jz 9f ; /* edge, don't mask */ \ 80 MASK_IRQ(irq_num) ; \ 819: ; \ 82 83/* 84 * Test to see if the source is currntly masked, clear if so. 85 */ 86#define UNMASK_IRQ(irq_num) \ 87 cmpl $0,%eax ; \ 88 jnz 8f ; \ 89 APIC_IMASK_LOCK ; /* into critical reg */ \ 90 testl $IRQ_LBIT(irq_num), apic_imen ; \ 91 je 7f ; /* bit clear, not masked */ \ 92 andl $~IRQ_LBIT(irq_num), apic_imen ;/* clear mask bit */ \ 93 movq IOAPICADDR(irq_num),%rcx ; /* ioapic addr */ \ 94 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 95 movl %eax,(%rcx) ; /* write the index */ \ 96 movl IOAPIC_WINDOW(%rcx),%eax ; /* current value */ \ 97 andl $~IOART_INTMASK,%eax ; /* clear the mask */ \ 98 movl %eax,IOAPIC_WINDOW(%rcx) ; /* new value */ \ 997: ; \ 100 APIC_IMASK_UNLOCK ; \ 1018: ; \ 102 103#ifdef APIC_IO 104 105/* 106 * Fast interrupt call handlers run in the following sequence: 107 * 108 * - Push the trap frame required by doreti 109 * - Mask the interrupt and reenable its source 110 * - If we cannot take the interrupt set its fpending bit and 111 * doreti. Note that we cannot mess with mp_lock at all 112 * if we entered from a critical section! 113 * - If we can take the interrupt clear its fpending bit, 114 * call the handler, then unmask and doreti. 115 * 116 * YYY can cache gd base opitner instead of using hidden %fs prefixes. 117 */ 118 119#define FAST_INTR(irq_num, vec_name) \ 120 .text ; \ 121 SUPERALIGN_TEXT ; \ 122IDTVEC(vec_name) ; \ 123 APIC_PUSH_FRAME ; \ 124 FAKE_MCOUNT(TF_RIP(%rsp)) ; \ 125 MASK_LEVEL_IRQ(irq_num) ; \ 126 movq lapic, %rax ; \ 127 movl $0, LA_EOI(%rax) ; \ 128 movq PCPU(curthread),%rbx ; \ 129 testl $-1,TD_NEST_COUNT(%rbx) ; \ 130 jne 1f ; \ 131 testl $-1,TD_CRITCOUNT(%rbx) ; \ 132 je 2f ; \ 1331: ; \ 134 /* in critical section, make interrupt pending */ \ 135 /* set the pending bit and return, leave interrupt masked */ \ 136 orl $IRQ_LBIT(irq_num),PCPU(fpending) ; \ 137 orl $RQF_INTPEND,PCPU(reqflags) ; \ 138 jmp 5f ; \ 1392: ; \ 140 /* clear pending bit, run handler */ \ 141 andl $~IRQ_LBIT(irq_num),PCPU(fpending) ; \ 142 pushq $irq_num ; /* trapframe -> intrframe */ \ 143 movq %rsp, %rdi ; /* pass frame by reference */ \ 144 incl TD_CRITCOUNT(%rbx) ; \ 145 sti ; \ 146 call ithread_fast_handler ; /* returns 0 to unmask */ \ 147 decl TD_CRITCOUNT(%rbx) ; \ 148 addq $8, %rsp ; /* intrframe -> trapframe */ \ 149 UNMASK_IRQ(irq_num) ; \ 1505: ; \ 151 MEXITCOUNT ; \ 152 jmp doreti ; \ 153 154#endif 155 156/* 157 * Handle "spurious INTerrupts". 158 * Notes: 159 * This is different than the "spurious INTerrupt" generated by an 160 * 8259 PIC for missing INTs. See the APIC documentation for details. 161 * This routine should NOT do an 'EOI' cycle. 162 */ 163 .text 164 SUPERALIGN_TEXT 165 .globl Xspuriousint 166Xspuriousint: 167 168 /* No EOI cycle used here */ 169 170 jmp doreti_iret 171 172 173/* 174 * Handle TLB shootdowns. 175 * 176 * NOTE: interrupts are left disabled. 177 */ 178 .text 179 SUPERALIGN_TEXT 180 .globl Xinvltlb 181Xinvltlb: 182 APIC_PUSH_FRAME 183 movq lapic, %rax 184 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 185 FAKE_MCOUNT(TF_RIP(%rsp)) 186 subq $8,%rsp /* make same as interrupt frame */ 187 movq %rsp,%rdi /* pass frame by reference */ 188 call smp_invltlb_intr 189 addq $8,%rsp /* turn into trapframe */ 190 MEXITCOUNT 191 APIC_POP_FRAME 192 jmp doreti_iret 193 194/* 195 * Executed by a CPU when it receives an Xcpustop IPI from another CPU, 196 * 197 * - We cannot call doreti 198 * - Signals its receipt. 199 * - Waits for permission to restart. 200 * - Processing pending IPIQ events while waiting. 201 * - Signals its restart. 202 */ 203 204 .text 205 SUPERALIGN_TEXT 206 .globl Xcpustop 207Xcpustop: 208 APIC_PUSH_FRAME 209 movq lapic, %rax 210 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 211 212 movl PCPU(cpuid), %eax 213 imull $PCB_SIZE, %eax 214 leaq CNAME(stoppcbs), %rdi 215 addq %rax, %rdi 216 call CNAME(savectx) /* Save process context */ 217 218 movl PCPU(cpuid), %eax 219 220 /* 221 * Indicate that we have stopped and loop waiting for permission 222 * to start again. We must still process IPI events while in a 223 * stopped state. 224 * 225 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts 226 * (e.g. Xtimer, Xinvltlb). 227 */ 228 MPLOCKED 229 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */ 230 sti 2311: 232 andl $~RQF_IPIQ,PCPU(reqflags) 233 pushq %rax 234 call lwkt_smp_stopped 235 popq %rax 236 pause 237 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */ 238 jnc 1b 239 240 MPLOCKED 241 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */ 242 MPLOCKED 243 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */ 244 245 test %eax, %eax 246 jnz 2f 247 248 movq CNAME(cpustop_restartfunc), %rax 249 test %rax, %rax 250 jz 2f 251 movq $0, CNAME(cpustop_restartfunc) /* One-shot */ 252 253 call *%rax 2542: 255 MEXITCOUNT 256 APIC_POP_FRAME 257 jmp doreti_iret 258 259 /* 260 * For now just have one ipiq IPI, but what we really want is 261 * to have one for each source cpu to the APICs don't get stalled 262 * backlogging the requests. 263 */ 264 .text 265 SUPERALIGN_TEXT 266 .globl Xipiq 267Xipiq: 268 APIC_PUSH_FRAME 269 movq lapic, %rax 270 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 271 FAKE_MCOUNT(TF_RIP(%rsp)) 272 273 incl PCPU(cnt) + V_IPI 274 movq PCPU(curthread),%rbx 275 testl $-1,TD_CRITCOUNT(%rbx) 276 jne 1f 277 subq $8,%rsp /* make same as interrupt frame */ 278 movq %rsp,%rdi /* pass frame by reference */ 279 incl PCPU(intr_nesting_level) 280 incl TD_CRITCOUNT(%rbx) 281 sti 282 call lwkt_process_ipiq_frame 283 decl TD_CRITCOUNT(%rbx) 284 decl PCPU(intr_nesting_level) 285 addq $8,%rsp /* turn into trapframe */ 286 MEXITCOUNT 287 jmp doreti 2881: 289 orl $RQF_IPIQ,PCPU(reqflags) 290 MEXITCOUNT 291 APIC_POP_FRAME 292 jmp doreti_iret 293 294 .text 295 SUPERALIGN_TEXT 296 .globl Xtimer 297Xtimer: 298 APIC_PUSH_FRAME 299 movq lapic, %rax 300 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 301 FAKE_MCOUNT(TF_RIP(%rsp)) 302 303 incl PCPU(cnt) + V_TIMER 304 movq PCPU(curthread),%rbx 305 testl $-1,TD_CRITCOUNT(%rbx) 306 jne 1f 307 testl $-1,TD_NEST_COUNT(%rbx) 308 jne 1f 309 subq $8,%rsp /* make same as interrupt frame */ 310 movq %rsp,%rdi /* pass frame by reference */ 311 incl PCPU(intr_nesting_level) 312 incl TD_CRITCOUNT(%rbx) 313 sti 314 call lapic_timer_process_frame 315 decl TD_CRITCOUNT(%rbx) 316 decl PCPU(intr_nesting_level) 317 addq $8,%rsp /* turn into trapframe */ 318 MEXITCOUNT 319 jmp doreti 3201: 321 orl $RQF_TIMER,PCPU(reqflags) 322 MEXITCOUNT 323 APIC_POP_FRAME 324 jmp doreti_iret 325 326#ifdef APIC_IO 327 328MCOUNT_LABEL(bintr) 329 FAST_INTR(0,apic_fastintr0) 330 FAST_INTR(1,apic_fastintr1) 331 FAST_INTR(2,apic_fastintr2) 332 FAST_INTR(3,apic_fastintr3) 333 FAST_INTR(4,apic_fastintr4) 334 FAST_INTR(5,apic_fastintr5) 335 FAST_INTR(6,apic_fastintr6) 336 FAST_INTR(7,apic_fastintr7) 337 FAST_INTR(8,apic_fastintr8) 338 FAST_INTR(9,apic_fastintr9) 339 FAST_INTR(10,apic_fastintr10) 340 FAST_INTR(11,apic_fastintr11) 341 FAST_INTR(12,apic_fastintr12) 342 FAST_INTR(13,apic_fastintr13) 343 FAST_INTR(14,apic_fastintr14) 344 FAST_INTR(15,apic_fastintr15) 345 FAST_INTR(16,apic_fastintr16) 346 FAST_INTR(17,apic_fastintr17) 347 FAST_INTR(18,apic_fastintr18) 348 FAST_INTR(19,apic_fastintr19) 349 FAST_INTR(20,apic_fastintr20) 350 FAST_INTR(21,apic_fastintr21) 351 FAST_INTR(22,apic_fastintr22) 352 FAST_INTR(23,apic_fastintr23) 353MCOUNT_LABEL(eintr) 354 355#endif 356 357 .data 358 359/* variables used by stop_cpus()/restart_cpus()/Xcpustop */ 360 .globl stopped_cpus, started_cpus 361stopped_cpus: 362 .long 0 363started_cpus: 364 .long 0 365 366 .globl CNAME(cpustop_restartfunc) 367CNAME(cpustop_restartfunc): 368 .quad 0 369 370 .globl apic_pin_trigger 371apic_pin_trigger: 372 .long 0 373 374 .text 375 376