1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 4. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: vector.s, 386BSD 0.1 unknown origin 31 * $FreeBSD$ 32 */ 33 34/* 35 * Interrupt entry points for external interrupts triggered by I/O APICs 36 * as well as IPI handlers. 37 */ 38 39#include "opt_smp.h" 40 41#include <machine/asmacros.h> 42#include <x86/apicreg.h> 43 44#include "assym.s" 45 46/* 47 * I/O Interrupt Entry Point. Rather than having one entry point for 48 * each interrupt source, we use one entry point for each 32-bit word 49 * in the ISR. The handler determines the highest bit set in the ISR, 50 * translates that into a vector, and passes the vector to the 51 * lapic_handle_intr() function. 52 */ 53#define ISR_VEC(index, vec_name) \ 54 .text ; \ 55 SUPERALIGN_TEXT ; \ 56IDTVEC(vec_name) ; \ 57 PUSH_FRAME ; \ 58 FAKE_MCOUNT(TF_RIP(%rsp)) ; \ 59 movq lapic, %rdx ; /* pointer to local APIC */ \ 60 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \ 61 bsrl %eax, %eax ; /* index of highest set bit in ISR */ \ 62 jz 1f ; \ 63 addl $(32 * index),%eax ; \ 64 movq %rsp, %rsi ; \ 65 movl %eax, %edi ; /* pass the IRQ */ \ 66 call lapic_handle_intr ; \ 671: ; \ 68 MEXITCOUNT ; \ 69 jmp doreti 70 71/* 72 * Handle "spurious INTerrupts". 73 * Notes: 74 * This is different than the "spurious INTerrupt" generated by an 75 * 8259 PIC for missing INTs. See the APIC documentation for details. 76 * This routine should NOT do an 'EOI' cycle. 77 */ 78 .text 79 SUPERALIGN_TEXT 80IDTVEC(spuriousint) 81 82 /* No EOI cycle used here */ 83 84 jmp doreti_iret 85 86 ISR_VEC(1, apic_isr1) 87 ISR_VEC(2, apic_isr2) 88 ISR_VEC(3, apic_isr3) 89 ISR_VEC(4, apic_isr4) 90 ISR_VEC(5, apic_isr5) 91 ISR_VEC(6, apic_isr6) 92 ISR_VEC(7, apic_isr7) 93 94/* 95 * Local APIC periodic timer handler. 96 */ 97 .text 98 SUPERALIGN_TEXT 99IDTVEC(timerint) 100 PUSH_FRAME 101 FAKE_MCOUNT(TF_RIP(%rsp)) 102 movq %rsp, %rdi 103 call lapic_handle_timer 104 MEXITCOUNT 105 jmp doreti 106 107/* 108 * Local APIC CMCI handler. 109 */ 110 .text 111 SUPERALIGN_TEXT 112IDTVEC(cmcint) 113 PUSH_FRAME 114 FAKE_MCOUNT(TF_RIP(%rsp)) 115 call lapic_handle_cmc 116 MEXITCOUNT 117 jmp doreti 118 119/* 120 * Local APIC error interrupt handler. 121 */ 122 .text 123 SUPERALIGN_TEXT 124IDTVEC(errorint) 125 PUSH_FRAME 126 FAKE_MCOUNT(TF_RIP(%rsp)) 127 call lapic_handle_error 128 MEXITCOUNT 129 jmp doreti 130 131#ifdef SMP 132/* 133 * Global address space TLB shootdown. 134 */ 135 .text 136 SUPERALIGN_TEXT 137IDTVEC(invltlb) 138#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) 139 PUSH_FRAME 140 movl PCPU(CPUID), %eax 141#ifdef COUNT_XINVLTLB_HITS 142 incl xhits_gbl(,%rax,4) 143#endif 144#ifdef COUNT_IPIS 145 movq ipi_invltlb_counts(,%rax,8),%rax 146 incq (%rax) 147#endif 148 POP_FRAME 149#endif 150 151 pushq %rax 152 153 movq %cr3, %rax /* invalidate the TLB */ 154 movq %rax, %cr3 155 156 movq lapic, %rax 157 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 158 159 lock 160 incl smp_tlb_wait 161 162 popq %rax 163 jmp doreti_iret 164 165/* 166 * Single page TLB shootdown 167 */ 168 .text 169 SUPERALIGN_TEXT 170IDTVEC(invlpg) 171#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) 172 PUSH_FRAME 173 movl PCPU(CPUID), %eax 174#ifdef COUNT_XINVLTLB_HITS 175 incl xhits_pg(,%rax,4) 176#endif 177#ifdef COUNT_IPIS 178 movq ipi_invlpg_counts(,%rax,8),%rax 179 incq (%rax) 180#endif 181 POP_FRAME 182#endif 183 184 pushq %rax 185 186 movq smp_tlb_addr1, %rax 187 invlpg (%rax) /* invalidate single page */ 188 189 movq lapic, %rax 190 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 191 192 lock 193 incl smp_tlb_wait 194 195 popq %rax 196 jmp doreti_iret 197 198/* 199 * Page range TLB shootdown. 200 */ 201 .text 202 SUPERALIGN_TEXT 203IDTVEC(invlrng) 204#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) 205 PUSH_FRAME 206 movl PCPU(CPUID), %eax 207#ifdef COUNT_XINVLTLB_HITS 208 incl xhits_rng(,%rax,4) 209#endif 210#ifdef COUNT_IPIS 211 movq ipi_invlrng_counts(,%rax,8),%rax 212 incq (%rax) 213#endif 214 POP_FRAME 215#endif 216 217 pushq %rax 218 pushq %rdx 219 220 movq smp_tlb_addr1, %rdx 221 movq smp_tlb_addr2, %rax 2221: invlpg (%rdx) /* invalidate single page */ 223 addq $PAGE_SIZE, %rdx 224 cmpq %rax, %rdx 225 jb 1b 226 227 movq lapic, %rax 228 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 229 230 lock 231 incl smp_tlb_wait 232 233 popq %rdx 234 popq %rax 235 jmp doreti_iret 236 237/* 238 * Invalidate cache. 239 */ 240 .text 241 SUPERALIGN_TEXT 242IDTVEC(invlcache) 243#ifdef COUNT_IPIS 244 PUSH_FRAME 245 movl PCPU(CPUID), %eax 246 movq ipi_invlcache_counts(,%rax,8),%rax 247 incq (%rax) 248 POP_FRAME 249#endif 250 251 pushq %rax 252 253 wbinvd 254 255 movq lapic, %rax 256 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 257 258 lock 259 incl smp_tlb_wait 260 261 popq %rax 262 jmp doreti_iret 263 264/* 265 * Handler for IPIs sent via the per-cpu IPI bitmap. 266 */ 267 .text 268 SUPERALIGN_TEXT 269IDTVEC(ipi_intr_bitmap_handler) 270 PUSH_FRAME 271 272 movq lapic, %rdx 273 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */ 274 275 FAKE_MCOUNT(TF_RIP(%rsp)) 276 277 call ipi_bitmap_handler 278 MEXITCOUNT 279 jmp doreti 280 281/* 282 * Executed by a CPU when it receives an IPI_STOP from another CPU. 283 */ 284 .text 285 SUPERALIGN_TEXT 286IDTVEC(cpustop) 287 PUSH_FRAME 288 289 movq lapic, %rax 290 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 291 292 call cpustop_handler 293 jmp doreti 294 295/* 296 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 297 */ 298 .text 299 SUPERALIGN_TEXT 300IDTVEC(cpususpend) 301 PUSH_FRAME 302 303 call cpususpend_handler 304 movq lapic, %rax 305 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 306 jmp doreti 307 308/* 309 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 310 * 311 * - Calls the generic rendezvous action function. 312 */ 313 .text 314 SUPERALIGN_TEXT 315IDTVEC(rendezvous) 316 PUSH_FRAME 317#ifdef COUNT_IPIS 318 movl PCPU(CPUID), %eax 319 movq ipi_rendezvous_counts(,%rax,8), %rax 320 incq (%rax) 321#endif 322 call smp_rendezvous_action 323 movq lapic, %rax 324 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 325 jmp doreti 326#endif /* SMP */ 327