1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 4. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: vector.s, 386BSD 0.1 unknown origin 31 * $FreeBSD$ 32 */ 33 34/* 35 * Interrupt entry points for external interrupts triggered by I/O APICs 36 * as well as IPI handlers. 37 */ 38 39#include "opt_smp.h" 40 41#include <machine/asmacros.h> 42#include <machine/specialreg.h> 43#include <x86/apicreg.h> 44 45#include "assym.s" 46 47#ifdef SMP 48#define LK lock ; 49#else 50#define LK 51#endif 52 53 .text 54 SUPERALIGN_TEXT 55 /* End Of Interrupt to APIC */ 56as_lapic_eoi: 57 cmpl $0,x2apic_mode 58 jne 1f 59 movq lapic_map,%rax 60 movl $0,LA_EOI(%rax) 61 ret 621: 63 movl $MSR_APIC_EOI,%ecx 64 xorl %eax,%eax 65 xorl %edx,%edx 66 wrmsr 67 ret 68 69/* 70 * I/O Interrupt Entry Point. Rather than having one entry point for 71 * each interrupt source, we use one entry point for each 32-bit word 72 * in the ISR. The handler determines the highest bit set in the ISR, 73 * translates that into a vector, and passes the vector to the 74 * lapic_handle_intr() function. 75 */ 76#define ISR_VEC(index, vec_name) \ 77 .text ; \ 78 SUPERALIGN_TEXT ; \ 79IDTVEC(vec_name) ; \ 80 PUSH_FRAME ; \ 81 FAKE_MCOUNT(TF_RIP(%rsp)) ; \ 82 cmpl $0,x2apic_mode ; \ 83 je 1f ; \ 84 movl $(MSR_APIC_ISR0 + index),%ecx ; \ 85 rdmsr ; \ 86 jmp 2f ; \ 871: ; \ 88 movq lapic_map, %rdx ; /* pointer to local APIC */ \ 89 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \ 902: ; \ 91 bsrl %eax, %eax ; /* index of highest set bit in ISR */ \ 92 jz 3f ; \ 93 addl $(32 * index),%eax ; \ 94 movq %rsp, %rsi ; \ 95 movl %eax, %edi ; /* pass the IRQ */ \ 96 call lapic_handle_intr ; \ 973: ; \ 98 MEXITCOUNT ; \ 99 jmp doreti 100 101/* 102 * Handle "spurious INTerrupts". 103 * Notes: 104 * This is different than the "spurious INTerrupt" generated by an 105 * 8259 PIC for missing INTs. See the APIC documentation for details. 106 * This routine should NOT do an 'EOI' cycle. 107 */ 108 .text 109 SUPERALIGN_TEXT 110IDTVEC(spuriousint) 111 112 /* No EOI cycle used here */ 113 114 jmp doreti_iret 115 116 ISR_VEC(1, apic_isr1) 117 ISR_VEC(2, apic_isr2) 118 ISR_VEC(3, apic_isr3) 119 ISR_VEC(4, apic_isr4) 120 ISR_VEC(5, apic_isr5) 121 ISR_VEC(6, apic_isr6) 122 ISR_VEC(7, apic_isr7) 123 124/* 125 * Local APIC periodic timer handler. 126 */ 127 .text 128 SUPERALIGN_TEXT 129IDTVEC(timerint) 130 PUSH_FRAME 131 FAKE_MCOUNT(TF_RIP(%rsp)) 132 movq %rsp, %rdi 133 call lapic_handle_timer 134 MEXITCOUNT 135 jmp doreti 136 137/* 138 * Local APIC CMCI handler. 139 */ 140 .text 141 SUPERALIGN_TEXT 142IDTVEC(cmcint) 143 PUSH_FRAME 144 FAKE_MCOUNT(TF_RIP(%rsp)) 145 call lapic_handle_cmc 146 MEXITCOUNT 147 jmp doreti 148 149/* 150 * Local APIC error interrupt handler. 151 */ 152 .text 153 SUPERALIGN_TEXT 154IDTVEC(errorint) 155 PUSH_FRAME 156 FAKE_MCOUNT(TF_RIP(%rsp)) 157 call lapic_handle_error 158 MEXITCOUNT 159 jmp doreti 160 161#ifdef XENHVM 162/* 163 * Xen event channel upcall interrupt handler. 164 * Only used when the hypervisor supports direct vector callbacks. 165 */ 166 .text 167 SUPERALIGN_TEXT 168IDTVEC(xen_intr_upcall) 169 PUSH_FRAME 170 FAKE_MCOUNT(TF_RIP(%rsp)) 171 movq %rsp, %rdi 172 call xen_intr_handle_upcall 173 MEXITCOUNT 174 jmp doreti 175#endif 176 177#ifdef SMP 178/* 179 * Global address space TLB shootdown. 180 */ 181 .text 182 183#define NAKE_INTR_CS 24 184 185 SUPERALIGN_TEXT 186invltlb_ret: 187 call as_lapic_eoi 188 POP_FRAME 189 jmp doreti_iret 190 191 SUPERALIGN_TEXT 192IDTVEC(invltlb_pcid) 193 PUSH_FRAME 194 195 call invltlb_pcid_handler 196 jmp invltlb_ret 197 198 199 SUPERALIGN_TEXT 200IDTVEC(invltlb) 201 PUSH_FRAME 202 203 call invltlb_handler 204 jmp invltlb_ret 205 206/* 207 * Single page TLB shootdown 208 */ 209 .text 210 SUPERALIGN_TEXT 211IDTVEC(invlpg_pcid) 212 PUSH_FRAME 213 214 call invlpg_pcid_handler 215 jmp invltlb_ret 216 217 SUPERALIGN_TEXT 218IDTVEC(invlpg) 219 PUSH_FRAME 220 221 call invlpg_handler 222 jmp invltlb_ret 223 224/* 225 * Page range TLB shootdown. 226 */ 227 .text 228 SUPERALIGN_TEXT 229IDTVEC(invlrng) 230 PUSH_FRAME 231 232 call invlrng_handler 233 jmp invltlb_ret 234 235/* 236 * Invalidate cache. 237 */ 238 .text 239 SUPERALIGN_TEXT 240IDTVEC(invlcache) 241 PUSH_FRAME 242 243 call invlcache_handler 244 jmp invltlb_ret 245 246/* 247 * Handler for IPIs sent via the per-cpu IPI bitmap. 248 */ 249 .text 250 SUPERALIGN_TEXT 251IDTVEC(ipi_intr_bitmap_handler) 252 PUSH_FRAME 253 254 call as_lapic_eoi 255 256 FAKE_MCOUNT(TF_RIP(%rsp)) 257 258 call ipi_bitmap_handler 259 MEXITCOUNT 260 jmp doreti 261 262/* 263 * Executed by a CPU when it receives an IPI_STOP from another CPU. 264 */ 265 .text 266 SUPERALIGN_TEXT 267IDTVEC(cpustop) 268 PUSH_FRAME 269 270 call as_lapic_eoi 271 272 call cpustop_handler 273 jmp doreti 274 275/* 276 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 277 */ 278 .text 279 SUPERALIGN_TEXT 280IDTVEC(cpususpend) 281 PUSH_FRAME 282 283 call cpususpend_handler 284 call as_lapic_eoi 285 jmp doreti 286 287/* 288 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 289 * 290 * - Calls the generic rendezvous action function. 291 */ 292 .text 293 SUPERALIGN_TEXT 294IDTVEC(rendezvous) 295 PUSH_FRAME 296#ifdef COUNT_IPIS 297 movl PCPU(CPUID), %eax 298 movq ipi_rendezvous_counts(,%rax,8), %rax 299 incq (%rax) 300#endif 301 call smp_rendezvous_action 302 call as_lapic_eoi 303 jmp doreti 304 305/* 306 * IPI handler whose purpose is to interrupt the CPU with minimum overhead. 307 * This is used by bhyve to force a host cpu executing in guest context to 308 * trap into the hypervisor. 309 * 310 * This handler is different from other IPI handlers in the following aspects: 311 * 312 * 1. It doesn't push a trapframe on the stack. 313 * 314 * This implies that a DDB backtrace involving 'justreturn' will skip the 315 * function that was interrupted by this handler. 316 * 317 * 2. It doesn't 'swapgs' when userspace is interrupted. 318 * 319 * The 'justreturn' handler does not access any pcpu data so it is not an 320 * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI 321 * whose handler already doesn't trust GS.base when kernel code is interrupted. 322 */ 323 .text 324 SUPERALIGN_TEXT 325IDTVEC(justreturn) 326 pushq %rax 327 pushq %rcx 328 pushq %rdx 329 call as_lapic_eoi 330 popq %rdx 331 popq %rcx 332 popq %rax 333 jmp doreti_iret 334 335#endif /* SMP */ 336