1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 2014-2018 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by 9 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 10 * the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: vector.s, 386BSD 0.1 unknown origin 37 * $FreeBSD$ 38 */ 39 40/* 41 * Interrupt entry points for external interrupts triggered by I/O APICs 42 * as well as IPI handlers. 43 */ 44 45#include "opt_smp.h" 46 47#include "assym.inc" 48 49#include <machine/asmacros.h> 50#include <machine/specialreg.h> 51#include <x86/apicreg.h> 52 53#ifdef SMP 54#define LK lock ; 55#else 56#define LK 57#endif 58 59 .text 60 SUPERALIGN_TEXT 61 /* End Of Interrupt to APIC */ 62as_lapic_eoi: 63 cmpl $0,x2apic_mode 64 jne 1f 65 movq lapic_map,%rax 66 movl $0,LA_EOI(%rax) 67 ret 681: 69 movl $MSR_APIC_EOI,%ecx 70 xorl %eax,%eax 71 xorl %edx,%edx 72 wrmsr 73 ret 74 75/* 76 * I/O Interrupt Entry Point. Rather than having one entry point for 77 * each interrupt source, we use one entry point for each 32-bit word 78 * in the ISR. The handler determines the highest bit set in the ISR, 79 * translates that into a vector, and passes the vector to the 80 * lapic_handle_intr() function. 81 */ 82 .macro ISR_VEC index, vec_name 83 INTR_HANDLER \vec_name 84 FAKE_MCOUNT(TF_RIP(%rsp)) 85 cmpl $0,x2apic_mode 86 je 1f 87 movl $(MSR_APIC_ISR0 + \index),%ecx 88 rdmsr 89 jmp 2f 901: 91 movq lapic_map, %rdx /* pointer to local APIC */ 92 movl LA_ISR + 16 * (\index)(%rdx), %eax /* load ISR */ 932: 94 bsrl %eax, %eax /* index of highest set bit in ISR */ 95 jz 3f 96 addl $(32 * \index),%eax 97 movq %rsp, %rsi 98 movl %eax, %edi /* pass the IRQ */ 99 call lapic_handle_intr 1003: 101 MEXITCOUNT 102 jmp doreti 103 .endm 104 105/* 106 * Handle "spurious INTerrupts". 107 * Notes: 108 * This is different than the "spurious INTerrupt" generated by an 109 * 8259 PIC for missing INTs. See the APIC documentation for details. 110 * This routine should NOT do an 'EOI' cycle. 111 */ 112 .text 113 SUPERALIGN_TEXT 114IDTVEC(spuriousint) 115 /* No EOI cycle used here */ 116 jmp doreti_iret 117 118 ISR_VEC 1, apic_isr1 119 ISR_VEC 2, apic_isr2 120 ISR_VEC 3, apic_isr3 121 ISR_VEC 4, apic_isr4 122 ISR_VEC 5, apic_isr5 123 ISR_VEC 6, apic_isr6 124 ISR_VEC 7, apic_isr7 125 126/* 127 * Local APIC periodic timer handler. 128 */ 129 INTR_HANDLER timerint 130 FAKE_MCOUNT(TF_RIP(%rsp)) 131 movq %rsp, %rdi 132 call lapic_handle_timer 133 MEXITCOUNT 134 jmp doreti 135 136/* 137 * Local APIC CMCI handler. 138 */ 139 INTR_HANDLER cmcint 140 FAKE_MCOUNT(TF_RIP(%rsp)) 141 call lapic_handle_cmc 142 MEXITCOUNT 143 jmp doreti 144 145/* 146 * Local APIC error interrupt handler. 147 */ 148 INTR_HANDLER errorint 149 FAKE_MCOUNT(TF_RIP(%rsp)) 150 call lapic_handle_error 151 MEXITCOUNT 152 jmp doreti 153 154#ifdef XENHVM 155/* 156 * Xen event channel upcall interrupt handler. 157 * Only used when the hypervisor supports direct vector callbacks. 158 */ 159 INTR_HANDLER xen_intr_upcall 160 FAKE_MCOUNT(TF_RIP(%rsp)) 161 movq %rsp, %rdi 162 call xen_intr_handle_upcall 163 MEXITCOUNT 164 jmp doreti 165#endif 166 167#ifdef SMP 168/* 169 * Global address space TLB shootdown. 170 */ 171 .text 172 173 SUPERALIGN_TEXT 174invltlb_ret: 175 call as_lapic_eoi 176 jmp ld_regs 177 178 SUPERALIGN_TEXT 179 INTR_HANDLER invltlb 180 call invltlb_handler 181 jmp invltlb_ret 182 183 INTR_HANDLER invltlb_pcid 184 call invltlb_pcid_handler 185 jmp invltlb_ret 186 187 INTR_HANDLER invltlb_invpcid_nopti 188 call invltlb_invpcid_handler 189 jmp invltlb_ret 190 191 INTR_HANDLER invltlb_invpcid_pti 192 call invltlb_invpcid_pti_handler 193 jmp invltlb_ret 194 195/* 196 * Single page TLB shootdown 197 */ 198 INTR_HANDLER invlpg 199 call invlpg_handler 200 jmp invltlb_ret 201 202 INTR_HANDLER invlpg_invpcid 203 call invlpg_invpcid_handler 204 jmp invltlb_ret 205 206 INTR_HANDLER invlpg_pcid 207 call invlpg_pcid_handler 208 jmp invltlb_ret 209 210/* 211 * Page range TLB shootdown. 212 */ 213 INTR_HANDLER invlrng 214 call invlrng_handler 215 jmp invltlb_ret 216 217 INTR_HANDLER invlrng_invpcid 218 call invlrng_invpcid_handler 219 jmp invltlb_ret 220 221 INTR_HANDLER invlrng_pcid 222 call invlrng_pcid_handler 223 jmp invltlb_ret 224 225/* 226 * Invalidate cache. 227 */ 228 INTR_HANDLER invlcache 229 call invlcache_handler 230 jmp invltlb_ret 231 232/* 233 * Handler for IPIs sent via the per-cpu IPI bitmap. 234 */ 235 INTR_HANDLER ipi_intr_bitmap_handler 236 call as_lapic_eoi 237 FAKE_MCOUNT(TF_RIP(%rsp)) 238 call ipi_bitmap_handler 239 MEXITCOUNT 240 jmp doreti 241 242/* 243 * Executed by a CPU when it receives an IPI_STOP from another CPU. 244 */ 245 INTR_HANDLER cpustop 246 call as_lapic_eoi 247 call cpustop_handler 248 jmp doreti 249 250/* 251 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 252 */ 253 INTR_HANDLER cpususpend 254 call cpususpend_handler 255 call as_lapic_eoi 256 jmp doreti 257 258/* 259 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 260 * 261 * - Calls the generic rendezvous action function. 262 */ 263 INTR_HANDLER rendezvous 264#ifdef COUNT_IPIS 265 movl PCPU(CPUID), %eax 266 movq ipi_rendezvous_counts(,%rax,8), %rax 267 incq (%rax) 268#endif 269 call smp_rendezvous_action 270 call as_lapic_eoi 271 jmp doreti 272 273/* 274 * IPI handler whose purpose is to interrupt the CPU with minimum overhead. 275 * This is used by bhyve to force a host cpu executing in guest context to 276 * trap into the hypervisor. 277 * 278 * This handler is different from other IPI handlers in the following aspects: 279 * 280 * 1. It doesn't push a trapframe on the stack. 281 * 282 * This implies that a DDB backtrace involving 'justreturn' will skip the 283 * function that was interrupted by this handler. 284 * 285 * 2. It doesn't 'swapgs' when userspace is interrupted. 286 * 287 * The 'justreturn' handler does not access any pcpu data so it is not an 288 * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI 289 * whose handler already doesn't trust GS.base when kernel code is interrupted. 290 */ 291 .text 292 SUPERALIGN_TEXT 293IDTVEC(justreturn) 294 pushq %rax 295 pushq %rcx 296 pushq %rdx 297 call as_lapic_eoi 298 popq %rdx 299 popq %rcx 300 popq %rax 301 jmp doreti_iret 302 303 INTR_HANDLER justreturn1 304 call as_lapic_eoi 305 jmp doreti 306 307#endif /* SMP */ 308