1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 2014-2018 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by 9 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 10 * the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: vector.s, 386BSD 0.1 unknown origin 37 */ 38 39/* 40 * Interrupt entry points for external interrupts triggered by I/O APICs 41 * as well as IPI handlers. 42 */ 43 44#include "opt_smp.h" 45 46#include "assym.inc" 47 48#include <machine/asmacros.h> 49#include <machine/specialreg.h> 50#include <x86/apicreg.h> 51 52#ifdef SMP 53#define LK lock ; 54#else 55#define LK 56#endif 57 58 .text 59 SUPERALIGN_TEXT 60 /* End Of Interrupt to APIC */ 61as_lapic_eoi: 62 cmpl $0,x2apic_mode 63 jne 1f 64 movq lapic_map,%rax 65 movl $0,LA_EOI(%rax) 66 ret 671: 68 movl $MSR_APIC_EOI,%ecx 69 xorl %eax,%eax 70 xorl %edx,%edx 71 wrmsr 72 ret 73 74/* 75 * I/O Interrupt Entry Point. Rather than having one entry point for 76 * each interrupt source, we use one entry point for each 32-bit word 77 * in the ISR. The handler determines the highest bit set in the ISR, 78 * translates that into a vector, and passes the vector to the 79 * lapic_handle_intr() function. 80 */ 81 .macro ISR_VEC index, vec_name 82 INTR_HANDLER \vec_name 83 KMSAN_ENTER 84 cmpl $0,x2apic_mode 85 je 1f 86 movl $(MSR_APIC_ISR0 + \index),%ecx 87 rdmsr 88 jmp 2f 891: 90 movq lapic_map, %rdx /* pointer to local APIC */ 91 movl LA_ISR + 16 * (\index)(%rdx), %eax /* load ISR */ 922: 93 bsrl %eax, %eax /* index of highest set bit in ISR */ 94 jz 3f 95 addl $(32 * \index),%eax 96 movq %rsp, %rsi 97 movl %eax, %edi /* pass the IRQ */ 98 call lapic_handle_intr 993: 100 KMSAN_LEAVE 101 jmp doreti 102 .endm 103 104/* 105 * Handle "spurious INTerrupts". 106 * Notes: 107 * This is different than the "spurious INTerrupt" generated by an 108 * 8259 PIC for missing INTs. See the APIC documentation for details. 109 * This routine should NOT do an 'EOI' cycle. 110 */ 111 .text 112 SUPERALIGN_TEXT 113IDTVEC(spuriousint) 114 /* No EOI cycle used here */ 115 jmp doreti_iret 116 117 ISR_VEC 1, apic_isr1 118 ISR_VEC 2, apic_isr2 119 ISR_VEC 3, apic_isr3 120 ISR_VEC 4, apic_isr4 121 ISR_VEC 5, apic_isr5 122 ISR_VEC 6, apic_isr6 123 ISR_VEC 7, apic_isr7 124 125/* 126 * Local APIC periodic timer handler. 127 */ 128 INTR_HANDLER timerint 129 KMSAN_ENTER 130 movq %rsp, %rdi 131 call lapic_handle_timer 132 KMSAN_LEAVE 133 jmp doreti 134 135/* 136 * Local APIC CMCI handler. 137 */ 138 INTR_HANDLER cmcint 139 KMSAN_ENTER 140 call lapic_handle_cmc 141 KMSAN_LEAVE 142 jmp doreti 143 144/* 145 * Local APIC error interrupt handler. 146 */ 147 INTR_HANDLER errorint 148 KMSAN_ENTER 149 call lapic_handle_error 150 KMSAN_LEAVE 151 jmp doreti 152 153#ifdef XENHVM 154/* 155 * Xen event channel upcall interrupt handler. 156 * Only used when the hypervisor supports direct vector callbacks. 157 */ 158 INTR_HANDLER xen_intr_upcall 159 KMSAN_ENTER 160 movq %rsp, %rdi 161 call xen_arch_intr_handle_upcall 162 KMSAN_LEAVE 163 jmp doreti 164#endif 165 166#ifdef SMP 167/* 168 * Global address space TLB shootdown. 169 */ 170 .text 171 172 SUPERALIGN_TEXT 173/* 174 * IPI handler for cache and TLB shootdown 175 */ 176 INTR_HANDLER invlop 177 KMSAN_ENTER 178 call invlop_handler 179 call as_lapic_eoi 180 KMSAN_LEAVE 181 jmp ld_regs 182 183/* 184 * Handler for IPIs sent via the per-cpu IPI bitmap. 185 */ 186 INTR_HANDLER ipi_intr_bitmap_handler 187 call as_lapic_eoi 188 KMSAN_ENTER 189 call ipi_bitmap_handler 190 KMSAN_LEAVE 191 jmp doreti 192 193/* 194 * Executed by a CPU when it receives an IPI_STOP from another CPU. 195 */ 196 INTR_HANDLER cpustop 197 call as_lapic_eoi 198 KMSAN_ENTER 199 call cpustop_handler 200 KMSAN_LEAVE 201 jmp doreti 202 203/* 204 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 205 */ 206 INTR_HANDLER cpususpend 207 KMSAN_ENTER 208 call cpususpend_handler 209 call as_lapic_eoi 210 KMSAN_LEAVE 211 jmp doreti 212 213/* 214 * Executed by a CPU when it receives an IPI_SWI. 215 */ 216 INTR_HANDLER ipi_swi 217 call as_lapic_eoi 218 KMSAN_ENTER 219 call ipi_swi_handler 220 KMSAN_LEAVE 221 jmp doreti 222 223/* 224 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 225 * 226 * - Calls the generic rendezvous action function. 227 */ 228 INTR_HANDLER rendezvous 229#ifdef COUNT_IPIS 230 movl PCPU(CPUID), %eax 231 movq ipi_rendezvous_counts(,%rax,8), %rax 232 incq (%rax) 233#endif 234 KMSAN_ENTER 235 call smp_rendezvous_action 236 call as_lapic_eoi 237 KMSAN_LEAVE 238 jmp doreti 239 240/* 241 * IPI handler whose purpose is to interrupt the CPU with minimum overhead. 242 * This is used by bhyve to force a host cpu executing in guest context to 243 * trap into the hypervisor. 244 * 245 * This handler is different from other IPI handlers in the following aspects: 246 * 247 * 1. It doesn't push a trapframe on the stack. 248 * 249 * This implies that a DDB backtrace involving 'justreturn' will skip the 250 * function that was interrupted by this handler. 251 * 252 * 2. It doesn't 'swapgs' when userspace is interrupted. 253 * 254 * The 'justreturn' handler does not access any pcpu data so it is not an 255 * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI 256 * whose handler already doesn't trust GS.base when kernel code is interrupted. 257 */ 258 .text 259 SUPERALIGN_TEXT 260IDTVEC(justreturn) 261 pushq %rax 262 pushq %rcx 263 pushq %rdx 264 call as_lapic_eoi 265 popq %rdx 266 popq %rcx 267 popq %rax 268 jmp doreti_iret 269 270 INTR_HANDLER justreturn1 271 call as_lapic_eoi 272 jmp doreti 273 274#endif /* SMP */ 275