1/* 2 * Copyright (c) 2008 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * --- 35 * 36 * Copyright (c) 1989, 1990 William F. Jolitz. 37 * Copyright (c) 1990 The Regents of the University of California. 38 * All rights reserved. 39 * 40 * This code is derived from software contributed to Berkeley by 41 * William Jolitz. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by the University of 54 * California, Berkeley and its contributors. 55 * 4. Neither the name of the University nor the names of its contributors 56 * may be used to endorse or promote products derived from this software 57 * without specific prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 60 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 61 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 62 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 63 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 64 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 65 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * @(#)ipl.s 72 * 73 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $ 74 */ 75 76#include <machine/asmacros.h> 77#include <machine/segments.h> 78#include <machine/ipl.h> 79#include <machine/lock.h> 80#include <machine/psl.h> 81#include <machine/trap.h> 82 83#include "assym.s" 84 85/* 86 * AT/386 87 * Vector interrupt control section 88 * 89 * fpending - Pending interrupts (set when a masked interrupt occurs) 90 * spending - Pending software interrupts 91 */ 92 .data 93 ALIGN_DATA 94 95 .globl fastunpend_count 96fastunpend_count: .long 0 97 98 .text 99 SUPERALIGN_TEXT 100 101 /* 102 * GENERAL NOTES 103 * 104 * - fast interrupts are always called with a critical section 105 * held 106 * 107 * - we release our critical section when scheduling interrupt 108 * or softinterrupt threads in order so they can preempt 109 * (unless we are called manually from a critical section, in 110 * which case there will still be a critical section and 111 * they won't preempt anyway). 112 * 113 * - TD_NEST_COUNT prevents splz from nesting too deeply within 114 * itself. It is *not* actually an interrupt nesting count. 115 * PCPU(intr_nesting_level) is an interrupt nesting count. 116 * 117 * - We have to be careful in regards to local interrupts 118 * occuring simultaniously with our doreti and splz 119 * processing. 120 * 121 * - Interrupts must be enabled when calling higher level 122 * functions in order to avoid deadlocking against things 123 * like smp_invltlb. 124 */ 125 126 /* 127 * DORETI 128 * 129 * Handle return from interrupts, traps and syscalls. This function 130 * checks the cpl for unmasked pending interrupts (fast, normal, or 131 * soft) and schedules them if appropriate, then irets. 132 * 133 * If we are in a critical section we cannot run any pending ints. 134 * 135 * The stack contains a trapframe at the start of doreti. 136 */ 137 SUPERALIGN_TEXT 138 .globl doreti 139 .type doreti,@function 140doreti: 141 FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */ 142 movq $0,%rax /* irq mask unavailable due to BGL */ 143 movq PCPU(curthread),%rbx 144 cli /* interlock with critical section */ 145 cmpl $0,PCPU(reqflags) /* short cut if nothing to do */ 146 je 5f 147 testl $-1,TD_CRITCOUNT(%rbx) /* can't unpend if in critical sec */ 148 jne 5f 149 incl TD_CRITCOUNT(%rbx) /* force all ints to pending */ 150doreti_next: 151 cli /* re-assert cli on loop */ 152 movl %eax,%ecx /* irq mask unavailable due to BGL */ 153 notl %ecx 154#ifdef SMP 155 testl $RQF_IPIQ,PCPU(reqflags) 156 jnz doreti_ipiq 157 testl $RQF_TIMER,PCPU(reqflags) 158 jnz doreti_timer 159#endif 160 testl PCPU(fpending),%ecx /* check for an unmasked fast int */ 161 jnz doreti_fast 162 163 movl PCPU(spending),%ecx /* check for a pending software int */ 164 cmpl $0,%ecx 165 jnz doreti_soft 166 167 testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */ 168 jz 2f 169 170 /* ASTs are only applicable when returning to userland */ 171 testb $SEL_RPL_MASK,TF_CS(%rsp) 172 jnz doreti_ast 1732: 174 /* 175 * Nothing left to do, finish up. Interrupts are still disabled. 176 * %eax contains the mask of IRQ's that are not available due to 177 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending 178 * interrupts have been processed. 179 */ 180 decl TD_CRITCOUNT(%rbx) /* interlocked with cli */ 181 testl %eax,%eax 182 jnz 5f 183 andl $~RQF_INTPEND,PCPU(reqflags) 1845: 185 MEXITCOUNT 186 187 /* 188 * (interrupts are disabled here) 189 * 190 * Restore register and iret. iret can fault on %rip (which is 191 * really stupid). If this occurs we re-fault and vector to 192 * doreti_iret_fault(). 193 * 194 * ... 195 * can be set from user mode, this can result in a kernel mode 196 * exception. The trap code will revector to the *_fault code 197 * which then sets up a T_PROTFLT signal. If the signal is 198 * sent to userland, sendsig() will automatically clean up all 199 * the segment registers to avoid a loop. 200 */ 201 .globl doreti_iret 202 .globl doreti_syscall_ret 203doreti_syscall_ret: 204 POP_FRAME /* registers and %gs (+cli) */ 205 /* WARNING: special global doreti_iret is also used by exception.S */ 206doreti_iret: 207 iretq 208 209 /* 210 * doreti_iret_fault. Alternative return code for the case where 211 * we get a fault in the doreti_exit code above. trap() 212 * (sys/platform/pc64/x86_64/trap.c) catches this specific * case, 213 * sends the process a signal and continues in the corresponding 214 * place in the code below. 215 * 216 * Interrupts are likely disabled due to the above interlock 217 * between cli/iretq. We must enable them before calling any 218 * high level function. 219 */ 220 ALIGN_TEXT 221 .globl doreti_iret_fault 222doreti_iret_fault: 223 PUSH_FRAME_NOSWAP 224 sti 225 movq $T_PROTFLT,TF_TRAPNO(%rsp) 226 movq $0,TF_ERR(%rsp) /* XXX should be the error code */ 227 movq $0,TF_ADDR(%rsp) 228 FAKE_MCOUNT(TF_RIP(%rsp)) 229 jmp calltrap 230 231 /* 232 * FAST interrupt pending. NOTE: stack context holds frame structure 233 * for fast interrupt procedure, do not do random pushes or pops! 234 */ 235 ALIGN_TEXT 236doreti_fast: 237 andl PCPU(fpending),%ecx /* only check fast ints */ 238 sti 239 bsfl %ecx, %ecx /* locate the next dispatchable int */ 240 btrl %ecx, PCPU(fpending) /* is it really still pending? */ 241 jnc doreti_next 242 pushq %rax /* save IRQ mask unavailable for BGL */ 243 /* NOTE: is also CPL in frame */ 244 call dofastunpend /* unpend fast intr %ecx */ 245 popq %rax 246 jmp doreti_next 247 248 /* 249 * SOFT interrupt pending 250 * 251 * Temporarily back-out our critical section to allow an interrupt 252 * preempt us when we schedule it. Bump intr_nesting_level to 253 * prevent the switch code from recursing via splz too deeply. 254 */ 255 ALIGN_TEXT 256doreti_soft: 257 sti 258 bsfl %ecx,%ecx /* locate the next pending softint */ 259 btrl %ecx,PCPU(spending) /* make sure its still pending */ 260 jnc doreti_next 261 addl $FIRST_SOFTINT,%ecx /* actual intr number */ 262 pushq %rax 263 movl %ecx,%edi /* argument to C call */ 264 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */ 265 decl TD_CRITCOUNT(%rbx) /* so we can preempt */ 266 call sched_ithd /* YYY must pull in imasks */ 267 incl TD_CRITCOUNT(%rbx) 268 decl TD_NEST_COUNT(%rbx) 269 popq %rax 270 jmp doreti_next 271 272 /* 273 * AST pending. We clear RQF_AST_SIGNAL automatically, the others 274 * are cleared by the trap as they are processed. 275 * 276 * Temporarily back-out our critical section because trap() can be 277 * a long-winded call, and we want to be more syscall-like. 278 * 279 * YYY theoretically we can call lwkt_switch directly if all we need 280 * to do is a reschedule. 281 */ 282doreti_ast: 283 andl $~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags) 284 sti 285 movl %eax,%r12d /* save cpl (can't use stack) */ 286 movl $T_ASTFLT,TF_TRAPNO(%rsp) 287 movq %rsp,%rdi /* pass frame by ref (%edi = C arg) */ 288 decl TD_CRITCOUNT(%rbx) 289 call trap 290 incl TD_CRITCOUNT(%rbx) 291 movl %r12d,%eax /* restore cpl for loop */ 292 jmp doreti_next 293 294#ifdef SMP 295 /* 296 * IPIQ message pending. We clear RQF_IPIQ automatically. 297 */ 298doreti_ipiq: 299 movl %eax,%r12d /* save cpl (can't use stack) */ 300 incl PCPU(intr_nesting_level) 301 andl $~RQF_IPIQ,PCPU(reqflags) 302 sti 303 subq $8,%rsp /* trapframe->intrframe */ 304 movq %rsp,%rdi /* pass frame by ref (C arg) */ 305 call lwkt_process_ipiq_frame 306 addq $8,%rsp /* intrframe->trapframe */ 307 decl PCPU(intr_nesting_level) 308 movl %r12d,%eax /* restore cpl for loop */ 309 jmp doreti_next 310 311doreti_timer: 312 movl %eax,%r12d /* save cpl (can't use stack) */ 313 incl PCPU(intr_nesting_level) 314 andl $~RQF_TIMER,PCPU(reqflags) 315 sti 316 subq $8,%rsp /* trapframe->intrframe */ 317 movq %rsp,%rdi /* pass frame by ref (C arg) */ 318 call lapic_timer_process_frame 319 addq $8,%rsp /* intrframe->trapframe */ 320 decl PCPU(intr_nesting_level) 321 movl %r12d,%eax /* restore cpl for loop */ 322 jmp doreti_next 323 324#endif 325 326 /* 327 * SPLZ() a C callable procedure to dispatch any unmasked pending 328 * interrupts regardless of critical section nesting. ASTs 329 * are not dispatched. 330 * 331 * Use %eax to track those IRQs that could not be processed 332 * due to BGL requirements. 333 */ 334 SUPERALIGN_TEXT 335 336ENTRY(splz) 337 pushfq 338 pushq %rbx 339 movq PCPU(curthread),%rbx 340 incl TD_CRITCOUNT(%rbx) 341 movl $0,%eax 342 343splz_next: 344 cli 345 movl %eax,%ecx /* ecx = ~CPL */ 346 notl %ecx 347#ifdef SMP 348 testl $RQF_IPIQ,PCPU(reqflags) 349 jnz splz_ipiq 350 testl $RQF_TIMER,PCPU(reqflags) 351 jnz splz_timer 352#endif 353 testl PCPU(fpending),%ecx /* check for an unmasked fast int */ 354 jnz splz_fast 355 356 movl PCPU(spending),%ecx 357 cmpl $0,%ecx 358 jnz splz_soft 359 360 decl TD_CRITCOUNT(%rbx) 361 362 /* 363 * Nothing left to do, finish up. Interrupts are still disabled. 364 * If our mask of IRQs we couldn't process due to BGL requirements 365 * is 0 then there are no pending interrupt sources left and we 366 * can clear RQF_INTPEND. 367 */ 368 testl %eax,%eax 369 jnz 5f 370 andl $~RQF_INTPEND,PCPU(reqflags) 3715: 372 popq %rbx 373 popfq 374 ret 375 376 /* 377 * FAST interrupt pending 378 */ 379 ALIGN_TEXT 380splz_fast: 381 andl PCPU(fpending),%ecx /* only check fast ints */ 382 sti 383 bsfl %ecx, %ecx /* locate the next dispatchable int */ 384 btrl %ecx, PCPU(fpending) /* is it really still pending? */ 385 jnc splz_next 386 pushq %rax 387 call dofastunpend /* unpend fast intr %ecx */ 388 popq %rax 389 jmp splz_next 3901: 391 btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */ 392 popq %rax 393 orl PCPU(fpending),%eax 394 jmp splz_next 395 396 /* 397 * SOFT interrupt pending 398 * 399 * Temporarily back-out our critical section to allow the interrupt 400 * preempt us. 401 */ 402 ALIGN_TEXT 403splz_soft: 404 sti 405 bsfl %ecx,%ecx /* locate the next pending softint */ 406 btrl %ecx,PCPU(spending) /* make sure its still pending */ 407 jnc splz_next 408 addl $FIRST_SOFTINT,%ecx /* actual intr number */ 409 sti 410 pushq %rax 411 movl %ecx,%edi /* C argument */ 412 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */ 413 decl TD_CRITCOUNT(%rbx) 414 call sched_ithd /* YYY must pull in imasks */ 415 incl TD_CRITCOUNT(%rbx) 416 decl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */ 417 popq %rax 418 jmp splz_next 419 420#ifdef SMP 421splz_ipiq: 422 andl $~RQF_IPIQ,PCPU(reqflags) 423 sti 424 pushq %rax 425 call lwkt_process_ipiq 426 popq %rax 427 jmp splz_next 428 429splz_timer: 430 andl $~RQF_TIMER,PCPU(reqflags) 431 sti 432 pushq %rax 433 call lapic_timer_process 434 popq %rax 435 jmp splz_next 436#endif 437 438 /* 439 * dofastunpend(%ecx:intr) 440 * 441 * A FAST interrupt previously made pending can now be run, 442 * execute it by pushing a dummy interrupt frame and 443 * calling ithread_fast_handler to execute or schedule it. 444 * 445 * ithread_fast_handler() returns 0 if it wants us to unmask 446 * further interrupts. 447 */ 448#define PUSH_DUMMY \ 449 pushfq ; /* phys int frame / flags */ \ 450 movl %cs,%eax ; \ 451 pushq %rax ; /* phys int frame / cs */ \ 452 pushq 3*8(%rsp) ; /* original caller eip */ \ 453 subq $TF_RIP,%rsp ; /* trap frame */ \ 454 movq $0,TF_XFLAGS(%rsp) ; /* extras */ \ 455 movq $0,TF_TRAPNO(%rsp) ; /* extras */ \ 456 movq $0,TF_ADDR(%rsp) ; /* extras */ \ 457 movq $0,TF_FLAGS(%rsp) ; /* extras */ \ 458 movq $0,TF_ERR(%rsp) ; /* extras */ \ 459 460#define POP_DUMMY \ 461 addq $TF_RIP+(3*8),%rsp ; \ 462 463dofastunpend: 464 pushq %rbp /* frame for backtrace */ 465 movq %rsp,%rbp 466 PUSH_DUMMY 467 pushq %rcx /* last part of intrframe = intr */ 468 incl fastunpend_count 469 movq %rsp,%rdi /* pass frame by reference C arg */ 470 call ithread_fast_handler /* returns 0 to unmask */ 471 popq %rdi /* intrframe->trapframe */ 472 /* + also rdi C arg to next call */ 473 cmpl $0,%eax 474 jnz 1f 475 movq MachIntrABI + MACHINTR_INTREN, %rax 476 callq *%rax /* MachIntrABI.intren(intr) */ 4771: 478 POP_DUMMY 479 popq %rbp 480 ret 481 482