xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision 9452b29c) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26
27EXTERN KdSetOwedBreakpoints:PROC
28
29
30/* Helper Macros *************************************************************/
31
32MACRO(DispatchException, Status, Number, P1, P2, P3)
33    mov eax, Status
34    mov edx, Number
35    mov r9, P1
36    mov r10, P2
37    mov r11, P3
38    call InternalDispatchException
39ENDM
40
41MACRO(Fatal, BugcheckCode)
42    /* Bugcheck */
43    mov ecx, BugcheckCode
44    mov rdx, rbp
45    call KiSystemFatalException
46ENDM
47
48
49/* FUNCTIONS *****************************************************************/
50
51.code64
52
53ALIGN 8
54
55MACRO(UnexpectedVectorStub, Vector)
56    /* This nop is to make the relative jmp address 4 bytes aligned and to
57       make the whole code 8 bytes long */
58    nop
59    /* This is a push instruction with 8bit operand. Since the instruction
60       sign extends the value to 32 bits, we need to offset it */
61PUBLIC KxUnexpectedInterrupt&Vector
62KxUnexpectedInterrupt&Vector:
63    push (Vector - 128)
64    jmp KiUnexpectedInterrupt
65ENDM
66
67PUBLIC KiUnexpectedRange
68KiUnexpectedRange:
69Vector = 0
70REPEAT 256
71    UnexpectedVectorStub %Vector
72    Vector = Vector+1
73ENDR
74PUBLIC KiUnexpectedRangeEnd
75KiUnexpectedRangeEnd:
76
77PUBLIC KiInterruptDispatchTemplate
78KiInterruptDispatchTemplate:
79    /* This instruction pushes the return address on the stack, which is the
80       address of the interrupt object's DispatchCode member, then jumps
81       to the address stored in the interrupt object's DispatchAddress member */
82    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
83
84
85// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
86FUNC InternalDispatchException
87
88    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
89    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
90    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
91    .endprolog
92
93    /* Set up EXCEPTION_RECORD */
94    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
95    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
96    xor rax, rax
97    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
98    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
99    mov rax, [rbp + KTRAP_FRAME_Rip]
100    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
101    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
102    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
105
106    /* Set up KEXCEPTION_FRAME */
107    mov rax, [rbp + KTRAP_FRAME_Rbp]
108    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
109    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
110    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
111    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
112    mov [rsp + KEXCEPTION_FRAME_R12], r12
113    mov [rsp + KEXCEPTION_FRAME_R13], r13
114    mov [rsp + KEXCEPTION_FRAME_R14], r14
115    mov [rsp + KEXCEPTION_FRAME_R15], r15
116    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
126    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
127
128    /* Call KiDispatchException */
129    // rcx already points to ExceptionRecord
130    mov rdx, rsp                                // ExceptionFrame
131    mov r8, rbp                                 // TrapFrame
132    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
133    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
134    call KiDispatchException
135
136    /* Restore registers */
137    mov r12, [rsp + KEXCEPTION_FRAME_R12]
138    mov r13, [rsp + KEXCEPTION_FRAME_R13]
139    mov r14, [rsp + KEXCEPTION_FRAME_R14]
140    mov r15, [rsp + KEXCEPTION_FRAME_R15]
141    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
142    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
143    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
144    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
145    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
146    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
147    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
148    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
149    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
150    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
151
152    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
153    ret
154ENDFUNC
155
156
157/* CPU EXCEPTION HANDLERS ****************************************************/
158
159PUBLIC KiDivideErrorFault
160FUNC KiDivideErrorFault
161    /* Push pseudo error code */
162    EnterTrap TF_SAVE_ALL
163
164    /* Enable interrupts */
165    sti
166
167    /* Dispatch the exception */
168    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
169
170    /* Return */
171    ExitTrap TF_SAVE_ALL
172ENDFUNC
173
174
175PUBLIC KiDebugTrapOrFault
176FUNC KiDebugTrapOrFault
177    /* Push pseudo error code */
178    EnterTrap TF_SAVE_ALL
179
180    /* Check if the frame was from kernelmode */
181    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
182    jz KiDebugTrapOrFaultKMode
183
184    /* Enable interrupts for user-mode */
185    sti
186
187KiDebugTrapOrFaultKMode:
188    /* Dispatch the exception */
189    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
190
191    /* Return */
192    ExitTrap TF_SAVE_ALL
193ENDFUNC
194
195
196PUBLIC KiNmiInterrupt
197FUNC KiNmiInterrupt
198    /* Push pseudo error code */
199    EnterTrap TF_SAVE_ALL
200
201    UNIMPLEMENTED KiNmiInterrupt
202    int 3
203
204    /* Return */
205    ExitTrap TF_SAVE_ALL
206ENDFUNC
207
208
209PUBLIC KiBreakpointTrap
210FUNC KiBreakpointTrap
211    /* Push pseudo error code */
212    EnterTrap TF_SAVE_ALL
213
214    /* Check if the frame was from kernelmode */
215    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
216    jz KiBreakpointTrapKMode
217
218    /* Enable interrupts for user-mode */
219    sti
220
221KiBreakpointTrapKMode:
222    /* Dispatch the exception */
223    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
224
225    /* Return */
226    ExitTrap TF_SAVE_ALL
227ENDFUNC
228
229
230PUBLIC KiOverflowTrap
231FUNC KiOverflowTrap
232    /* Push pseudo error code */
233    EnterTrap TF_SAVE_ALL
234
235    /* Enable interrupts */
236    sti
237
238    /* Dispatch the exception */
239    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
240
241    /* Return */
242    ExitTrap TF_SAVE_ALL
243ENDFUNC
244
245
246PUBLIC KiBoundFault
247FUNC KiBoundFault
248    /* Push pseudo error code */
249    EnterTrap TF_SAVE_ALL
250
251    /* Check if the frame was from kernelmode */
252    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
253    jnz KiBoundFaultUserMode
254
255    /* Bugcheck */
256    Fatal EXCEPTION_BOUND_CHECK
257
258KiBoundFaultUserMode:
259    /* Enable interrupts for user-mode */
260    sti
261
262    /* Dispatch the exception */
263    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
264
265    /* Return */
266    ExitTrap TF_SAVE_ALL
267ENDFUNC
268
269
270PUBLIC KiInvalidOpcodeFault
271FUNC KiInvalidOpcodeFault
272    /* Push pseudo error code */
273    EnterTrap TF_SAVE_ALL
274
275    /* Enable interrupts */
276    sti
277
278    /* Check if the frame was from kernelmode */
279    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
280    jz KiInvalidOpcodeKernel
281
282    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
283
284KiInvalidOpcodeKernel:
285    /* Kernel mode fault */
286
287    /* Dispatch the exception */
288    DispatchException STATUS_ILLEGAL_INSTRUCTION, 0, 0, 0, 0
289
290    /* Return */
291    ExitTrap TF_SAVE_ALL
292ENDFUNC
293
294
295PUBLIC KiNpxNotAvailableFault
296FUNC KiNpxNotAvailableFault
297    /* Push pseudo error code */
298    EnterTrap TF_SAVE_ALL
299
300    /* Call the C handler */
301    mov rcx, rbp
302    call KiNpxNotAvailableFaultHandler
303
304    /* Check the return status code */
305    test eax, eax
306    jz KiNpxNotAvailableFaultExit
307
308    /* Dispatch the exception */
309    DispatchException eax, 3, 0, 0, 0
310
311KiNpxNotAvailableFaultExit:
312    /* Return */
313    ExitTrap TF_SAVE_ALL
314ENDFUNC
315
316
317PUBLIC KiDoubleFaultAbort
318FUNC KiDoubleFaultAbort
319
320    /* Hack for VBox, which "forgets" to push an error code on the stack! */
321    and rsp, HEX(FFFFFFFFFFFFFFF0)
322
323    /* A zero error code is pushed */
324    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
325
326    int 3
327
328    /* Bugcheck */
329    Fatal 8 // EXCEPTION_DOUBLE_FAULT
330    jmp $
331ENDFUNC
332
333
334PUBLIC KiNpxSegmentOverrunAbort
335FUNC KiNpxSegmentOverrunAbort
336    /* Push pseudo error code */
337    EnterTrap TF_SAVE_ALL
338
339    /* Bugcheck */
340    Fatal EXCEPTION_NPX_OVERRUN
341
342    jmp $
343ENDFUNC
344
345
346PUBLIC KiInvalidTssFault
347FUNC KiInvalidTssFault
348    /* We have an error code */
349    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
350
351    /* Bugcheck */
352    Fatal EXCEPTION_INVALID_TSS
353    jmp $
354ENDFUNC
355
356
357PUBLIC KiSegmentNotPresentFault
358FUNC KiSegmentNotPresentFault
359    /* We have an error code */
360    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
361
362    /* Bugcheck */
363    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
364    jmp $
365ENDFUNC
366
367
368PUBLIC KiStackFault
369FUNC KiStackFault
370    /* We have an error code */
371    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
372
373    /* Bugcheck */
374    Fatal EXCEPTION_STACK_FAULT
375    jmp $
376ENDFUNC
377
378
379PUBLIC KiGeneralProtectionFault
380FUNC KiGeneralProtectionFault
381    /* We have an error code */
382    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
383
384    /* Call the C handler */
385    mov rcx, rbp
386    call KiGeneralProtectionFaultHandler
387
388    /* Check for success */
389    test eax, eax
390    jge KiGpfExit
391
392    /* Check for access violation */
393    cmp eax, STATUS_ACCESS_VIOLATION
394    je DispatchAccessViolation
395
396    /* Dispatch privileged instruction fault */
397    DispatchException eax, 0, 0, 0, 0
398    jmp KiGpfExit
399
400DispatchAccessViolation:
401
402    /* Dispatch access violation */
403    DispatchException eax, 2, 0, -1, 0
404
405KiGpfExit:
406    /* Return */
407    ExitTrap TF_SAVE_ALL
408ENDFUNC
409
410
411PUBLIC KiPageFault
412FUNC KiPageFault
413    /* We have an error code */
414    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
415
416    /* Save page fault address */
417    mov rdx, cr2
418    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
419
420    /* If interrupts are off, do not enable them */
421    test dword ptr [rbp + KTRAP_FRAME_EFlags], EFLAGS_IF_MASK
422    jz IntsDisabled
423
424    /* Enable interrupts for the page fault handler */
425    sti
426
427IntsDisabled:
428
429    /* Call page fault handler */
430    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
431    // rdx == Address
432    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
433    and r8b, 1
434    mov r9, rbp // TrapInformation
435    call MmAccessFault
436
437    /* Check for success */
438    test eax, eax
439    jl PageFaultError
440
441    /* Check whether the kernel debugger has owed breakpoints to be inserted */
442    call KdSetOwedBreakpoints
443    /* We succeeded, return */
444    jmp PageFaultReturn
445
446PageFaultError:
447
448    /* Set parameter 1 to error code */
449    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
450
451    /* Set parameter 2 to faulting address */
452    mov r10, cr2  // Param2 = faulting address
453
454    cmp eax, STATUS_ACCESS_VIOLATION
455    je AccessViolation
456    cmp eax, STATUS_GUARD_PAGE_VIOLATION
457    je SpecialCode
458    cmp eax, STATUS_STACK_OVERFLOW
459    je SpecialCode
460
461InPageException:
462    /* Dispatch in-page exception */
463    mov r11d, eax // Param3 = Status
464    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
465    mov edx, 3    // ParamCount
466    call InternalDispatchException
467    jmp PageFaultReturn
468
469AccessViolation:
470    /* Use more proper status code */
471    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
472
473SpecialCode:
474    /* Setup a normal page fault exception */
475    mov edx, 2   // ParamCount
476    call InternalDispatchException
477
478PageFaultReturn:
479
480    /* Disable interrupts for the return */
481    cli
482
483    /* Return */
484    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
485ENDFUNC
486
487
488PUBLIC KiFloatingErrorFault
489FUNC KiFloatingErrorFault
490    /* Push pseudo error code */
491    EnterTrap TF_SAVE_ALL
492
493    UNIMPLEMENTED KiFloatingErrorFault
494    int 3
495
496    /* Return */
497    ExitTrap TF_SAVE_ALL
498ENDFUNC
499
500
501PUBLIC KiAlignmentFault
502FUNC KiAlignmentFault
503    /* We have an error code */
504    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
505
506    /* Bugcheck */
507    Fatal EXCEPTION_ALIGNMENT_CHECK
508    jmp $
509ENDFUNC
510
511
512PUBLIC KiMcheckAbort
513FUNC KiMcheckAbort
514    /* Push pseudo error code */
515    EnterTrap TF_SAVE_ALL
516
517    /* Bugcheck */
518    Fatal HEX(12)
519    jmp $
520ENDFUNC
521
522
523PUBLIC KiXmmException
524FUNC KiXmmException
525    /* Push pseudo error code */
526    EnterTrap TF_SAVE_ALL
527
528    /* Call the C handler */
529    mov rcx, rbp
530    call KiXmmExceptionHandler
531
532    /* Check for success */
533    test eax, eax
534    jge KiXmmExit
535
536    /* Dispatch the exception */
537    DispatchException eax, 2, 0, [rbp+KTRAP_FRAME_MxCsr], 0
538
539    // FIXME: STATUS_FLOAT_MULTIPLE_TRAPS / STATUS_FLOAT_MULTIPLE_FAULTS
540
541KiXmmExit:
542    /* Return */
543    ExitTrap TF_SAVE_ALL
544ENDFUNC
545
546
547/* SOFTWARE INTERRUPT SERVICES ***********************************************/
548
549PUBLIC KiRaiseAssertion
550FUNC KiRaiseAssertion
551    /* We have an error code */
552    EnterTrap (TF_SAVE_ALL)
553
554    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
555    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
556
557    /* Dispatch the exception */
558    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
559
560    /* Return */
561    ExitTrap TF_SAVE_ALL
562ENDFUNC
563
564
565PUBLIC KiDebugServiceTrap
566FUNC KiDebugServiceTrap
567   /* No error code */
568    EnterTrap TF_SAVE_ALL
569
570    /* Increase Rip to skip the int3 */
571    inc qword ptr [rbp + KTRAP_FRAME_Rip]
572
573    /* Dispatch the exception (Params = service, buffer, legth) */
574    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
575
576    /* Return */
577    ExitTrap TF_SAVE_ALL
578ENDFUNC
579
580
581PUBLIC KiApcInterrupt
582.PROC KiApcInterrupt
583   /* No error code */
584    EnterTrap (TF_SAVE_ALL or TF_IRQL)
585
586    /* Raise to APC_LEVEL */
587    mov rax, APC_LEVEL
588    mov cr8, rax
589
590    /* End the interrupt */
591    mov dword ptr [APIC_EOI], 0
592
593    /* Enable interrupts  */
594    sti
595
596    /* Call the worker routine */
597    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
598    and cl, 1
599    mov rdx, 0                        // ExceptionFrame
600    mov r8, rbp                       // TrapFrame
601    call KiDeliverApc
602
603    /* Disable interrupts */
604    cli
605
606    /* Lower IRQL back to PASSIVE */
607    mov rax, PASSIVE_LEVEL
608    mov cr8, rax
609
610    /* Return */
611    ExitTrap (TF_SAVE_ALL or TF_IRQL)
612.ENDP
613
614/*
615 * VOID
616 * KiRetireDpcList(
617 *     PKPRCB Prcb);
618 */
619EXTERN KiRetireDpcList:PROC
620
621/*
622 * VOID
623 * KiRetireDpcListInDpcStack(
624 *     PKPRCB Prcb,
625 *     PVOID DpcStack);
626 */
627PUBLIC KiRetireDpcListInDpcStack
628.PROC KiRetireDpcListInDpcStack
629    push rbp
630    .pushreg rbp
631    mov rbp, rsp
632    .setframe rbp, 0
633    .endprolog
634
635    /* Switch to the DpcStack */
636    mov rsp, rdx
637
638    /* The stack is 16 byte aligned, allocate 32 bytes home space */
639    sub rsp, 32
640
641    /* Call KiRetireDpcList on the given stack */
642    call KiRetireDpcList
643
644    /* Restore stack, cleanup and return */
645    mov rsp, rbp
646    pop rbp
647    ret
648.ENDP
649
650PUBLIC KiDpcInterrupt
651.PROC KiDpcInterrupt
652   /* No error code */
653    EnterTrap (TF_SAVE_ALL or TF_IRQL)
654
655    /* Call the worker routine */
656    call KiDpcInterruptHandler
657
658    /* Return, but don't send an EOI! */
659    ExitTrap (TF_SAVE_ALL or TF_IRQL)
660.ENDP
661
662
663PUBLIC KiIpiInterrupt
664.PROC KiIpiInterrupt
665   /* No error code */
666    EnterTrap (TF_SAVE_ALL or TF_IRQL)
667
668    /* Raise to IPI_LEVEL */
669    mov rax, IPI_LEVEL
670    mov cr8, rax
671
672    /* End the interrupt */
673    mov dword ptr [APIC_EOI], 0
674
675    int 3
676
677    /* Return */
678    ExitTrap (TF_SAVE_ALL or TF_IRQL)
679.ENDP
680
681
682PUBLIC KiUnexpectedInterrupt
683FUNC KiUnexpectedInterrupt
684    /* The error code is the vector */
685    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
686
687#if 0
688    /* Set bugcheck parameters */
689    mov ecx, TRAP_CAUSE_UNKNOWN
690    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
691    mov r8, 0 // The unknown floating-point exception
692    mov r9, 0 // The enabled and asserted status bits
693    sub rsp, 8
694    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
695    call KeBugCheckWithTf
696    jmp $
697#endif
698    /* Return */
699    ExitTrap TF_SAVE_ALL
700ENDFUNC
701
702PUBLIC KiInterruptDispatch
703FUNC KiInterruptDispatch
704    /* The error code is a pointer to the interrupt object's code */
705    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
706
707    /* Increase interrupt count */
708    inc dword ptr gs:[PcInterruptCount];
709
710    /* Save rbx and rsi in the trap frame */
711    mov [rbp + KTRAP_FRAME_Rbx], rbx
712    mov [rbp + KTRAP_FRAME_Rsi], rsi
713
714    /* Load the address of the dispatch code into rbx */
715    mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
716
717    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
718    sub rbx, KINTERRUPT_DispatchCode + 6
719
720    /* Save the address of the InterruptListEntry in rsi */
721    lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
722
723.DoDispatchInterrupt:
724    /* Raise IRQL to SynchronizeIrql */
725    movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
726    mov cr8, rax
727
728#ifdef CONFIG_SMP
729    /* Acquire interrupt lock */
730    mov r8, [rbx + KINTERRUPT_ActualLock]
731
732    //KxAcquireSpinLock(Interrupt->ActualLock);
733#endif
734
735    /* Call the ISR */
736    mov rcx, rbx
737    mov rdx, [rbx + KINTERRUPT_ServiceContext]
738    call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
739
740#ifdef CONFIG_SMP
741    /* Release interrupt lock */
742    //KxReleaseSpinLock(Interrupt->ActualLock);
743#endif
744
745    /* Go back to old irql */
746    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
747    mov cr8, rax
748
749    /* Check for chained interrupts */
750    mov rax, [rbx + KINTERRUPT_InterruptListEntry]
751    cmp rax, rsi
752    je .Done
753
754    /* Load the next interrupt object into rbx and repeat */
755    lea rbx, [rax - KINTERRUPT_InterruptListEntry]
756    jmp .DoDispatchInterrupt
757
758.Done:
759    /* Restore rbx and rsi */
760    mov rbx, [rbp + KTRAP_FRAME_Rbx]
761    mov rsi, [rbp + KTRAP_FRAME_Rsi]
762
763    /* Return */
764    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
765ENDFUNC
766
767EXTERN KiSystemCallHandler:PROC
768
769/*! \name KiSystemCallEntry64
770 *
771 *  \brief This is the entrypoint for syscalls from 64bit user mode
772 *
773 *  \param rax - The system call number
774 *  \param rcx - User mode return address, set by the syscall instruction
775 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
776 *  \param r10 - Parameter 1 to the service function
777 *  \param r11 - RFLAGS saved by the syscall instruction
778 *--*/
779PUBLIC KiSystemCallEntry64
780.PROC KiSystemCallEntry64
781    /* The unwind info pretends we have a machine frame */
782    .PUSHFRAME
783    .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
784    .SAVEREG rbp, MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp
785    .ENDPROLOG
786
787    /* Swap gs to kernel, so we can access the PCR */
788    swapgs
789
790    /* Save the user mode rsp in the PCR */
791    mov gs:[PcUserRsp], rsp
792
793    /* Get the kernel stack from the PCR */
794    mov rsp, gs:[PcRspBase]
795
796    /* Allocate a TRAP_FRAME and space for parameters */
797    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
798
799    /* Save volatile registers and rbp in the trap frame */
800    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
801    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
802    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
803    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
804    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
805    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
806    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
807    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
808
809    /* Store user stack pointer in the trap frame */
810    mov rax, gs:[PcUserRsp]
811    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
812
813    /* Set sane segments */
814    mov ax, (KGDT64_R3_DATA or RPL_MASK)
815    mov ds, ax
816    mov es, ax
817
818    /* Save MCXSR and set kernel value */
819    stmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
820    ldmxcsr gs:[PcMxCsr]
821
822    /* Get the current thread and the trap frame */
823    mov rax, gs:[PcCurrentThread]
824    mov rcx, [rax + ThTrapFrame]
825
826    /* Save the old trap frame */
827    lea rdx, [rsp + MAX_SYSCALL_PARAM_SIZE]
828    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame], rcx
829    mov [rax + ThTrapFrame], rdx
830
831#if DBG
832    /* Check IRQL */
833    mov rax, cr8
834    test eax, eax
835    jz KiSystemCall64Again
836    int HEX(2C)
837#endif
838
839GLOBAL_LABEL KiSystemCall64Again
840
841    /* Call the C-handler (will enable interrupts) */
842    call KiSystemCallHandler
843
844    /* The return value from KiSystemCallHandler is the address of the Nt-function */
845    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
846    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
847    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
848    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
849    call rax
850
851GLOBAL_LABEL KiSystemServiceExit
852
853    ASSERT_TRAP_FRAME_INTS_ENABLED rsp + MAX_SYSCALL_PARAM_SIZE
854
855    /* Check for pending user APCs */
856    mov rcx, gs:[PcCurrentThread]
857    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
858    jz NoUserApcPending
859
860    /* Save missing regs in the trap frame */
861    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
862    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
863    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], rbp
864    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
865    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], rax
866    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
867    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rax
868    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
869    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], rax
870    xor rax, rax
871    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rax
872    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], rax
873    pxor xmm0, xmm0
874    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm0], xmm0
875    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm1], xmm0
876    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm2], xmm0
877    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm3], xmm0
878    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm4], xmm0
879    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm5], xmm0
880
881    lea rcx, [rsp + MAX_SYSCALL_PARAM_SIZE]
882    call KiInitiateUserApc
883
884NoUserApcPending:
885    /* Disable interrupts for return */
886    cli
887
888    /* Restore MCXSR */
889    ldmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
890
891    /* Restore old trap frame */
892    mov rcx, gs:[PcCurrentThread]
893    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
894    mov [rcx + KTHREAD_TrapFrame], rdx
895
896    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
897    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
898    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
899
900    /* Load user mode stack (It was copied to the trap frame) */
901    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
902
903    /* r8 points to the user stack */
904    mov r8, rsp
905
906    /* r9 matches rbp */
907    mov r9, rbp
908
909    /* Swap gs back to user */
910    swapgs
911
912    /* Zero out volatiles */
913    pxor xmm0, xmm0
914    pxor xmm1, xmm1
915    pxor xmm2, xmm2
916    pxor xmm3, xmm3
917    pxor xmm4, xmm4
918    pxor xmm5, xmm5
919    xor rdx, rdx
920    xor r10, r10
921
922    /* return to user mode */
923    sysretq
924
925.ENDP
926
927
928/*!
929 * VOID
930 * DECLSPEC_NORETURN
931 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
932 */
933PUBLIC KiServiceExit
934.PROC KiServiceExit
935    .endprolog
936
937    lea rsp, [rcx - MAX_SYSCALL_PARAM_SIZE]
938    jmp KiSystemServiceExit
939
940.ENDP
941
942
943/*!
944 * VOID
945 * DECLSPEC_NORETURN
946 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
947 */
948PUBLIC KiServiceExit2
949.PROC KiServiceExit2
950    .ENDPROLOG
951
952    // FIXME: this should probably also restore an exception frame
953
954    mov rsp, rcx
955.ENDP
956
957PUBLIC KiServiceExit3
958.PROC KiServiceExit3
959    .PUSHFRAME
960    .ALLOCSTACK (KTRAP_FRAME_LENGTH - MachineFrameLength)
961    .ENDPROLOG
962
963#if DBG
964    /* Get the current IRQL and compare it to the trap frame */
965    mov rax, cr8
966    cmp byte ptr [rsp + KTRAP_FRAME_PreviousIrql], al
967    je KiServiceExit2_ok1
968    int HEX(2C)
969
970KiServiceExit2_ok1:
971    /* Check if this is a user mode exit */
972    mov ah, byte ptr [rsp + KTRAP_FRAME_SegCs]
973    test ah, 1
974    jz KiServiceExit2_kernel
975
976    /* Validate that we are at PASSIVE_LEVEL */
977    test al, al
978    jz KiServiceExit2_kernel
979    int HEX(2C)
980
981KiServiceExit2_kernel:
982#endif
983
984    /* Return */
985    mov rbp, rsp
986    ExitTrap TF_SAVE_ALL
987.ENDP
988
989
990PUBLIC KiSystemCallEntry32
991KiSystemCallEntry32:
992    swapgs
993    int 3
994
995
996PUBLIC KiZwSystemService
997FUNC KiZwSystemService
998    push rbp
999    .pushreg rbp
1000    sub rsp, KTRAP_FRAME_LENGTH
1001    .allocstack KTRAP_FRAME_LENGTH
1002    mov [rsp + KTRAP_FRAME_Rsi], rsi
1003    .savereg rsi, KTRAP_FRAME_Rsi
1004    mov [rsp + KTRAP_FRAME_Rdi], rdi
1005    .savereg rdi, KTRAP_FRAME_Rdi
1006    mov rbp, rsp
1007    .setframe rbp, 0
1008    .endprolog
1009
1010    /* Get current thread */
1011    mov r11, gs:[PcCurrentThread]
1012
1013    /* Save PreviousMode in the trap frame */
1014    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
1015    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
1016
1017    /* Save the old trap frame in TrapFrame.Rdx */
1018    mov rdi, [r11 + KTHREAD_TrapFrame]
1019    mov [rbp + KTRAP_FRAME_Rdx], rdi
1020
1021    /* Set the new trap frame and previous mode */
1022    mov [r11 + ThTrapFrame], rbp
1023    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
1024
1025    /* allocate space for parameters */
1026    sub rsp, r10
1027    and rsp, HEX(0fffffffffffffff0)
1028
1029    /* Save rcx */
1030    mov [rbp + KTRAP_FRAME_Rcx], rcx
1031
1032    /* copy parameters to the new location */
1033    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
1034    lea rdi, [rsp]
1035    mov rcx, r10
1036    shr rcx, 3
1037    rep movsq
1038
1039    /* Restore rcx */
1040    mov rcx, [rbp + KTRAP_FRAME_Rcx]
1041
1042    /* Call the service function */
1043    call rax
1044
1045    /* Restore the old trap frame */
1046    mov r11, gs:[PcCurrentThread]
1047    mov rsi, [rbp + KTRAP_FRAME_Rdx]
1048    mov [r11 + KTHREAD_TrapFrame], rsi
1049
1050    /* Restore PreviousMode from the trap frame */
1051    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
1052    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
1053
1054    /* Restore rdi and rsi */
1055    mov rsi, [rbp + KTRAP_FRAME_Rsi]
1056    mov rdi, [rbp + KTRAP_FRAME_Rdi]
1057
1058    /* Cleanup the stack and return */
1059    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
1060    pop rbp
1061    ret
1062
1063ENDFUNC
1064
1065PUBLIC KiConvertToGuiThread
1066FUNC KiConvertToGuiThread
1067
1068    sub rsp, 40
1069    .allocstack 40
1070    .endprolog
1071
1072    /* Check if we already have a large stack */
1073    mov rax, gs:[PcCurrentThread]
1074    cmp byte ptr [rax + KTHREAD_LargeStack], 0
1075    jnz AlreadyLargeStack
1076
1077    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
1078    mov cl, 1
1079    xor rdx, rdx
1080    call MmCreateKernelStack
1081
1082    /* Check for failure */
1083    test rax, rax
1084    jz KiConvertToGuiThreadFailed
1085
1086    /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_LARGE_STACK_COMMIT )); */
1087    mov rcx, rax
1088    mov rdx, rax
1089    sub rdx, KERNEL_LARGE_STACK_COMMIT
1090    call KeSwitchKernelStack
1091
1092    // MmDeleteKernelStack(OldStack, FALSE);
1093    mov rcx, rax
1094    xor rdx, rdx
1095    call MmDeleteKernelStack
1096
1097AlreadyLargeStack:
1098
1099    /* Call the worker function */
1100    call PsConvertToGuiThread
1101
1102    /* Check for failure */
1103    test eax, eax
1104    js KiConvertToGuiThreadFailed
1105
1106    /* Disable interrupts for return */
1107    cli
1108
1109    // Restore register parameters
1110    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
1111    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
1112    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
1113    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1114
1115    /* Run KiSystemCallHandler again */
1116    add rsp, 48
1117    jmp KiSystemCall64Again
1118
1119KiConvertToGuiThreadFailed:
1120
1121    /* Clean up the stack and return failure */
1122    add rsp, 40
1123    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1124    ret
1125
1126ENDFUNC
1127
1128
1129EXTERN KiSetTrapContextInternal:PROC
1130
1131/*
1132 * VOID
1133 * KiSetTrapContext(
1134 *     _Out_ PKTRAP_FRAME TrapFrame,
1135 *     _In_ PCONTEXT Context,
1136 *     _In_ KPROCESSOR_MODE RequestorMode);
1137 */
1138PUBLIC KiSetTrapContext
1139.PROC KiSetTrapContext
1140
1141    /* Generate a KEXCEPTION_FRAME on the stack */
1142    GENERATE_EXCEPTION_FRAME
1143
1144    call KiSetTrapContextInternal
1145
1146    /* Restore the registers from the KEXCEPTION_FRAME */
1147    RESTORE_EXCEPTION_STATE
1148
1149    /* Return */
1150    ret
1151
1152.ENDP
1153
1154
1155/*
1156 * VOID
1157 * KiDeliverApc(
1158 *     _In_ KPROCESSOR_MODE DeliveryMode,
1159 *     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1160 *     _In_ PKTRAP_FRAME TrapFrame);
1161 *
1162 */
1163EXTERN KiDeliverApc:PROC
1164
1165/*
1166 * VOID
1167 * KiInitiateUserApc(
1168 *     _In_ PKTRAP_FRAME TrapFrame@<rcx>);
1169 *
1170 * This function is called to deliver user mode APCs.
1171 * It clobbers all non-volatile registers, except rax.
1172 */
1173PUBLIC KiInitiateUserApc
1174.PROC KiInitiateUserApc
1175
1176    /* Generate a KEXCEPTION_FRAME on the stack */
1177    GENERATE_EXCEPTION_FRAME
1178
1179    /* Raise IRQL to APC_LEVEL */
1180    mov rax, APC_LEVEL
1181    mov cr8, rax
1182
1183    /* Get the current thread */
1184    mov rbp, gs:[PcCurrentThread]
1185
1186    /* Save the trap frame in rsi */
1187    mov rsi, rcx
1188
1189    /* Enable interrupts */
1190    sti
1191
1192    /* Call the C function */
1193    mov ecx, 1
1194    mov rdx, rsp
1195    mov r8, rsi
1196    call KiDeliverApc
1197
1198    /* Disable interrupts again */
1199    cli
1200
1201    /* Go back to PASSIVE_LEVEL */
1202    mov rax, PASSIVE_LEVEL
1203    mov cr8, rax
1204
1205    /* Restore the registers from the KEXCEPTION_FRAME */
1206    RESTORE_EXCEPTION_STATE
1207
1208    /* Return */
1209    ret
1210
1211.ENDP
1212
1213
1214PUBLIC KiInitializeSegments
1215KiInitializeSegments:
1216    mov ax, KGDT64_R3_DATA or RPL_MASK
1217    mov gs, ax
1218    swapgs
1219    mov gs, ax
1220    ret
1221
1222/*!
1223 * VOID
1224 * KiSwitchKernelStackHelper(
1225 *     LONG_PTR StackOffset,
1226 *     PVOID OldStackBase);
1227 */
1228PUBLIC KiSwitchKernelStackHelper
1229KiSwitchKernelStackHelper:
1230
1231    /* Pop return address from the current stack */
1232    pop rax
1233
1234    /* Switch to new stack */
1235    lea rsp, [rsp + rcx]
1236
1237    /* Push return address on the new stack */
1238    push rax
1239
1240    /* Return on new stack */
1241    mov rax, rdx
1242    ret
1243
1244EXTERN KiSwitchKernelStack:PROC
1245
1246PUBLIC KeSwitchKernelStack
1247FUNC KeSwitchKernelStack
1248
1249    /* Save rcx and allocate callee home space */
1250    mov [rsp + P1Home], rcx
1251    .savereg rcx, P1Home
1252    sub rsp, 40
1253    .allocstack 40
1254    .endprolog
1255
1256    /* Call the C handler, which returns the old stack in rax */
1257    call KiSwitchKernelStack
1258
1259    /* Restore rcx (StackBase) */
1260    mov rcx, [rsp + 40 + P1Home]
1261
1262    /* Switch to new stack: RSP += (StackBase - OldStackBase) */
1263    sub rcx, rax
1264    add rsp, rcx
1265
1266    /* Deallocate the home frame */
1267    add rsp, 40
1268    ret
1269
1270ENDFUNC
1271
1272
1273
1274
1275#ifdef _MSC_VER
1276#undef lgdt
1277#undef lidt
1278
1279//void __lgdt(void *Source);
1280PUBLIC __lgdt
1281__lgdt:
1282    lgdt fword ptr [rcx]
1283    ret
1284
1285//void __sgdt(void *Destination);
1286PUBLIC __sgdt
1287__sgdt:
1288    sgdt fword ptr [rcx]
1289    ret
1290
1291// void __lldt(unsigned short Value)
1292PUBLIC __lldt
1293__lldt:
1294    lldt cx
1295    ret
1296
1297//void __sldt(void *Destination);
1298PUBLIC __sldt
1299__sldt:
1300    sldt word ptr [rcx]
1301    ret
1302
1303//void __ltr(unsigned short Source);
1304PUBLIC __ltr
1305__ltr:
1306    ltr cx
1307    ret
1308
1309//void __str(unsigned short *Destination);
1310PUBLIC __str
1311__str:
1312    str word ptr [rcx]
1313    ret
1314
1315PUBLIC __swapgs
1316__swapgs:
1317    swapgs
1318    ret
1319
1320#endif
1321
1322END
1323