xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision 34593d93) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26
27EXTERN KdSetOwedBreakpoints:PROC
28
29
30/* Helper Macros *************************************************************/
31
32MACRO(DispatchException, Status, Number, P1, P2, P3)
33    mov eax, Status
34    mov edx, Number
35    mov r9, P1
36    mov r10, P2
37    mov r11, P3
38    call InternalDispatchException
39ENDM
40
41MACRO(Fatal, BugcheckCode)
42    /* Bugcheck */
43    mov ecx, BugcheckCode
44    mov rdx, rbp
45    call KiSystemFatalException
46ENDM
47
48
49/* FUNCTIONS *****************************************************************/
50
51.code64
52
53ALIGN 8
54
55MACRO(UnexpectedVectorStub, Vector)
56    /* This nop is to make the relative jmp address 4 bytes aligned and to
57       make the whole code 8 bytes long */
58    nop
59    /* This is a push instruction with 8bit operand. Since the instruction
60       sign extends the value to 32 bits, we need to offset it */
61PUBLIC KxUnexpectedInterrupt&Vector
62KxUnexpectedInterrupt&Vector:
63    push (Vector - 128)
64    jmp KiUnexpectedInterrupt
65ENDM
66
67PUBLIC KiUnexpectedRange
68KiUnexpectedRange:
69Vector = 0
70REPEAT 256
71    UnexpectedVectorStub %Vector
72    Vector = Vector+1
73ENDR
74PUBLIC KiUnexpectedRangeEnd
75KiUnexpectedRangeEnd:
76
77PUBLIC KiInterruptDispatchTemplate
78KiInterruptDispatchTemplate:
79    /* This instruction pushes the return address on the stack, which is the
80       address of the interrupt object's DispatchCode member, then jumps
81       to the address stored in the interrupt object's DispatchAddress member */
82    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
83
84
85// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
86FUNC InternalDispatchException
87
88    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
89    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
90    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
91    .endprolog
92
93    /* Set up EXCEPTION_RECORD */
94    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
95    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
96    xor rax, rax
97    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
98    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
99    mov rax, [rbp + KTRAP_FRAME_Rip]
100    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
101    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
102    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
105
106    /* Set up KEXCEPTION_FRAME */
107    mov rax, [rbp + KTRAP_FRAME_Rbp]
108    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
109    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
110    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
111    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
112    mov [rsp + KEXCEPTION_FRAME_R12], r12
113    mov [rsp + KEXCEPTION_FRAME_R13], r13
114    mov [rsp + KEXCEPTION_FRAME_R14], r14
115    mov [rsp + KEXCEPTION_FRAME_R15], r15
116    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
126    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
127
128    /* Call KiDispatchException */
129    // rcx already points to ExceptionRecord
130    mov rdx, rsp                                // ExceptionFrame
131    mov r8, rbp                                 // TrapFrame
132    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
133    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
134    call KiDispatchException
135
136    /* Restore registers */
137    mov r12, [rsp + KEXCEPTION_FRAME_R12]
138    mov r13, [rsp + KEXCEPTION_FRAME_R13]
139    mov r14, [rsp + KEXCEPTION_FRAME_R14]
140    mov r15, [rsp + KEXCEPTION_FRAME_R15]
141    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
142    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
143    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
144    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
145    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
146    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
147    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
148    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
149    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
150    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
151
152    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
153    ret
154ENDFUNC
155
156
157/* CPU EXCEPTION HANDLERS ****************************************************/
158
159PUBLIC KiDivideErrorFault
160FUNC KiDivideErrorFault
161    /* Push pseudo error code */
162    EnterTrap TF_SAVE_ALL
163
164    /* Enable interrupts */
165    sti
166
167    /* Dispatch the exception */
168    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
169
170    /* Return */
171    ExitTrap TF_SAVE_ALL
172ENDFUNC
173
174
175PUBLIC KiDebugTrapOrFault
176FUNC KiDebugTrapOrFault
177    /* Push pseudo error code */
178    EnterTrap TF_SAVE_ALL
179
180    /* Check if the frame was from kernelmode */
181    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
182    jz KiDebugTrapOrFaultKMode
183
184    /* Enable interrupts for user-mode */
185    sti
186
187KiDebugTrapOrFaultKMode:
188    /* Dispatch the exception */
189    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
190
191    /* Return */
192    ExitTrap TF_SAVE_ALL
193ENDFUNC
194
195
196PUBLIC KiNmiInterrupt
197FUNC KiNmiInterrupt
198    /* Push pseudo error code */
199    EnterTrap TF_SAVE_ALL
200
201    UNIMPLEMENTED KiNmiInterrupt
202    int 3
203
204    /* Return */
205    ExitTrap TF_SAVE_ALL
206ENDFUNC
207
208
209PUBLIC KiBreakpointTrap
210FUNC KiBreakpointTrap
211    /* Push pseudo error code */
212    EnterTrap TF_SAVE_ALL
213
214    /* Check if the frame was from kernelmode */
215    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
216    jz KiBreakpointTrapKMode
217
218    /* Enable interrupts for user-mode */
219    sti
220
221KiBreakpointTrapKMode:
222    /* Dispatch the exception */
223    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
224
225    /* Return */
226    ExitTrap TF_SAVE_ALL
227ENDFUNC
228
229
230PUBLIC KiOverflowTrap
231FUNC KiOverflowTrap
232    /* Push pseudo error code */
233    EnterTrap TF_SAVE_ALL
234
235    /* Enable interrupts */
236    sti
237
238    /* Dispatch the exception */
239    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
240
241    /* Return */
242    ExitTrap TF_SAVE_ALL
243ENDFUNC
244
245
246PUBLIC KiBoundFault
247FUNC KiBoundFault
248    /* Push pseudo error code */
249    EnterTrap TF_SAVE_ALL
250
251    /* Check if the frame was from kernelmode */
252    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
253    jnz KiBoundFaultUserMode
254
255    /* Bugcheck */
256    Fatal EXCEPTION_BOUND_CHECK
257
258KiBoundFaultUserMode:
259    /* Enable interrupts for user-mode */
260    sti
261
262    /* Dispatch the exception */
263    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
264
265    /* Return */
266    ExitTrap TF_SAVE_ALL
267ENDFUNC
268
269
270PUBLIC KiInvalidOpcodeFault
271FUNC KiInvalidOpcodeFault
272    /* Push pseudo error code */
273    EnterTrap TF_SAVE_ALL
274
275    /* Enable interrupts */
276    sti
277
278    /* Check if the frame was from kernelmode */
279    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
280    jz KiInvalidOpcodeKernel
281
282    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
283
284KiInvalidOpcodeKernel:
285    /* Kernel mode fault */
286
287    /* Dispatch the exception */
288    DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
289
290    /* Return */
291    ExitTrap TF_SAVE_ALL
292ENDFUNC
293
294
295PUBLIC KiNpxNotAvailableFault
296FUNC KiNpxNotAvailableFault
297    /* Push pseudo error code */
298    EnterTrap TF_SAVE_ALL
299
300    /* Call the C handler */
301    mov rcx, rbp
302    call KiNpxNotAvailableFaultHandler
303
304    /* Check the return status code */
305    test eax, eax
306    jz KiNpxNotAvailableFaultExit
307
308    /* Dispatch the exception */
309    DispatchException eax, 3, 0, 0, 0
310
311KiNpxNotAvailableFaultExit:
312    /* Return */
313    ExitTrap TF_SAVE_ALL
314ENDFUNC
315
316
317PUBLIC KiDoubleFaultAbort
318FUNC KiDoubleFaultAbort
319    /* A zero error code is pushed */
320    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
321
322    int 3
323
324    /* Bugcheck */
325    Fatal 8 // EXCEPTION_DOUBLE_FAULT
326    jmp $
327ENDFUNC
328
329
330PUBLIC KiNpxSegmentOverrunAbort
331FUNC KiNpxSegmentOverrunAbort
332    /* Push pseudo error code */
333    EnterTrap TF_SAVE_ALL
334
335    /* Bugcheck */
336    Fatal EXCEPTION_NPX_OVERRUN
337
338    jmp $
339ENDFUNC
340
341
342PUBLIC KiInvalidTssFault
343FUNC KiInvalidTssFault
344    /* We have an error code */
345    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
346
347    /* Bugcheck */
348    Fatal EXCEPTION_INVALID_TSS
349    jmp $
350ENDFUNC
351
352
353PUBLIC KiSegmentNotPresentFault
354FUNC KiSegmentNotPresentFault
355    /* We have an error code */
356    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
357
358    /* Bugcheck */
359    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
360    jmp $
361ENDFUNC
362
363
364PUBLIC KiStackFault
365FUNC KiStackFault
366    /* We have an error code */
367    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
368
369    /* Bugcheck */
370    Fatal EXCEPTION_STACK_FAULT
371    jmp $
372ENDFUNC
373
374
375PUBLIC KiGeneralProtectionFault
376FUNC KiGeneralProtectionFault
377    /* We have an error code */
378    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
379
380    /* Call the C handler */
381    mov rcx, rbp
382    call KiGeneralProtectionFaultHandler
383
384    /* Check for success */
385    test eax, eax
386    jge KiGpfExit
387
388    /* Dispatch the exception */
389    DispatchException eax, 3, 0, 0, 0
390
391KiGpfFatal:
392
393    /* Bugcheck */
394    mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
395    mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
396    xor r8, r8
397    mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
398    sub rsp, 8
399    mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
400    call KeBugCheckWithTf
401
402KiGpfExit:
403    /* Return */
404    /* Return */
405    ExitTrap TF_SAVE_ALL
406ENDFUNC
407
408
409PUBLIC KiPageFault
410FUNC KiPageFault
411    /* We have an error code */
412    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
413
414    /* Save page fault address */
415    mov rdx, cr2
416    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
417
418    /* Enable interrupts for the page fault handler */
419    sti
420
421    /* Call page fault handler */
422    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
423    // rdx == Address
424    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
425    and r8b, 1
426    mov r9, rbp // TrapInformation
427    call MmAccessFault
428
429    /* Check for success */
430    test eax, eax
431    jl PageFaultError
432
433    /* Check whether the kernel debugger has owed breakpoints to be inserted */
434    call KdSetOwedBreakpoints
435    /* We succeeded, return */
436    jmp PageFaultReturn
437
438PageFaultError:
439
440    /* Disable interrupts again for the debugger */
441    cli
442
443    /* Set parameter 1 to error code */
444    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
445
446    /* Set parameter2 to faulting address */
447    mov r10, cr2  // Param2 = faulting address
448
449    cmp eax, STATUS_ACCESS_VIOLATION
450    je AccessViolation
451    cmp eax, STATUS_GUARD_PAGE_VIOLATION
452    je SpecialCode
453    cmp eax, STATUS_STACK_OVERFLOW
454    je SpecialCode
455
456InPageException:
457    /* Dispatch in-page exception */
458    mov r11d, eax // Param3 = Status
459    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
460    mov edx, 3    // ParamCount
461    call InternalDispatchException
462    jmp PageFaultReturn
463
464AccessViolation:
465    /* Use more proper status code */
466    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
467
468SpecialCode:
469    /* Setup a normal page fault exception */
470    mov edx, 2   // ParamCount
471    call InternalDispatchException
472
473PageFaultReturn:
474    /* Return */
475    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
476ENDFUNC
477
478
479PUBLIC KiFloatingErrorFault
480FUNC KiFloatingErrorFault
481    /* Push pseudo error code */
482    EnterTrap TF_SAVE_ALL
483
484    UNIMPLEMENTED KiFloatingErrorFault
485    int 3
486
487    /* Return */
488    ExitTrap TF_SAVE_ALL
489ENDFUNC
490
491
492PUBLIC KiAlignmentFault
493FUNC KiAlignmentFault
494    /* We have an error code */
495    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
496
497    /* Bugcheck */
498    Fatal EXCEPTION_ALIGNMENT_CHECK
499    jmp $
500ENDFUNC
501
502
503PUBLIC KiMcheckAbort
504FUNC KiMcheckAbort
505    /* Push pseudo error code */
506    EnterTrap TF_SAVE_ALL
507
508    /* Bugcheck */
509    Fatal HEX(12)
510    jmp $
511ENDFUNC
512
513
514PUBLIC KiXmmException
515FUNC KiXmmException
516    /* Push pseudo error code */
517    EnterTrap TF_SAVE_ALL
518
519    /* Call the C handler */
520    mov rcx, rbp
521    call KiXmmExceptionHandler
522
523    /* Check for success */
524    test eax, eax
525    jge KiXmmExit
526
527    /* Dispatch the exception */
528    DispatchException eax, 3, 0, 0, 0
529
530KiXmmExit:
531    /* Return */
532    ExitTrap TF_SAVE_ALL
533ENDFUNC
534
535
536/* SOFTWARE INTERRUPT SERVICES ***********************************************/
537
538PUBLIC KiRaiseAssertion
539FUNC KiRaiseAssertion
540    /* We have an error code */
541    EnterTrap (TF_SAVE_ALL)
542
543    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
544    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
545
546    /* Dispatch the exception */
547    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
548
549    /* Return */
550    ExitTrap TF_SAVE_ALL
551ENDFUNC
552
553
554PUBLIC KiDebugServiceTrap
555FUNC KiDebugServiceTrap
556   /* No error code */
557    EnterTrap TF_SAVE_ALL
558
559    /* Increase Rip to skip the int3 */
560    inc qword ptr [rbp + KTRAP_FRAME_Rip]
561
562    /* Dispatch the exception (Params = service, buffer, legth) */
563    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
564
565    /* Return */
566    ExitTrap TF_SAVE_ALL
567ENDFUNC
568
569
570PUBLIC KiApcInterrupt
571.PROC KiApcInterrupt
572   /* No error code */
573    EnterTrap (TF_VOLATILES or TF_IRQL)
574
575    /* Raise to APC_LEVEL */
576    mov rax, APC_LEVEL
577    mov cr8, rax
578
579    /* End the interrupt */
580    mov dword ptr [APIC_EOI], 0
581
582    /* Enable interrupts  */
583    sti
584
585    /* Call the worker routine */
586    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
587    and cl, 1
588    mov rdx, 0                        // ExceptionFrame
589    mov r8, rbp                       // TrapFrame
590    call KiDeliverApc
591
592    /* Disable interrupts */
593    cli
594
595    /* Lower IRQL back to PASSIVE */
596    mov rax, PASSIVE_LEVEL
597    mov cr8, rax
598
599    /* Return */
600    ExitTrap (TF_VOLATILES or TF_IRQL)
601.ENDP
602
603/*
604 * VOID
605 * KiRetireDpcList(
606 *     PKPRCB Prcb);
607 */
608EXTERN KiRetireDpcList:PROC
609
610/*
611 * VOID
612 * KiRetireDpcListInDpcStack(
613 *     PKPRCB Prcb,
614 *     PVOID DpcStack);
615 */
616PUBLIC KiRetireDpcListInDpcStack
617.PROC KiRetireDpcListInDpcStack
618    push rbp
619    .pushreg rbp
620    mov rbp, rsp
621    .setframe rbp, 0
622    .endprolog
623
624    /* Switch to the DpcStack */
625    mov rsp, rdx
626
627    /* The stack is 16 byte aligned, allocate 32 bytes home space */
628    sub rsp, 32
629
630    /* Call KiRetireDpcList on the given stack */
631    call KiRetireDpcList
632
633    /* Restore stack, cleanup and return */
634    mov rsp, rbp
635    pop rbp
636    ret
637.ENDP
638
639PUBLIC KiDpcInterrupt
640.PROC KiDpcInterrupt
641   /* No error code */
642    EnterTrap (TF_VOLATILES or TF_IRQL)
643
644    /* Call the worker routine */
645    call KiDpcInterruptHandler
646
647    /* Return, but don't send an EOI! */
648    ExitTrap (TF_VOLATILES or TF_IRQL)
649.ENDP
650
651
652PUBLIC KiIpiInterrupt
653.PROC KiIpiInterrupt
654   /* No error code */
655    EnterTrap (TF_VOLATILES or TF_IRQL)
656
657    /* Raise to IPI_LEVEL */
658    mov rax, IPI_LEVEL
659    mov cr8, rax
660
661    /* End the interrupt */
662    mov dword ptr [APIC_EOI], 0
663
664    int 3
665
666    /* Return */
667    ExitTrap (TF_VOLATILES or TF_IRQL)
668.ENDP
669
670
671PUBLIC KiUnexpectedInterrupt
672FUNC KiUnexpectedInterrupt
673    /* The error code is the vector */
674    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
675
676#if 0
677    /* Set bugcheck parameters */
678    mov ecx, TRAP_CAUSE_UNKNOWN
679    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
680    mov r8, 0 // The unknown floating-point exception
681    mov r9, 0 // The enabled and asserted status bits
682    sub rsp, 8
683    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
684    call KeBugCheckWithTf
685    jmp $
686#endif
687    /* Return */
688    ExitTrap TF_SAVE_ALL
689ENDFUNC
690
691PUBLIC KiInterruptDispatch
692FUNC KiInterruptDispatch
693    /* The error code is a pointer to the interrupt object's code */
694    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
695
696    /* Increase interrupt count */
697    inc dword ptr gs:[PcInterruptCount];
698
699    /* Save rbx and rsi in the trap frame */
700    mov [rbp + KTRAP_FRAME_Rbx], rbx
701    mov [rbp + KTRAP_FRAME_Rsi], rsi
702
703    /* Load the address of the dispatch code into rbx */
704    mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
705
706    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
707    sub rbx, KINTERRUPT_DispatchCode + 6
708
709    /* Save the address of the InterruptListEntry in rsi */
710    lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
711
712.DoDispatchInterrupt:
713    /* Raise IRQL to SynchronizeIrql */
714    movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
715    mov cr8, rax
716
717#ifdef CONFIG_SMP
718    /* Acquire interrupt lock */
719    mov r8, [rbx + KINTERRUPT_ActualLock]
720
721    //KxAcquireSpinLock(Interrupt->ActualLock);
722#endif
723
724    /* Call the ISR */
725    mov rcx, rbx
726    mov rdx, [rbx + KINTERRUPT_ServiceContext]
727    call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
728
729#ifdef CONFIG_SMP
730    /* Release interrupt lock */
731    //KxReleaseSpinLock(Interrupt->ActualLock);
732#endif
733
734    /* Go back to old irql */
735    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
736    mov cr8, rax
737
738    /* Check for chained interrupts */
739    mov rax, [rbx + KINTERRUPT_InterruptListEntry]
740    cmp rax, rsi
741    je .Done
742
743    /* Load the next interrupt object into rbx and repeat */
744    lea rbx, [rax - KINTERRUPT_InterruptListEntry]
745    jmp .DoDispatchInterrupt
746
747.Done:
748    /* Restore rbx and rsi */
749    mov rbx, [rbp + KTRAP_FRAME_Rbx]
750    mov rsi, [rbp + KTRAP_FRAME_Rsi]
751
752    /* Return */
753    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
754ENDFUNC
755
756EXTERN KiSystemCallHandler:PROC
757
758/*! \name KiSystemCallEntry64
759 *
760 *  \brief This is the entrypoint for syscalls from 64bit user mode
761 *
762 *  \param rax - The system call number
763 *  \param rcx - User mode return address, set by the syscall instruction
764 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
765 *  \param r10 - Parameter 1 to the service function
766 *  \param r11 - RFLAGS saved by the syscall instruction
767 *--*/
768PUBLIC KiSystemCallEntry64
769.PROC KiSystemCallEntry64
770    /* The unwind info pretends we have a machine frame */
771    .PUSHFRAME
772    .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
773    .ENDPROLOG
774
775    /* Swap gs to kernel, so we can access the PCR */
776    swapgs
777
778    /* Save the user mode rsp in the PCR */
779    mov gs:[PcUserRsp], rsp
780
781    /* Get the kernel stack from the PCR */
782    mov rsp, gs:[PcRspBase]
783
784    /* Allocate a TRAP_FRAME and space for parameters */
785    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
786
787    /* Save volatile registers in the trap frame */
788    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
789    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
790    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
791    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
792    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
793    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
794    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
795
796    /* Store user stack pointer in the trap frame */
797    mov rax, gs:[PcUserRsp]
798    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
799
800    /* Set sane segments */
801    mov ax, (KGDT64_R3_DATA or RPL_MASK)
802    mov ds, ax
803    mov es, ax
804
805#if DBG
806    /* Check IRQL */
807    mov rax, cr8
808    test eax, eax
809    jz KiSystemCall64Again
810    int HEX(2C)
811#endif
812
813GLOBAL_LABEL KiSystemCall64Again
814
815    /* Call the C-handler (will enable interrupts) */
816    call KiSystemCallHandler
817
818    /* The return value from KiSystemCallHandler is the address of the Nt-function */
819    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
820    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
821    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
822    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
823    call rax
824
825GLOBAL_LABEL KiSystemServiceExit
826
827#if DBG
828    test dword ptr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], HEX(200)
829    jnz IntsEnabled
830    int 3
831IntsEnabled:
832#endif
833
834    /* Check for pending user APC */
835    mov rcx, gs:qword ptr [PcCurrentThread]
836    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
837    jz no_user_apc_pending
838    call KiInitiateUserApc
839no_user_apc_pending:
840
841    /* Disable interrupts for return */
842    cli
843
844    /* Restore old trap frame */
845    mov rcx, gs:[PcCurrentThread]
846    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
847    mov [rcx + KTHREAD_TrapFrame], rdx
848
849    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
850    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
851    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
852
853    /* Load user mode stack (It was copied to the trap frame) */
854    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
855
856    /* r8 points to the user stack */
857    mov r8, rsp
858
859    /* r9 matches rbp */
860    mov r9, rbp
861
862    /* Swap gs back to user */
863    swapgs
864
865    /* Zero out volatiles */
866    pxor xmm0, xmm0
867    pxor xmm1, xmm1
868    pxor xmm2, xmm2
869    pxor xmm3, xmm3
870    pxor xmm4, xmm4
871    pxor xmm5, xmm5
872    xor rdx, rdx
873    xor r10, r10
874
875    /* return to user mode */
876    .byte HEX(48) // REX prefix to return to long mode
877    sysretq
878
879.ENDP
880
881
882/*!
883 * VOID
884 * DECLSPEC_NORETURN
885 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
886 */
887PUBLIC KiServiceExit
888.PROC KiServiceExit
889    .endprolog
890
891    lea rsp, [rcx - MAX_SYSCALL_PARAM_SIZE]
892    jmp KiSystemServiceExit
893
894.ENDP
895
896
897/*!
898 * VOID
899 * DECLSPEC_NORETURN
900 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
901 */
902PUBLIC KiServiceExit2
903.PROC KiServiceExit2
904    .ENDPROLOG
905
906#if DBG
907    /* Get the current IRQL and compare it to the trap frame */
908    mov rax, cr8
909    cmp byte ptr [rcx + KTRAP_FRAME_PreviousIrql], al
910    je KiServiceExit2_ok1
911    int HEX(2C)
912
913KiServiceExit2_ok1:
914    /* Check if this is a user mode exit */
915    mov ah, byte ptr [rcx + KTRAP_FRAME_SegCs]
916    test ah, 1
917    jz KiServiceExit2_kernel
918
919    /* Validate that we are at PASSIVE_LEVEL */
920    test al, al
921    jz KiServiceExit2_kernel
922    int HEX(2C)
923
924KiServiceExit2_kernel:
925#endif
926
927    mov rbp, rcx
928    mov rsp, rcx
929
930    /* Return */
931    ExitTrap TF_SAVE_ALL
932.ENDP
933
934
935PUBLIC KiSystemCallEntry32
936KiSystemCallEntry32:
937    swapgs
938    int 3
939
940
941PUBLIC KiZwSystemService
942FUNC KiZwSystemService
943    push rbp
944    .pushreg rbp
945    sub rsp, KTRAP_FRAME_LENGTH
946    .allocstack KTRAP_FRAME_LENGTH
947    mov [rsp + KTRAP_FRAME_Rsi], rsi
948    .savereg rsi, KTRAP_FRAME_Rsi
949    mov [rsp + KTRAP_FRAME_Rdi], rdi
950    .savereg rdi, KTRAP_FRAME_Rdi
951    mov rbp, rsp
952    .setframe rbp, 0
953    .endprolog
954
955    /* Get current thread */
956    mov r11, gs:[PcCurrentThread]
957
958    /* Save PreviousMode in the trap frame */
959    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
960    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
961
962    /* Save the old trap frame in TrapFrame.Rdx */
963    mov rdi, [r11 + KTHREAD_TrapFrame]
964    mov [rbp + KTRAP_FRAME_Rdx], rdi
965
966    /* Set the new trap frame and previous mode */
967    mov [r11 + ThTrapFrame], rbp
968    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
969
970    /* allocate space for parameters */
971    sub rsp, r10
972    and rsp, HEX(0fffffffffffffff0)
973
974    /* Save rcx */
975    mov [rbp + KTRAP_FRAME_Rcx], rcx
976
977    /* copy parameters to the new location */
978    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
979    lea rdi, [rsp]
980    mov rcx, r10
981    shr rcx, 3
982    rep movsq
983
984    /* Restore rcx */
985    mov rcx, [rbp + KTRAP_FRAME_Rcx]
986
987    /* Call the service function */
988    call rax
989
990    /* Restore the old trap frame */
991    mov r11, gs:[PcCurrentThread]
992    mov rsi, [rbp + KTRAP_FRAME_Rdx]
993    mov [r11 + KTHREAD_TrapFrame], rsi
994
995    /* Restore PreviousMode from the trap frame */
996    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
997    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
998
999    /* Restore rdi and rsi */
1000    mov rsi, [rbp + KTRAP_FRAME_Rsi]
1001    mov rdi, [rbp + KTRAP_FRAME_Rdi]
1002
1003    /* Cleanup the stack and return */
1004    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
1005    pop rbp
1006    ret
1007
1008ENDFUNC
1009
1010PUBLIC KiConvertToGuiThread
1011FUNC KiConvertToGuiThread
1012
1013    sub rsp, 40
1014    .allocstack 40
1015    .endprolog
1016
1017    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
1018    mov cl, 1
1019    xor rdx, rdx
1020    call MmCreateKernelStack
1021
1022    /* Check for failure */
1023    test rax, rax
1024    jz KiConvertToGuiThreadFailed
1025
1026    /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_STACK_SIZE)); */
1027    mov rcx, rax
1028    mov rdx, rax
1029    sub rdx, KERNEL_STACK_SIZE
1030    call KeSwitchKernelStack
1031
1032    // MmDeleteKernelStack(OldStack, FALSE);
1033    mov rcx, rax
1034    xor rdx, rdx
1035    call MmDeleteKernelStack
1036
1037    /* Call the worker function */
1038    call PsConvertToGuiThread
1039
1040    /* Check for failure */
1041    test rax, rax
1042    js KiConvertToGuiThreadFailed
1043
1044    /* Disable interrupts for return */
1045    cli
1046
1047    // FIXME: should just do the trap frame switch in KiSystemCallHandler64
1048    /* Restore old trap frame */
1049    mov rcx, gs:[PcCurrentThread]
1050    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
1051    mov [rcx + KTHREAD_TrapFrame], rdx
1052
1053    // Restore register parameters
1054    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
1055    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
1056    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
1057    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1058
1059    /* Run KiSystemCallHandler again */
1060    add rsp, 48
1061    jmp KiSystemCall64Again
1062
1063KiConvertToGuiThreadFailed:
1064
1065    /* Clean up the stack and return failure */
1066    add rsp, 40
1067    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1068    ret
1069
1070ENDFUNC
1071
1072
1073EXTERN KiSetTrapContextInternal:PROC
1074
1075/*
1076 * VOID
1077 * KiSetTrapContext(
1078 *     _Out_ PKTRAP_FRAME TrapFrame,
1079 *     _In_ PCONTEXT Context,
1080 *     _In_ KPROCESSOR_MODE RequestorMode);
1081 */
1082PUBLIC KiSetTrapContext
1083.PROC KiSetTrapContext
1084
1085    /* Generate a KEXCEPTION_FRAME on the stack */
1086    GENERATE_EXCEPTION_FRAME
1087
1088    call KiSetTrapContextInternal
1089
1090    /* Restore the registers from the KEXCEPTION_FRAME */
1091    RESTORE_EXCEPTION_STATE
1092
1093    /* Return */
1094    ret
1095
1096.ENDP
1097
1098
1099/*
1100 * VOID
1101 * KiDeliverApc(
1102 *     _In_ KPROCESSOR_MODE DeliveryMode,
1103 *     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1104 *     _In_ PKTRAP_FRAME TrapFrame);
1105 *
1106 */
1107EXTERN KiDeliverApc:PROC
1108
1109PUBLIC KiInitiateUserApc
1110.PROC KiInitiateUserApc
1111
1112    /* Generate a KEXCEPTION_FRAME on the stack */
1113    GENERATE_EXCEPTION_FRAME
1114
1115    /* Raise IRQL to APC_LEVEL */
1116    mov rax, APC_LEVEL
1117    mov cr8, rax
1118
1119    /* Enable interrupts */
1120    sti
1121
1122    /* Get the current trap frame */
1123    mov rax, gs:[PcCurrentThread]
1124    mov r8, [rax + KTHREAD_TrapFrame]
1125
1126    /* Call the C function */
1127    mov ecx, 1
1128    mov rdx, rsp
1129    call KiDeliverApc
1130
1131    /* Disable interrupts again */
1132    cli
1133
1134    /* Go back to PASSIVE_LEVEL */
1135    mov rax, PASSIVE_LEVEL
1136    mov cr8, rax
1137
1138    /* Restore the registers from the KEXCEPTION_FRAME */
1139    RESTORE_EXCEPTION_STATE
1140
1141    /* Return */
1142    ret
1143
1144.ENDP
1145
1146
1147PUBLIC KiInitializeSegments
1148KiInitializeSegments:
1149    mov ax, KGDT64_R3_DATA or RPL_MASK
1150    mov gs, ax
1151    swapgs
1152    mov gs, ax
1153    ret
1154
1155/*!
1156 * VOID
1157 * KiSwitchKernelStackHelper(
1158 *     LONG_PTR StackOffset,
1159 *     PVOID OldStackBase);
1160 */
1161PUBLIC KiSwitchKernelStackHelper
1162KiSwitchKernelStackHelper:
1163
1164    /* Pop return address from the current stack */
1165    pop rax
1166
1167    /* Switch to new stack */
1168    lea rsp, [rsp + rcx]
1169
1170    /* Push return address on the new stack */
1171    push rax
1172
1173    /* Return on new stack */
1174    mov rax, rdx
1175    ret
1176
1177EXTERN KiSwitchKernelStack:PROC
1178
1179PUBLIC KeSwitchKernelStack
1180FUNC KeSwitchKernelStack
1181
1182    sub rsp, 40
1183    .allocstack 40
1184
1185    /* Save rcx */
1186    mov [rsp], rcx
1187    .savereg rcx, 0
1188    .endprolog
1189
1190    /* Call the C handler, which returns the old stack in rax */
1191    call KiSwitchKernelStack
1192
1193    /* Restore rcx (StackBase) */
1194    mov rcx, [rsp]
1195
1196    /* Switch to new stack: RSP += (StackBase - OldStackBase) */
1197    sub rcx, rax
1198    add rsp, rcx
1199
1200    /* Deallocate the home frame */
1201    add rsp, 40
1202    ret
1203
1204ENDFUNC
1205
1206
1207
1208
1209#ifdef _MSC_VER
1210#undef lgdt
1211#undef lidt
1212
1213//void __lgdt(void *Source);
1214PUBLIC __lgdt
1215__lgdt:
1216    lgdt fword ptr [rcx]
1217    ret
1218
1219//void __sgdt(void *Destination);
1220PUBLIC __sgdt
1221__sgdt:
1222    sgdt fword ptr [rcx]
1223    ret
1224
1225// void __lldt(unsigned short Value)
1226PUBLIC __lldt
1227__lldt:
1228    lldt cx
1229    ret
1230
1231//void __sldt(void *Destination);
1232PUBLIC __sldt
1233__sldt:
1234    sldt word ptr [rcx]
1235    ret
1236
1237//void __ltr(unsigned short Source);
1238PUBLIC __ltr
1239__ltr:
1240    ltr cx
1241    ret
1242
1243//void __str(unsigned short *Destination);
1244PUBLIC __str
1245__str:
1246    str word ptr [rcx]
1247    ret
1248
1249PUBLIC __swapgs
1250__swapgs:
1251    swapgs
1252    ret
1253
1254#endif
1255
1256END
1257